diff --git a/channels/chan_sip.c b/channels/chan_sip.c
index 993b93acc7..dad51eaee4 100644
--- a/channels/chan_sip.c
+++ b/channels/chan_sip.c
@@ -11514,10 +11514,8 @@ static void handle_response_invite(struct sip_pvt *p, int resp, char *rest, stru
 static void handle_response_refer(struct sip_pvt *p, int resp, char *rest, struct sip_request *req, int seqno)
 {
 	/* If no refer structure exists, then do nothing */
-	if (!p->refer) {
-		ast_set_flag(&p->flags[0], SIP_NEEDDESTROY);
+	if (!p->refer)
 		return;
-	}
 
 	switch (resp) {
 	case 202:   /* Transfer accepted */