diff --git a/daemon/media_socket.c b/daemon/media_socket.c
index fe230d544..0f1a952ce 100644
--- a/daemon/media_socket.c
+++ b/daemon/media_socket.c
@@ -3510,8 +3510,6 @@ static int stream_packet(struct packet_handler_ctx *phc) {
 	if (!phc->mp.stream->selected_sfd)
 		goto out;
 
-	CALL_CLEAR(phc->mp.call, FOREIGN_MEDIA);
-
 	if (CALL_ISSET(phc->mp.call, DROP_TRAFFIC))
 		goto drop;
 
@@ -3553,6 +3551,9 @@ static int stream_packet(struct packet_handler_ctx *phc) {
 	if (media_packet_address_check(phc))
 		goto drop;
 
+	// only media we accept ends the grace period after a takeover
+	CALL_CLEAR(phc->mp.call, FOREIGN_MEDIA);
+
 	PS_SET(phc->mp.stream, RECEIVED);
 
 	if (rtpe_config.active_switchover && IS_FOREIGN_CALL(phc->mp.call))