diff --git a/daemon/call.c b/daemon/call.c
index fbaee346d..c9f5fbc4f 100644
--- a/daemon/call.c
+++ b/daemon/call.c
@@ -1445,7 +1445,9 @@ static void callmaster_timer(void *ptr) {
 		if (sink) {
 			mutex_lock(&sink->out_lock);
 			if (sink->crypto.params.crypto_suite
-					&& ke->target.encrypt.last_index - sink->crypto.last_index > 0x4000) {
+					&& ke->target.ssrc == sink->crypto.ssrc
+					&& ke->target.encrypt.last_index - sink->crypto.last_index > 0x4000)
+			{
 				sink->crypto.last_index = ke->target.encrypt.last_index;
 				update = 1;
 			}
@@ -1454,7 +1456,9 @@ static void callmaster_timer(void *ptr) {
 
 		mutex_lock(&ps->in_lock);
 		if (sfd->crypto.params.crypto_suite
-				&& ke->target.decrypt.last_index - sfd->crypto.last_index > 0x4000) {
+				&& ke->target.ssrc == sfd->crypto.ssrc
+				&& ke->target.decrypt.last_index - sfd->crypto.last_index > 0x4000)
+		{
 			sfd->crypto.last_index = ke->target.decrypt.last_index;
 			update = 1;
 		}