diff --git a/daemon/call.c b/daemon/call.c
index 63ae3a14a..f1381d6a3 100644
--- a/daemon/call.c
+++ b/daemon/call.c
@@ -1904,9 +1904,13 @@ static void __fill_stream(struct packet_stream *ps, const struct endpoint *epp,
 
 	ps->endpoint = ep;
 	ps->advertised_endpoint = ep;
-	/* we reset crypto params whenever the endpoint changes */
-	crypto_reset(&ps->crypto);
-	dtls_shutdown(ps);
+
+	if (PS_ISSET(ps, FILLED)) {
+		/* we reset crypto params whenever the endpoint changes */
+		crypto_reset(&ps->crypto);
+		dtls_shutdown(ps);
+	}
+
 	PS_SET(ps, FILLED);
 }