diff --git a/res/res_rtp_asterisk.c b/res/res_rtp_asterisk.c
index 11238e99ef..68b00d981d 100644
--- a/res/res_rtp_asterisk.c
+++ b/res/res_rtp_asterisk.c
@@ -1218,7 +1218,6 @@ static int dtls_details_initialize(struct dtls_details *dtls, SSL_CTX *ssl_ctx,
 	dtls->connection = AST_RTP_DTLS_CONNECTION_NEW;
 
 	ast_mutex_init(&dtls->lock);
-	dtls->timeout_timer = -1;
 
 	return 0;
 
@@ -2500,6 +2499,7 @@ static int ast_rtp_new(struct ast_rtp_instance *instance,
 
 #ifdef HAVE_OPENSSL_SRTP
 	rtp->rekeyid = -1;
+	rtp->dtls.timeout_timer = -1;
 #endif
 
 	return 0;
@@ -4584,6 +4584,7 @@ static void ast_rtp_prop_set(struct ast_rtp_instance *instance, enum ast_rtp_pro
 #endif
 
 #ifdef HAVE_OPENSSL_SRTP
+			rtp->rtcp->dtls.timeout_timer = -1;
 			dtls_setup_rtcp(instance);
 #endif