diff --git a/daemon/call.c b/daemon/call.c
index 71db8f46a..92b866103 100644
--- a/daemon/call.c
+++ b/daemon/call.c
@@ -2837,6 +2837,7 @@ static void __call_monologue_init_from_flags(struct call_monologue *ml, struct c
 	__tos_change(call, flags);
 
 	if (flags->label.s) {
+		t_hash_table_remove(call->labels, &ml->label);
 		ml->label = call_str_cpy(&flags->label);
 		t_hash_table_replace(call->labels, &ml->label, ml);
 	}
diff --git a/daemon/call_interfaces.c b/daemon/call_interfaces.c
index 6a69c5f1f..6b2b6c448 100644
--- a/daemon/call_interfaces.c
+++ b/daemon/call_interfaces.c
@@ -1403,6 +1403,7 @@ static const char *media_block_match(call_t **call, struct call_monologue **mono
 
 	// for generic ops, handle set-label here if given
 	if (IS_OP_OTHER(flags->opmode) && flags->set_label.len && *monologue) {
+		t_hash_table_remove((*call)->labels, &(*monologue)->label);
 		(*monologue)->label = call_str_cpy(&flags->set_label);
 		t_hash_table_replace((*call)->labels, &(*monologue)->label, *monologue);
 	}