diff --git a/daemon/call.c b/daemon/call.c
index d5e74a4ad..2375d3ba2 100644
--- a/daemon/call.c
+++ b/daemon/call.c
@@ -2809,6 +2809,7 @@ static void __call_monologue_init_from_flags(struct call_monologue *ml, struct c
 	__tos_change(call, flags);
 
 	if (flags->label.s) {
+		t_hash_table_remove(call->labels, &ml->label);
 		ml->label = call_str_cpy(&flags->label);
 		t_hash_table_replace(call->labels, &ml->label, ml);
 	}
diff --git a/daemon/call_interfaces.c b/daemon/call_interfaces.c
index 0a4eb1e38..8e83c6c3d 100644
--- a/daemon/call_interfaces.c
+++ b/daemon/call_interfaces.c
@@ -1482,6 +1482,7 @@ static const char *media_block_match(call_t **call, struct call_monologue **mono
 
 	// for generic ops, handle set-label here if given
 	if (IS_OP_OTHER(flags->opmode) && flags->set_label.len && *monologue) {
+		t_hash_table_remove((*call)->labels, &(*monologue)->label);
 		(*monologue)->label = call_str_cpy(&flags->set_label);
 		t_hash_table_replace((*call)->labels, &(*monologue)->label, *monologue);
 	}