diff --git a/daemon/call.c b/daemon/call.c
index 12f4f6f91..02379b39d 100644
--- a/daemon/call.c
+++ b/daemon/call.c
@@ -2820,6 +2820,7 @@ static void __call_monologue_init_from_flags(struct call_monologue *ml, struct c
 	__tos_change(call, flags);
 
 	if (flags->label.s) {
+		t_hash_table_remove(call->labels, &ml->label);
 		ml->label = call_str_cpy(&flags->label);
 		t_hash_table_replace(call->labels, &ml->label, ml);
 	}
diff --git a/daemon/call_interfaces.c b/daemon/call_interfaces.c
index 5b7b97fc9..4f1af0cda 100644
--- a/daemon/call_interfaces.c
+++ b/daemon/call_interfaces.c
@@ -3370,6 +3370,7 @@ static const char *media_block_match(call_t **call, struct call_monologue **mono
 
 	// for generic ops, handle set-label here if given
 	if (IS_OP_OTHER(flags->opmode) && flags->set_label.len && *monologue) {
+		t_hash_table_remove((*call)->labels, &(*monologue)->label);
 		(*monologue)->label = call_str_cpy(&flags->set_label);
 		t_hash_table_replace((*call)->labels, &(*monologue)->label, *monologue);
 	}