diff --git a/daemon/main.c b/daemon/main.c
index e8ffc6171..8d8663332 100644
--- a/daemon/main.c
+++ b/daemon/main.c
@@ -531,7 +531,7 @@ static void create_listeners(const GQueue *endpoints_in, GQueue *objects_out,
 	for (GList *l = endpoints_in->head; l; l = l->next) {
 		endpoint_t *e = l->data;
 		if (exclude_port)
-			interfaces_exclude_port(e->port);
+			interfaces_exclude_port(e);
 		void *o = constructor(e);
 		if (!o)
 			die("Failed to open %s connection port (%s): %s",
diff --git a/daemon/media_socket.c b/daemon/media_socket.c
index 2d5972b4f..75558134f 100644
--- a/daemon/media_socket.c
+++ b/daemon/media_socket.c
@@ -893,7 +893,7 @@ void interfaces_init(intf_config_q *interfaces) {
 	local_media_socket_endpoints = local_sockets_ht_new();
 }
 
-void interfaces_exclude_port(unsigned int port) {
+void interfaces_exclude_port(endpoint_t *e) {
 	struct intf_spec *spec;
 
 	struct port_pool *pp;
@@ -901,12 +901,19 @@ void interfaces_exclude_port(unsigned int port) {
 	intf_spec_ht_iter iter;
 	t_hash_table_iter_init(&iter, __intf_spec_addr_type_hash);
 	while (t_hash_table_iter_next(&iter, NULL, &spec)) {
+		if (e->address.family != spec->local_address.addr.family)
+			continue;
+		if (!is_addr_unspecified(&e->address)) {
+			if (!sockaddr_eq(&e->address, &spec->local_address.addr))
+				continue;
+		}
+
 		pp = &spec->port_pool;
-		if (port < pp->min || port > pp->max)
+		if (e->port < pp->min || e->port > pp->max)
 			continue;
 
 		mutex_lock(&pp->free_list_lock);
-		__auto_type ll = free_ports_link(pp, port);
+		__auto_type ll = free_ports_link(pp, e->port);
 		if (ll) {
 			reserve_port(pp, ll);
 			t_list_free(ll);
diff --git a/include/media_socket.h b/include/media_socket.h
index 36bdcd6d6..1ee808e04 100644
--- a/include/media_socket.h
+++ b/include/media_socket.h
@@ -285,7 +285,7 @@ void interfaces_free(void);
 struct logical_intf *get_logical_interface(const str *name, sockfamily_t *fam, int num_ports);
 struct local_intf *get_interface_address(const struct logical_intf *lif, sockfamily_t *fam);
 struct local_intf *get_any_interface_address(const struct logical_intf *lif, sockfamily_t *fam);
-void interfaces_exclude_port(unsigned int port);
+void interfaces_exclude_port(endpoint_t *);
 int is_local_endpoint(const struct intf_address *addr, unsigned int port);
 
 int __get_consecutive_ports(socket_port_q *out, unsigned int num_ports, unsigned int wanted_start_port,