diff --git a/channels/chan_iax2.c b/channels/chan_iax2.c
index 2d9c291db2..1722411be7 100644
--- a/channels/chan_iax2.c
+++ b/channels/chan_iax2.c
@@ -7613,10 +7613,10 @@ static int check_access(int callno, struct sockaddr_in *sin, struct iax_ies *ies
 	i = ao2_iterator_init(users, 0);
 	while ((user = ao2_iterator_next(&i))) {
 		if ((ast_strlen_zero(iaxs[callno]->username) ||				/* No username specified */
-			!strcmp(iaxs[callno]->username, user->name))	/* Or this username specified */
-			&& ast_apply_ha(user->ha, &addr) 	/* Access is permitted from this IP */
+			!strcmp(iaxs[callno]->username, user->name))			/* Or this username specified */
+			&& ast_apply_ha(user->ha, &addr) == AST_SENSE_ALLOW		/* Access is permitted from this IP */
 			&& (ast_strlen_zero(iaxs[callno]->context) ||			/* No context specified */
-			     apply_context(user->contexts, iaxs[callno]->context))) {			/* Context is permitted */
+				apply_context(user->contexts, iaxs[callno]->context))) {			/* Context is permitted */
 			if (!ast_strlen_zero(iaxs[callno]->username)) {
 				/* Exact match, stop right now. */
 				if (best)
@@ -7672,8 +7672,9 @@ static int check_access(int callno, struct sockaddr_in *sin, struct iax_ies *ies
 	user = best;
 	if (!user && !ast_strlen_zero(iaxs[callno]->username)) {
 		user = realtime_user(iaxs[callno]->username, sin);
-		if (user && !ast_strlen_zero(iaxs[callno]->context) &&			/* No context specified */
-		    !apply_context(user->contexts, iaxs[callno]->context)) {		/* Context is permitted */
+		if (user && (ast_apply_ha(user->ha, &addr) == AST_SENSE_DENY		/* Access is denied from this IP */
+			|| (!ast_strlen_zero(iaxs[callno]->context) &&					/* No context specified */
+				!apply_context(user->contexts, iaxs[callno]->context)))) {	/* Context is permitted */
 			user = user_unref(user);
 		}
 	}