From f3003b4b9d320226132685f769b86b030842e9bb Mon Sep 17 00:00:00 2001 From: Kevin Binetruy Date: Wed, 14 Oct 2026 10:52:21 +0000 Subject: [PATCH] MT#55283 redis: write call data to a master only Closes #2212 --redis-write accepts any role, so after a failover that demotes its node, rtpengine reconnects to the same address, accepts the replica, and every later write is refused with READONLY. --redis alone requires the master at connect, but keeps the connection when its writes are refused. The write connection now requires the master role, as --redis alone does, and a connection that requires it is dropped on a write refused with READONLY or MASTERDOWN, so that the next operation reconnects and checks the role. Other refusals keep the connection. The read database of --redis, when --redis-write is given, still accepts any role. Tested by auto-daemon-tests-redis-master-only.pl; the READONLY cases of auto-daemon-tests-redis-errors.pl, which expected the connection to be kept, now use NOREPLICAS. Change-Id: Ie164c2c3495c6c8b77553caf9a449e40f8e13699 --- daemon/main.c | 2 +- daemon/redis.c | 21 +++- docs/rtpengine.md | 12 +- t/Makefile | 5 + t/auto-daemon-tests-redis-errors.pl | 10 +- t/auto-daemon-tests-redis-master-only.pl | 145 +++++++++++++++++++++++ 6 files changed, 183 insertions(+), 12 deletions(-) create mode 100755 t/auto-daemon-tests-redis-master-only.pl diff --git a/daemon/main.c b/daemon/main.c index cb1a0a220..b88deb850 100644 --- a/daemon/main.c +++ b/daemon/main.c @@ -1915,7 +1915,7 @@ static void create_everything(void) { rtpe_config.redis_write_db, rtpe_config.redis_write_hostname, rtpe_config.redis_write_auth, - ANY_REDIS_ROLE, + MASTER_REDIS_ROLE, rtpe_config.no_redis_required, rtpe_config.redis_resolve_on_reconnect); diff --git a/daemon/redis.c b/daemon/redis.c index 7ea5ef3b2..2a391015b 100644 --- a/daemon/redis.c +++ b/daemon/redis.c @@ -247,11 +247,21 @@ static void redis_write_result(struct redis *r, const redisReply *rp) { r->write_errors = 0; } +/* an error reply that says the server is not, or no longer, a usable master */ +static bool redis_is_role_error(const redisReply *rp) { + if (rp->type != REDIS_REPLY_ERROR) + return false; + return !strncmp(rp->str, "READONLY", 8) || !strncmp(rp->str, "MASTERDOWN", 10); +} + /* called with r->lock held - * Returns -1 if the replies could not be read (the context is then unusable). - * An error reply keeps the connection: a reconnect would get the same answer. */ + * Returns -1 if the replies could not be read (the context is then unusable), + * or after a role error on a connection that requires a master, so that the + * caller drops it and the reconnect checks the role. Other error replies keep + * the connection: a reconnect would get the same answer. */ static int redis_consume(struct redis *r) { redisReply *rp; + bool role_error = false; if (!r->ctx) { ilog(LOG_ERROR, "Unable to consume pipelined replies. No redis context"); @@ -265,8 +275,15 @@ static int redis_consume(struct redis *r) { } r->pipeline--; redis_write_result(r, rp); + if (redis_is_role_error(rp)) + role_error = true; freeReplyObject(rp); } + if (role_error && r->role == MASTER_REDIS_ROLE) { + rlog(LOG_WARN, "Redis %s is not a master: dropping the connection", + endpoint_print_buf(&r->endpoint)); + return -1; + } return 0; } diff --git a/docs/rtpengine.md b/docs/rtpengine.md index 6662f2b6c..a822083b5 100644 --- a/docs/rtpengine.md +++ b/docs/rtpengine.md @@ -571,14 +571,18 @@ call to inject-DTMF won't be sent to __\-\-dtmf-log-dest=__ or __\-\-listen-tcp- For password protected Redis servers, the environment variable for the password is __RTPENGINE\_REDIS\_WRITE\_AUTH\_PW__. - When both options are given, __rtpengine__ will start and use the Redis - database regardless of the database's role (master or slave). + When both options are given, __rtpengine__ will start and use the first + database regardless of its role (master or slave). The write database + must be the master, as the first one must be when it is used alone. Writes that the Redis server refuses (for example __READONLY__ from a replica, or __NOREPLICAS__ from a master that lacks replicas) are logged, at most once a second, and counted in the __rediswriteerrors__ statistic. - The connection is kept, as reconnecting to the same address would not - change the server's answer. + A write refused with __READONLY__ or __MASTERDOWN__ drops the connection: + the server is no longer a master, and the next operation reconnects and + checks the role (a replica is then a failed attempt, see + __\-\-redis-reconnect-pause__). Other refusals keep the connection, as + reconnecting to the same address would not change the server's answer. - __\-\-redis-subscribe=__\[*PW*@\]*IP*:*PORT*\[/*INT*\] diff --git a/t/Makefile b/t/Makefile index ebe73cb83..f344c73ed 100644 --- a/t/Makefile +++ b/t/Makefile @@ -137,6 +137,7 @@ include ../lib/common.Makefile daemon-tests-inject \ daemon-tests-dtx-dtmf \ daemon-tests-ice-force-answer \ + daemon-tests-redis-master-only \ # TESTS := test-bitstr test-str aes-crypt aead-aes-crypt test-const_str_hash.strhash @@ -236,6 +237,7 @@ daemon-tests: \ daemon-tests-dtmf-fanout \ daemon-tests-video \ daemon-tests-inject \ + daemon-tests-redis-master-only \ # daemon-test-deps: tests-preload.so @@ -355,6 +357,9 @@ daemon-tests-redis-destroyed: daemon-test-deps daemon-tests-redis-reconnect: daemon-test-deps ./auto-test-helper "$@" perl -I../perl auto-daemon-tests-redis-reconnect.pl +daemon-tests-redis-master-only: daemon-test-deps + ./auto-test-helper "$@" perl -I../perl auto-daemon-tests-redis-master-only.pl + daemon-tests-audio-player: daemon-test-deps ./auto-test-helper "$@" perl -I../perl auto-daemon-tests-audio-player.pl diff --git a/t/auto-daemon-tests-redis-errors.pl b/t/auto-daemon-tests-redis-errors.pl index 72dee65ed..2fd642f1b 100755 --- a/t/auto-daemon-tests-redis-errors.pl +++ b/t/auto-daemon-tests-redis-errors.pl @@ -62,11 +62,11 @@ is(redis_write_errors(), 0, 'no write error'); # writes refused by the server are counted, and the connection is kept: -# reconnecting to the same address cannot help +# reconnecting to the same address cannot help (a role error, which drops +# it, is in auto-daemon-tests-redis-master-only.pl) my $count = 0; -for my $err ("READONLY You can't write against a read only replica.", - "NOREPLICAS Not enough good replicas to write.", +for my $err ("NOREPLICAS Not enough good replicas to write.", "LOADING Redis is loading the dataset in memory") { my ($code) = $err =~ /^(\w+)/; @@ -77,7 +77,7 @@ for my $err ("READONLY You can't write against a read only replica.", $count++; is(redis_write_errors(), $count, "$code counted"); like(rtpe_log(), qr/ERR: .*Redis 203\.0\.113\.42:6379 refused a write: \Q$err\E \(1 refused since the last report\)/, - 'first refused write logged at once') if $code eq 'READONLY'; + 'first refused write logged at once') if $code eq 'NOREPLICAS'; # the next write goes through the same connection new_call; @@ -93,7 +93,7 @@ for my $err ("READONLY You can't write against a read only replica.", # a refused DEL is counted too $NGCP::Rtpengine::req_cb = sub { - redis_expect_del($fd, cid(), "-READONLY You can't write against a read only replica.\r\n", 'refused'); + redis_expect_del($fd, cid(), "-NOREPLICAS Not enough good replicas to write.\r\n", 'refused'); }; rtpe_req('delete', 'refused delete', { 'delete delay' => 0 }); $count++; diff --git a/t/auto-daemon-tests-redis-master-only.pl b/t/auto-daemon-tests-redis-master-only.pl new file mode 100755 index 000000000..4e9603d93 --- /dev/null +++ b/t/auto-daemon-tests-redis-master-only.pl @@ -0,0 +1,145 @@ +#!/usr/bin/perl + +# The write connection (--redis-write) requires the master role, and a role +# error (READONLY, MASTERDOWN) on it drops the connection, so that the +# reconnect checks the role again. The fake Redis plays a master that is +# demoted to a replica, then promoted again. + +use strict; +use warnings; +use NGCP::Rtpengine::Test; +use NGCP::Rtpengine::AutoTest; +use NGCP::Rtpengine::FakeRedis; +use Test::More; + + +my $listener = redis_listen('203.0.113.42'); +my ($write_fd, $read_fd); + +my $sdp = < 'slave', refused => 1); + redis_closed($write_fd, 'replica: connection closed after one INFO'); +}; +offer('write while the node is a replica', { }, $sdp); + +new_call; +$NGCP::Rtpengine::req_cb = sub { + ok(!readable($listener, 0.3), 'no reconnect within the pacing interval') + or die "reconnect attempted within the pacing interval\n"; +}; +offer('write within the pacing interval', { }, $sdp); + +sleep(1); + +new_call; +$NGCP::Rtpengine::req_cb = sub { + $write_fd = redis_conn($listener, 'master again'); + redis_expect_set($write_fd, cid(), "+OK\r\n", 'master again', no_ping => 1); +}; +offer('write after the pacing interval', { }, $sdp); + + +# MASTERDOWN drops the connection too; here the reconnect finds a master + +new_call; +$NGCP::Rtpengine::req_cb = sub { + redis_expect_set($write_fd, cid(), + "-MASTERDOWN Link with MASTER is down and replica-serve-stale-data is set to 'no'.\r\n", + 'MASTERDOWN'); + redis_closed($write_fd, 'MASTERDOWN: connection dropped'); +}; +offer('MASTERDOWN write', { }, $sdp); +is(redis_write_errors(), 3, 'MASTERDOWN counted'); + +new_call; +$NGCP::Rtpengine::req_cb = sub { + $write_fd = redis_conn($listener, 'after MASTERDOWN'); + redis_expect_set($write_fd, cid(), "+OK\r\n", 'after MASTERDOWN', no_ping => 1); +}; +offer('write after MASTERDOWN', { }, $sdp); + + +# the read connection was never touched +ok(!readable($read_fd, 0), 'nothing on the read connection'); + + +undef($NGCP::Rtpengine::req_cb); + +done_testing();