MT#55283 redis: write call data to a master only

Closes #2212

--redis-write accepts any role, so after a failover that demotes its
node, rtpengine reconnects to the same address, accepts the replica,
and every later write is refused with READONLY. --redis alone requires
the master at connect, but keeps the connection when its writes are
refused.

The write connection now requires the master role, as --redis alone
does, and a connection that requires it is dropped on a write refused
with READONLY or MASTERDOWN, so that the next operation reconnects and
checks the role. Other refusals keep the connection. The read database
of --redis, when --redis-write is given, still accepts any role.

Tested by auto-daemon-tests-redis-master-only.pl; the READONLY cases
of auto-daemon-tests-redis-errors.pl, which expected the connection to be
kept, now use NOREPLICAS.

Change-Id: Ie164c2c3495c6c8b77553caf9a449e40f8e13699
pull/2237/head
Kevin Binetruy 1 month ago committed by Richard Fuchs
parent aa813124da
commit f3003b4b9d

@ -1915,7 +1915,7 @@ static void create_everything(void) {
rtpe_config.redis_write_db,
rtpe_config.redis_write_hostname,
rtpe_config.redis_write_auth,
ANY_REDIS_ROLE,
MASTER_REDIS_ROLE,
rtpe_config.no_redis_required,
rtpe_config.redis_resolve_on_reconnect);

@ -247,11 +247,21 @@ static void redis_write_result(struct redis *r, const redisReply *rp) {
r->write_errors = 0;
}
/* an error reply that says the server is not, or no longer, a usable master */
static bool redis_is_role_error(const redisReply *rp) {
if (rp->type != REDIS_REPLY_ERROR)
return false;
return !strncmp(rp->str, "READONLY", 8) || !strncmp(rp->str, "MASTERDOWN", 10);
}
/* called with r->lock held
* Returns -1 if the replies could not be read (the context is then unusable).
* An error reply keeps the connection: a reconnect would get the same answer. */
* Returns -1 if the replies could not be read (the context is then unusable),
* or after a role error on a connection that requires a master, so that the
* caller drops it and the reconnect checks the role. Other error replies keep
* the connection: a reconnect would get the same answer. */
static int redis_consume(struct redis *r) {
redisReply *rp;
bool role_error = false;
if (!r->ctx) {
ilog(LOG_ERROR, "Unable to consume pipelined replies. No redis context");
@ -265,8 +275,15 @@ static int redis_consume(struct redis *r) {
}
r->pipeline--;
redis_write_result(r, rp);
if (redis_is_role_error(rp))
role_error = true;
freeReplyObject(rp);
}
if (role_error && r->role == MASTER_REDIS_ROLE) {
rlog(LOG_WARN, "Redis %s is not a master: dropping the connection",
endpoint_print_buf(&r->endpoint));
return -1;
}
return 0;
}

@ -571,14 +571,18 @@ call to inject-DTMF won't be sent to __\-\-dtmf-log-dest=__ or __\-\-listen-tcp-
For password protected Redis servers, the environment variable for the
password is __RTPENGINE\_REDIS\_WRITE\_AUTH\_PW__.
When both options are given, __rtpengine__ will start and use the Redis
database regardless of the database's role (master or slave).
When both options are given, __rtpengine__ will start and use the first
database regardless of its role (master or slave). The write database
must be the master, as the first one must be when it is used alone.
Writes that the Redis server refuses (for example __READONLY__ from a
replica, or __NOREPLICAS__ from a master that lacks replicas) are logged,
at most once a second, and counted in the __rediswriteerrors__ statistic.
The connection is kept, as reconnecting to the same address would not
change the server's answer.
A write refused with __READONLY__ or __MASTERDOWN__ drops the connection:
the server is no longer a master, and the next operation reconnects and
checks the role (a replica is then a failed attempt, see
__\-\-redis-reconnect-pause__). Other refusals keep the connection, as
reconnecting to the same address would not change the server's answer.
- __\-\-redis-subscribe=__\[*PW*@\]*IP*:*PORT*\[/*INT*\]

@ -137,6 +137,7 @@ include ../lib/common.Makefile
daemon-tests-inject \
daemon-tests-dtx-dtmf \
daemon-tests-ice-force-answer \
daemon-tests-redis-master-only \
#
TESTS := test-bitstr test-str aes-crypt aead-aes-crypt test-const_str_hash.strhash
@ -236,6 +237,7 @@ daemon-tests: \
daemon-tests-dtmf-fanout \
daemon-tests-video \
daemon-tests-inject \
daemon-tests-redis-master-only \
#
daemon-test-deps: tests-preload.so
@ -355,6 +357,9 @@ daemon-tests-redis-destroyed: daemon-test-deps
daemon-tests-redis-reconnect: daemon-test-deps
./auto-test-helper "$@" perl -I../perl auto-daemon-tests-redis-reconnect.pl
daemon-tests-redis-master-only: daemon-test-deps
./auto-test-helper "$@" perl -I../perl auto-daemon-tests-redis-master-only.pl
daemon-tests-audio-player: daemon-test-deps
./auto-test-helper "$@" perl -I../perl auto-daemon-tests-audio-player.pl

@ -62,11 +62,11 @@ is(redis_write_errors(), 0, 'no write error');
# writes refused by the server are counted, and the connection is kept:
# reconnecting to the same address cannot help
# reconnecting to the same address cannot help (a role error, which drops
# it, is in auto-daemon-tests-redis-master-only.pl)
my $count = 0;
for my $err ("READONLY You can't write against a read only replica.",
"NOREPLICAS Not enough good replicas to write.",
for my $err ("NOREPLICAS Not enough good replicas to write.",
"LOADING Redis is loading the dataset in memory")
{
my ($code) = $err =~ /^(\w+)/;
@ -77,7 +77,7 @@ for my $err ("READONLY You can't write against a read only replica.",
$count++;
is(redis_write_errors(), $count, "$code counted");
like(rtpe_log(), qr/ERR: .*Redis 203\.0\.113\.42:6379 refused a write: \Q$err\E \(1 refused since the last report\)/,
'first refused write logged at once') if $code eq 'READONLY';
'first refused write logged at once') if $code eq 'NOREPLICAS';
# the next write goes through the same connection
new_call;
@ -93,7 +93,7 @@ for my $err ("READONLY You can't write against a read only replica.",
# a refused DEL is counted too
$NGCP::Rtpengine::req_cb = sub {
redis_expect_del($fd, cid(), "-READONLY You can't write against a read only replica.\r\n", 'refused');
redis_expect_del($fd, cid(), "-NOREPLICAS Not enough good replicas to write.\r\n", 'refused');
};
rtpe_req('delete', 'refused delete', { 'delete delay' => 0 });
$count++;

@ -0,0 +1,145 @@
#!/usr/bin/perl
# The write connection (--redis-write) requires the master role, and a role
# error (READONLY, MASTERDOWN) on it drops the connection, so that the
# reconnect checks the role again. The fake Redis plays a master that is
# demoted to a replica, then promoted again.
use strict;
use warnings;
use NGCP::Rtpengine::Test;
use NGCP::Rtpengine::AutoTest;
use NGCP::Rtpengine::FakeRedis;
use Test::More;
my $listener = redis_listen('203.0.113.42');
my ($write_fd, $read_fd);
my $sdp = <<SDP;
v=0
o=- 1545997027 1 IN IP4 198.51.101.40
s=tester
t=0 0
m=audio 3000 RTP/AVP 0 8
c=IN IP4 198.51.100.1
----------------------------------
v=0
o=- 1545997027 1 IN IP4 198.51.101.40
s=tester
t=0 0
m=audio PORT RTP/AVP 0 8
c=IN IP4 203.0.113.1
a=rtpmap:0 PCMU/8000
a=rtpmap:8 PCMA/8000
a=sendrecv
a=rtcp:PORT
SDP
$NGCP::Rtpengine::AutoTest::launch_cb = sub {
# the write connection first, then the read connection, which restores
$write_fd = redis_conn($listener, 'startup write');
$read_fd = redis_conn($listener, 'startup read');
redis_expect($read_fd, ['PING'], "+PONG\r\n", 'startup PING');
redis_expect($read_fd, ['KEYS', '*'], "*0\r\n", 'startup KEYS');
};
autotest_start(qw(--config-file=none -t -1 -i 203.0.113.1
-n 2223 -f -L 7 -E
--redis=auth@203.0.113.42:6379/2 --redis-write=auth@203.0.113.42:6379/2
--redis-cmd-timeout=1000))
or die;
# a write that succeeds, then a refusal that is not about the role, which
# keeps the connection
new_call;
$NGCP::Rtpengine::req_cb = sub { redis_expect_set($write_fd, cid(), "+OK\r\n", 'ok') };
offer('accepted write', { }, $sdp);
new_call;
$NGCP::Rtpengine::req_cb = sub {
redis_expect_set($write_fd, cid(), "-NOREPLICAS Not enough good replicas to write.\r\n", 'NOREPLICAS');
};
offer('NOREPLICAS write', { }, $sdp);
is(redis_write_errors(), 1, 'NOREPLICAS counted');
new_call;
$NGCP::Rtpengine::req_cb = sub {
ok(!readable($listener, 0.2), 'no reconnect after NOREPLICAS');
redis_expect_set($write_fd, cid(), "+OK\r\n", 'after NOREPLICAS');
};
offer('write after NOREPLICAS', { }, $sdp);
# READONLY: the server has become a replica. The write is counted, and the
# connection is dropped at once.
new_call;
$NGCP::Rtpengine::req_cb = sub {
redis_expect_set($write_fd, cid(), "-READONLY You can't write against a read only replica.\r\n",
'READONLY');
redis_closed($write_fd, 'READONLY: connection dropped');
};
offer('READONLY write', { }, $sdp);
is(redis_write_errors(), 2, 'READONLY counted');
like(rtpe_log(), qr/Redis 203\.0\.113\.42:6379 is not a master: dropping the connection/,
'drop logged');
# The next write reconnects, finds a replica, and gives up after one INFO;
# no new attempt follows within the next second, and the one after it finds
# the master again.
new_call;
$NGCP::Rtpengine::req_cb = sub {
$write_fd = redis_conn($listener, 'replica', role => 'slave', refused => 1);
redis_closed($write_fd, 'replica: connection closed after one INFO');
};
offer('write while the node is a replica', { }, $sdp);
new_call;
$NGCP::Rtpengine::req_cb = sub {
ok(!readable($listener, 0.3), 'no reconnect within the pacing interval')
or die "reconnect attempted within the pacing interval\n";
};
offer('write within the pacing interval', { }, $sdp);
sleep(1);
new_call;
$NGCP::Rtpengine::req_cb = sub {
$write_fd = redis_conn($listener, 'master again');
redis_expect_set($write_fd, cid(), "+OK\r\n", 'master again', no_ping => 1);
};
offer('write after the pacing interval', { }, $sdp);
# MASTERDOWN drops the connection too; here the reconnect finds a master
new_call;
$NGCP::Rtpengine::req_cb = sub {
redis_expect_set($write_fd, cid(),
"-MASTERDOWN Link with MASTER is down and replica-serve-stale-data is set to 'no'.\r\n",
'MASTERDOWN');
redis_closed($write_fd, 'MASTERDOWN: connection dropped');
};
offer('MASTERDOWN write', { }, $sdp);
is(redis_write_errors(), 3, 'MASTERDOWN counted');
new_call;
$NGCP::Rtpengine::req_cb = sub {
$write_fd = redis_conn($listener, 'after MASTERDOWN');
redis_expect_set($write_fd, cid(), "+OK\r\n", 'after MASTERDOWN', no_ping => 1);
};
offer('write after MASTERDOWN', { }, $sdp);
# the read connection was never touched
ok(!readable($read_fd, 0), 'nothing on the read connection');
undef($NGCP::Rtpengine::req_cb);
done_testing();
Loading…
Cancel
Save