MT#55283 redis: keyspace subscription follows the Sentinel master

Closes #2212

A subscription made through "sentinel" stayed on the old master after a
failover, and missed the notifications of the new one.

With Sentinel, the notification event loop gets a one-second timer:
when the master changed, it closes the subscription, and
redis_notify_loop() subscribes again on the new master. The
subscription's context also gets a connect callback, since hiredis
frees a context that fails to connect without calling the disconnect
callback. Notifications missed during the switch are not replayed.

Tested by auto-daemon-tests-redis-sentinel-subscribe.pl.

Change-Id: Ia14d48e220bbf98cc48964e76f33ff2a497953e9
pull/2237/head
Kevin Binetruy 1 month ago committed by Richard Fuchs
parent 2c717d54e7
commit c54d3199b3

@ -656,6 +656,13 @@ void redis_delete_async_context_disconnect(const redisAsyncContext *redis_delete
}
void redis_notify_async_context_disconnect(const redisAsyncContext *redis_notify_async_context, int status) {
// forget the context, and let event_base_dispatch() return (the timer
// must not keep it running), so that redis_notify_loop() subscribes again
struct redis *r = rtpe_redis_notify;
if (r && r->notify_timer)
event_del(r->notify_timer);
if (r && r->async_ctx == redis_notify_async_context)
r->async_ctx = NULL;
if (status == REDIS_ERR) {
if (redis_notify_async_context->errstr) {
rlog(LOG_ERROR, "redis_notify_async_context_disconnect error %d on context free: %s",
@ -671,6 +678,12 @@ void redis_notify_async_context_disconnect(const redisAsyncContext *redis_notify
}
}
// hiredis frees a context that fails to connect without the disconnect callback
static void redis_notify_async_context_connect(const redisAsyncContext *ac, int status) {
if (status != REDIS_OK)
redis_notify_async_context_disconnect(ac, status);
}
// connect_cb = connect callback, disconnect_cb = disconnect callback
int redis_async_context_alloc(struct redis *r, void *connect_cb, void *disconnect_cb) {
// sanity checks
@ -868,8 +881,35 @@ static int redis_delete_async(struct redis *r) {
return 0;
}
/* With Sentinel, once a second: on a new master, close the subscription, so
* that event_base_dispatch() returns and redis_notify_loop() subscribes again.
* The timer is one-shot, re-armed only while nothing has changed: it never
* keeps the event loop running by itself. */
static void redis_notify_follow(evutil_socket_t fd, short what, void *p) {
struct redis *r = p;
struct timeval tv = timeval_from_us(1000000);
if (redis_sentinel_gen(r->sentinel) == r->notify_gen) {
evtimer_add(r->notify_timer, &tv);
return;
}
rlog(LOG_NOTICE, "Redis: the master of '%s' has changed: subscribing again; keyspace "
"notifications between the change and the new subscription are missed",
redis_sentinel_master_name(r->sentinel));
// forget the context first: one still connecting may be freed at once,
// with no callback; without a context, leave the loop
redisAsyncContext *ac = r->async_ctx;
r->async_ctx = NULL;
if (ac)
redisAsyncDisconnect(ac);
else
event_base_loopbreak(r->async_ev);
}
static int redis_notify(struct redis *r) {
GList *l;
int ret = 0;
if (!r) {
rlog(LOG_ERROR, "redis_notify database is NULL on redis_notify()");
@ -917,13 +957,34 @@ static int redis_notify(struct redis *r) {
}
rwlock_unlock_r(&rtpe_config.keyspaces_lock);
if (r->sentinel) {
struct timeval tv = timeval_from_us(1000000);
r->notify_timer = evtimer_new(r->async_ev, redis_notify_follow, r);
if (r->notify_timer)
evtimer_add(r->notify_timer, &tv);
}
// dispatch event base => thread blocks here
if (event_base_dispatch(r->async_ev) < 0) {
rlog(LOG_ERROR, "Fail event_base_dispatch()");
return -1;
ret = -1;
}
return 0;
if (r->notify_timer) {
event_free(r->notify_timer);
r->notify_timer = NULL;
}
return ret;
}
/* the subscription's context, with the generation of the master it is for */
static int redis_notify_context_alloc(struct redis *r) {
LOCK(&r->lock);
redis_sentinel_follow(r);
r->notify_gen = r->gen;
return redis_async_context_alloc(r, redis_notify_async_context_connect,
redis_notify_async_context_disconnect);
}
/* redis_check_conn() needs the lock: the CLI uses the same connection, and
@ -982,8 +1043,12 @@ void redis_notify_loop(void *d) {
return ;
}
// with Sentinel and no master known yet, the loop below subscribes once
// there is one
bool now = !r->sentinel || r->gen;
// alloc redis async context
if (redis_async_context_alloc(r, NULL, redis_notify_async_context_disconnect) < 0) {
if (now && redis_notify_context_alloc(r) < 0) {
return ;
}
@ -993,7 +1058,7 @@ void redis_notify_loop(void *d) {
}
// initial redis_notify
if (redis_check_conn_locked(r) == REDIS_STATE_CONNECTED) {
if (now && redis_check_conn_locked(r) == REDIS_STATE_CONNECTED) {
redis_notify_return = redis_notify(r);
}
@ -1010,7 +1075,7 @@ void redis_notify_loop(void *d) {
if (redis_check_conn_locked(r) == REDIS_STATE_CONNECTED || redis_notify_return < 0) {
r->async_ctx = NULL;
// alloc new redis async context upon redis breakdown
if (redis_async_context_alloc(r, NULL, redis_notify_async_context_disconnect) < 0) {
if (redis_notify_context_alloc(r) < 0) {
continue;
}
@ -1024,7 +1089,8 @@ void redis_notify_loop(void *d) {
redis_notify_subscribe_action(r, UNSUBSCRIBE_ALL, 0);
// free async context
redisAsyncDisconnect(r->async_ctx);
if (r->async_ctx)
redisAsyncDisconnect(r->async_ctx);
r->async_ctx = NULL;
}
}

@ -740,8 +740,9 @@ call to inject-DTMF won't be sent to __\-\-dtmf-log-dest=__ or __\-\-listen-tcp-
never an older one than the master in use: if all are older (Sentinels
redeployed without their state), an error is logged and __rtpengine__
must be restarted. The master is checked with __ROLE__, then every
connection through Sentinel moves to it at its next operation. These
connections require the master role, as the write connection does. Set
connection through Sentinel moves to it at its next operation, a
keyspace subscription within a second. These connections require the
master role, as the write connection does. Set
__\-\-redis-cmd-timeout__ (for example to 1000): without it, a master
that stops answering holds Redis operations until it answers again.

@ -16,6 +16,7 @@
#include "dtls.h"
struct redis_sentinel;
struct event;
#define REDIS_RESTORE_NUM_THREADS 4
@ -84,6 +85,8 @@ struct redis {
struct redis_sentinel *sentinel; /* address from Redis Sentinel, or NULL */
uint64_t gen; /* generation of the master in endpoint */
struct event *notify_timer; /* subscriber: checks the generation */
uint64_t notify_gen; /* generation the subscription was made at */
};
struct redis_hash {

@ -140,6 +140,7 @@ include ../lib/common.Makefile
daemon-tests-redis-master-only \
daemon-tests-redis-sentinel \
daemon-tests-redis-sentinel-start \
daemon-tests-redis-sentinel-subscribe \
#
TESTS := test-bitstr test-str aes-crypt aead-aes-crypt test-const_str_hash.strhash
@ -242,6 +243,7 @@ daemon-tests: \
daemon-tests-redis-master-only \
daemon-tests-redis-sentinel \
daemon-tests-redis-sentinel-start \
daemon-tests-redis-sentinel-subscribe \
#
daemon-test-deps: tests-preload.so
@ -370,6 +372,9 @@ daemon-tests-redis-sentinel: daemon-test-deps
daemon-tests-redis-sentinel-start: daemon-test-deps
./auto-test-helper "$@" perl -I../perl auto-daemon-tests-redis-sentinel-start.pl
daemon-tests-redis-sentinel-subscribe: daemon-test-deps
./auto-test-helper "$@" perl -I../perl auto-daemon-tests-redis-sentinel-subscribe.pl
daemon-tests-audio-player: daemon-test-deps
./auto-test-helper "$@" perl -I../perl auto-daemon-tests-audio-player.pl

@ -0,0 +1,147 @@
#!/usr/bin/perl
# Redis Sentinel and keyspace notifications: the subscription follows the
# master named by the Sentinels, and is made again on each new master.
use strict;
use warnings;
use NGCP::Rtpengine::Test;
use NGCP::Rtpengine::AutoTest;
use NGCP::Rtpengine::FakeRedis;
use Test::More;
my %l = map { $_ => redis_listen("203.0.113.$_") } (42, 44);
sentinels_start('spw', '203.0.113.43' => '203.0.113.42 6379 1', '203.0.113.45' => '203.0.113.42 6379 1');
# a missing reply fails the test with the daemon's last log lines
local $SIG{ALRM} = sub { die "timeout\n" };
# a role probe can give up before it is answered
local $SIG{PIPE} = 'IGNORE';
my $sdp = <<SDP;
v=0
o=- 1545997027 1 IN IP4 198.51.101.40
s=tester
t=0 0
m=audio 3000 RTP/AVP 0 8
c=IN IP4 198.51.100.1
----------------------------------
v=0
o=- 1545997027 1 IN IP4 198.51.101.40
s=tester
t=0 0
m=audio PORT RTP/AVP 0 8
c=IN IP4 203.0.113.1
a=rtpmap:0 PCMU/8000
a=rtpmap:8 PCMA/8000
a=sendrecv
a=rtcp:PORT
SDP
my $psubscribed = "*3\r\n\$10\r\npsubscribe\r\n\$16\r\n__keyspace\@3__:*\r\n:1\r\n";
# the subscription's connection: AUTH, then psubscribe
sub subscription {
my ($l, $n) = @_;
my $fd = redis_accept($l);
redis_expect($fd, ['AUTH', 'auth'], "+OK\r\n", "$n AUTH");
redis_expect($fd, ['psubscribe', '__keyspace@3__:*'], $psubscribed, "$n psubscribe");
return $fd;
}
my ($main_fd, $notify_fd, $sub_fd);
$NGCP::Rtpengine::AutoTest::launch_cb = sub {
# discovery (Sentinels, then a role probe on .42), then the main and
# notification connections
$main_fd = redis_conn($l{42}, 'startup main');
$notify_fd = redis_conn($l{42}, 'startup notify');
# two connections from two threads: the subscription, and the restore
# thread's copy of the notification connection
my @fds = map { redis_accept($l{42}) } (0, 1);
# the notification thread checks its connection before subscribing
redis_expect($notify_fd, ['PING'], "+PONG\r\n", 'startup notify PING');
for my $fd (@fds) {
redis_expect($fd, ['AUTH', 'auth'], "+OK\r\n", 'startup AUTH');
my $c = redis_read($fd);
if ($c->[0] eq 'psubscribe') {
$sub_fd = $fd;
is(join(' ', @$c), 'psubscribe __keyspace@3__:*', 'startup psubscribe');
redis_send($fd, $psubscribed);
next;
}
is(join(' ', @$c), 'SELECT 2', 'restore copy SELECT');
redis_send($fd, "+OK\r\n");
redis_expect($fd, ['INFO'], info_reply('master'), 'restore copy INFO');
redis_expect($fd, ['TYPE', 'calls'], "+none\r\n", 'restore copy TYPE');
redis_expect($fd, ['PING'], "+PONG\r\n", 'restore copy PING');
redis_expect($fd, ['SELECT', '3'], "+OK\r\n", 'restore copy SELECT 3');
redis_expect($fd, ['KEYS', '*'], "*0\r\n", 'restore copy KEYS');
redis_expect($fd, ['SELECT', '2'], "+OK\r\n", 'restore copy SELECT back');
}
# own calls
redis_expect($main_fd, ['PING'], "+PONG\r\n", 'restore PING');
redis_expect($main_fd, ['KEYS', '*'], "*0\r\n", 'restore KEYS');
};
autotest_start(qw(--config-file=none -t -1 -i 203.0.113.1
-n 2223 -f -L 7 -E
--redis=auth@sentinel/2
--redis-sentinel-master=mymaster --redis-cmd-timeout=1000
--subscribe-keyspace=3
--redis-num-threads=1),
'--redis-sentinel=spw@203.0.113.43:26379,203.0.113.45:26379')
or die;
is_deeply(\@probes, [['203.0.113.42', 'auth', 'master']], 'one role probe at startup');
new_call;
$NGCP::Rtpengine::req_cb = sub { redis_expect_set($main_fd, cid(), "+OK\r\n", 'first write') };
offer('first write on .42', { }, $sdp);
# The server closes the subscription, the master is unchanged: the event
# loop must end (nothing, the timer included, may keep it running), and the
# notification thread checks its connection and subscribes again.
redis_close($sub_fd);
redis_expect($notify_fd, ['PING'], "+PONG\r\n", 'subscription closed: connection checked');
$sub_fd = subscription($l{42}, 'subscription closed: again on .42');
# Failover, then back: each time, the old subscription and its sync
# connection are closed, both are made again on the new master, and the next
# write goes there, with nothing sent to the old master first.
for my $step ([44, 2, 'failover'], [42, 3, 'failback']) {
my ($to, $epoch, $n) = @$step;
sentinels_set('203.0.113.43' => "203.0.113.$to 6379 $epoch", '203.0.113.45' => "203.0.113.$to 6379 $epoch");
wait_probe($l{$to}, $n);
wait_log(qr/master of 'mymaster' is now 203\.0\.113\.$to:6379 \(config epoch $epoch\)/, "$n: published");
redis_closed($sub_fd, "$n: old subscription closed");
redis_closed($notify_fd, "$n: old notification connection closed");
$notify_fd = redis_conn($l{$to}, "$n: notify");
$sub_fd = subscription($l{$to}, "$n: subscription");
new_call;
$NGCP::Rtpengine::req_cb = sub {
redis_closed($main_fd, "$n: main connection closed without a write");
$main_fd = redis_conn($l{$to}, "$n: main");
redis_expect_set($main_fd, cid(), "+OK\r\n", "$n: write", no_ping => 1);
};
offer("first write after the $n", { }, $sdp);
}
like(rtpe_log(), qr/the master of 'mymaster' has changed: subscribing again/, 'subscriber change logged');
# no stray connection: everything above was scripted
ok(!readable($l{$_}, 0.5), "nothing more on .$_") for (42, 44);
undef($NGCP::Rtpengine::req_cb);
done_testing();
Loading…
Cancel
Save