From c54d3199b328e27d5dff75629abfeb65e17a847a Mon Sep 17 00:00:00 2001 From: Kevin Binetruy Date: Wed, 14 Oct 2026 10:52:21 +0000 Subject: [PATCH] MT#55283 redis: keyspace subscription follows the Sentinel master Closes #2212 A subscription made through "sentinel" stayed on the old master after a failover, and missed the notifications of the new one. With Sentinel, the notification event loop gets a one-second timer: when the master changed, it closes the subscription, and redis_notify_loop() subscribes again on the new master. The subscription's context also gets a connect callback, since hiredis frees a context that fails to connect without calling the disconnect callback. Notifications missed during the switch are not replayed. Tested by auto-daemon-tests-redis-sentinel-subscribe.pl. Change-Id: Ia14d48e220bbf98cc48964e76f33ff2a497953e9 --- daemon/redis.c | 78 +++++++++- docs/rtpengine.md | 5 +- include/redis.h | 3 + t/Makefile | 5 + ...o-daemon-tests-redis-sentinel-subscribe.pl | 147 ++++++++++++++++++ 5 files changed, 230 insertions(+), 8 deletions(-) create mode 100755 t/auto-daemon-tests-redis-sentinel-subscribe.pl diff --git a/daemon/redis.c b/daemon/redis.c index 372755c51..b4d04aa5e 100644 --- a/daemon/redis.c +++ b/daemon/redis.c @@ -656,6 +656,13 @@ void redis_delete_async_context_disconnect(const redisAsyncContext *redis_delete } void redis_notify_async_context_disconnect(const redisAsyncContext *redis_notify_async_context, int status) { + // forget the context, and let event_base_dispatch() return (the timer + // must not keep it running), so that redis_notify_loop() subscribes again + struct redis *r = rtpe_redis_notify; + if (r && r->notify_timer) + event_del(r->notify_timer); + if (r && r->async_ctx == redis_notify_async_context) + r->async_ctx = NULL; if (status == REDIS_ERR) { if (redis_notify_async_context->errstr) { rlog(LOG_ERROR, "redis_notify_async_context_disconnect error %d on context free: %s", @@ -671,6 +678,12 @@ void redis_notify_async_context_disconnect(const redisAsyncContext *redis_notify } } +// hiredis frees a context that fails to connect without the disconnect callback +static void redis_notify_async_context_connect(const redisAsyncContext *ac, int status) { + if (status != REDIS_OK) + redis_notify_async_context_disconnect(ac, status); +} + // connect_cb = connect callback, disconnect_cb = disconnect callback int redis_async_context_alloc(struct redis *r, void *connect_cb, void *disconnect_cb) { // sanity checks @@ -868,8 +881,35 @@ static int redis_delete_async(struct redis *r) { return 0; } +/* With Sentinel, once a second: on a new master, close the subscription, so + * that event_base_dispatch() returns and redis_notify_loop() subscribes again. + * The timer is one-shot, re-armed only while nothing has changed: it never + * keeps the event loop running by itself. */ +static void redis_notify_follow(evutil_socket_t fd, short what, void *p) { + struct redis *r = p; + struct timeval tv = timeval_from_us(1000000); + + if (redis_sentinel_gen(r->sentinel) == r->notify_gen) { + evtimer_add(r->notify_timer, &tv); + return; + } + + rlog(LOG_NOTICE, "Redis: the master of '%s' has changed: subscribing again; keyspace " + "notifications between the change and the new subscription are missed", + redis_sentinel_master_name(r->sentinel)); + // forget the context first: one still connecting may be freed at once, + // with no callback; without a context, leave the loop + redisAsyncContext *ac = r->async_ctx; + r->async_ctx = NULL; + if (ac) + redisAsyncDisconnect(ac); + else + event_base_loopbreak(r->async_ev); +} + static int redis_notify(struct redis *r) { GList *l; + int ret = 0; if (!r) { rlog(LOG_ERROR, "redis_notify database is NULL on redis_notify()"); @@ -917,13 +957,34 @@ static int redis_notify(struct redis *r) { } rwlock_unlock_r(&rtpe_config.keyspaces_lock); + if (r->sentinel) { + struct timeval tv = timeval_from_us(1000000); + r->notify_timer = evtimer_new(r->async_ev, redis_notify_follow, r); + if (r->notify_timer) + evtimer_add(r->notify_timer, &tv); + } + // dispatch event base => thread blocks here if (event_base_dispatch(r->async_ev) < 0) { rlog(LOG_ERROR, "Fail event_base_dispatch()"); - return -1; + ret = -1; } - return 0; + if (r->notify_timer) { + event_free(r->notify_timer); + r->notify_timer = NULL; + } + + return ret; +} + +/* the subscription's context, with the generation of the master it is for */ +static int redis_notify_context_alloc(struct redis *r) { + LOCK(&r->lock); + redis_sentinel_follow(r); + r->notify_gen = r->gen; + return redis_async_context_alloc(r, redis_notify_async_context_connect, + redis_notify_async_context_disconnect); } /* redis_check_conn() needs the lock: the CLI uses the same connection, and @@ -982,8 +1043,12 @@ void redis_notify_loop(void *d) { return ; } + // with Sentinel and no master known yet, the loop below subscribes once + // there is one + bool now = !r->sentinel || r->gen; + // alloc redis async context - if (redis_async_context_alloc(r, NULL, redis_notify_async_context_disconnect) < 0) { + if (now && redis_notify_context_alloc(r) < 0) { return ; } @@ -993,7 +1058,7 @@ void redis_notify_loop(void *d) { } // initial redis_notify - if (redis_check_conn_locked(r) == REDIS_STATE_CONNECTED) { + if (now && redis_check_conn_locked(r) == REDIS_STATE_CONNECTED) { redis_notify_return = redis_notify(r); } @@ -1010,7 +1075,7 @@ void redis_notify_loop(void *d) { if (redis_check_conn_locked(r) == REDIS_STATE_CONNECTED || redis_notify_return < 0) { r->async_ctx = NULL; // alloc new redis async context upon redis breakdown - if (redis_async_context_alloc(r, NULL, redis_notify_async_context_disconnect) < 0) { + if (redis_notify_context_alloc(r) < 0) { continue; } @@ -1024,7 +1089,8 @@ void redis_notify_loop(void *d) { redis_notify_subscribe_action(r, UNSUBSCRIBE_ALL, 0); // free async context - redisAsyncDisconnect(r->async_ctx); + if (r->async_ctx) + redisAsyncDisconnect(r->async_ctx); r->async_ctx = NULL; } } diff --git a/docs/rtpengine.md b/docs/rtpengine.md index ea607645f..d8f69b080 100644 --- a/docs/rtpengine.md +++ b/docs/rtpengine.md @@ -740,8 +740,9 @@ call to inject-DTMF won't be sent to __\-\-dtmf-log-dest=__ or __\-\-listen-tcp- never an older one than the master in use: if all are older (Sentinels redeployed without their state), an error is logged and __rtpengine__ must be restarted. The master is checked with __ROLE__, then every - connection through Sentinel moves to it at its next operation. These - connections require the master role, as the write connection does. Set + connection through Sentinel moves to it at its next operation, a + keyspace subscription within a second. These connections require the + master role, as the write connection does. Set __\-\-redis-cmd-timeout__ (for example to 1000): without it, a master that stops answering holds Redis operations until it answers again. diff --git a/include/redis.h b/include/redis.h index c29fb2bc5..1eb21f769 100644 --- a/include/redis.h +++ b/include/redis.h @@ -16,6 +16,7 @@ #include "dtls.h" struct redis_sentinel; +struct event; #define REDIS_RESTORE_NUM_THREADS 4 @@ -84,6 +85,8 @@ struct redis { struct redis_sentinel *sentinel; /* address from Redis Sentinel, or NULL */ uint64_t gen; /* generation of the master in endpoint */ + struct event *notify_timer; /* subscriber: checks the generation */ + uint64_t notify_gen; /* generation the subscription was made at */ }; struct redis_hash { diff --git a/t/Makefile b/t/Makefile index 5772183d5..18c3877ed 100644 --- a/t/Makefile +++ b/t/Makefile @@ -140,6 +140,7 @@ include ../lib/common.Makefile daemon-tests-redis-master-only \ daemon-tests-redis-sentinel \ daemon-tests-redis-sentinel-start \ + daemon-tests-redis-sentinel-subscribe \ # TESTS := test-bitstr test-str aes-crypt aead-aes-crypt test-const_str_hash.strhash @@ -242,6 +243,7 @@ daemon-tests: \ daemon-tests-redis-master-only \ daemon-tests-redis-sentinel \ daemon-tests-redis-sentinel-start \ + daemon-tests-redis-sentinel-subscribe \ # daemon-test-deps: tests-preload.so @@ -370,6 +372,9 @@ daemon-tests-redis-sentinel: daemon-test-deps daemon-tests-redis-sentinel-start: daemon-test-deps ./auto-test-helper "$@" perl -I../perl auto-daemon-tests-redis-sentinel-start.pl +daemon-tests-redis-sentinel-subscribe: daemon-test-deps + ./auto-test-helper "$@" perl -I../perl auto-daemon-tests-redis-sentinel-subscribe.pl + daemon-tests-audio-player: daemon-test-deps ./auto-test-helper "$@" perl -I../perl auto-daemon-tests-audio-player.pl diff --git a/t/auto-daemon-tests-redis-sentinel-subscribe.pl b/t/auto-daemon-tests-redis-sentinel-subscribe.pl new file mode 100755 index 000000000..38f7f243d --- /dev/null +++ b/t/auto-daemon-tests-redis-sentinel-subscribe.pl @@ -0,0 +1,147 @@ +#!/usr/bin/perl + +# Redis Sentinel and keyspace notifications: the subscription follows the +# master named by the Sentinels, and is made again on each new master. + +use strict; +use warnings; +use NGCP::Rtpengine::Test; +use NGCP::Rtpengine::AutoTest; +use NGCP::Rtpengine::FakeRedis; +use Test::More; + + +my %l = map { $_ => redis_listen("203.0.113.$_") } (42, 44); + +sentinels_start('spw', '203.0.113.43' => '203.0.113.42 6379 1', '203.0.113.45' => '203.0.113.42 6379 1'); + +# a missing reply fails the test with the daemon's last log lines +local $SIG{ALRM} = sub { die "timeout\n" }; +# a role probe can give up before it is answered +local $SIG{PIPE} = 'IGNORE'; + +my $sdp = <[0] eq 'psubscribe') { + $sub_fd = $fd; + is(join(' ', @$c), 'psubscribe __keyspace@3__:*', 'startup psubscribe'); + redis_send($fd, $psubscribed); + next; + } + is(join(' ', @$c), 'SELECT 2', 'restore copy SELECT'); + redis_send($fd, "+OK\r\n"); + redis_expect($fd, ['INFO'], info_reply('master'), 'restore copy INFO'); + redis_expect($fd, ['TYPE', 'calls'], "+none\r\n", 'restore copy TYPE'); + redis_expect($fd, ['PING'], "+PONG\r\n", 'restore copy PING'); + redis_expect($fd, ['SELECT', '3'], "+OK\r\n", 'restore copy SELECT 3'); + redis_expect($fd, ['KEYS', '*'], "*0\r\n", 'restore copy KEYS'); + redis_expect($fd, ['SELECT', '2'], "+OK\r\n", 'restore copy SELECT back'); + } + # own calls + redis_expect($main_fd, ['PING'], "+PONG\r\n", 'restore PING'); + redis_expect($main_fd, ['KEYS', '*'], "*0\r\n", 'restore KEYS'); +}; + +autotest_start(qw(--config-file=none -t -1 -i 203.0.113.1 + -n 2223 -f -L 7 -E + --redis=auth@sentinel/2 + --redis-sentinel-master=mymaster --redis-cmd-timeout=1000 + --subscribe-keyspace=3 + --redis-num-threads=1), + '--redis-sentinel=spw@203.0.113.43:26379,203.0.113.45:26379') + or die; + +is_deeply(\@probes, [['203.0.113.42', 'auth', 'master']], 'one role probe at startup'); + +new_call; +$NGCP::Rtpengine::req_cb = sub { redis_expect_set($main_fd, cid(), "+OK\r\n", 'first write') }; +offer('first write on .42', { }, $sdp); + + +# The server closes the subscription, the master is unchanged: the event +# loop must end (nothing, the timer included, may keep it running), and the +# notification thread checks its connection and subscribes again. + +redis_close($sub_fd); +redis_expect($notify_fd, ['PING'], "+PONG\r\n", 'subscription closed: connection checked'); +$sub_fd = subscription($l{42}, 'subscription closed: again on .42'); + + +# Failover, then back: each time, the old subscription and its sync +# connection are closed, both are made again on the new master, and the next +# write goes there, with nothing sent to the old master first. + +for my $step ([44, 2, 'failover'], [42, 3, 'failback']) { + my ($to, $epoch, $n) = @$step; + sentinels_set('203.0.113.43' => "203.0.113.$to 6379 $epoch", '203.0.113.45' => "203.0.113.$to 6379 $epoch"); + wait_probe($l{$to}, $n); + wait_log(qr/master of 'mymaster' is now 203\.0\.113\.$to:6379 \(config epoch $epoch\)/, "$n: published"); + + redis_closed($sub_fd, "$n: old subscription closed"); + redis_closed($notify_fd, "$n: old notification connection closed"); + $notify_fd = redis_conn($l{$to}, "$n: notify"); + $sub_fd = subscription($l{$to}, "$n: subscription"); + + new_call; + $NGCP::Rtpengine::req_cb = sub { + redis_closed($main_fd, "$n: main connection closed without a write"); + $main_fd = redis_conn($l{$to}, "$n: main"); + redis_expect_set($main_fd, cid(), "+OK\r\n", "$n: write", no_ping => 1); + }; + offer("first write after the $n", { }, $sdp); +} +like(rtpe_log(), qr/the master of 'mymaster' has changed: subscribing again/, 'subscriber change logged'); + +# no stray connection: everything above was scripted +ok(!readable($l{$_}, 0.5), "nothing more on .$_") for (42, 44); + + +undef($NGCP::Rtpengine::req_cb); + +done_testing();