MT#55283 redis: find the master through Redis Sentinel

Closes #2212

Behind Redis Sentinel, rtpengine stays on the address it was given:
after a failover its writes go to the old master, which refuses them
until it is demoted, then serves them as a replica.

New options --redis-sentinel, --redis-sentinel-master and
--redis-sentinel-interval; the keyword "sentinel" replaces IP:PORT in
--redis, --redis-write or --redis-subscribe. A "redis sentinel" thread
asks every Sentinel for the master, takes the highest config epoch
(never an older one than in use), checks the node with ROLE and
publishes it with a generation number. It also runs at once when a
connection meets a replica. redis_check_conn() compares the generation
and reconnects on a change; these connections require the master role.
Without "sentinel", none of this runs.

Tested by auto-daemon-tests-redis-sentinel.pl and -sentinel-start.pl.

Change-Id: I7b31b50968fd6a255b9933fbf2c95fc95a8b8408
pull/2237/head
Kevin Binetruy 1 month ago committed by Richard Fuchs
parent f3003b4b9d
commit 2c717d54e7

@ -85,7 +85,7 @@ endif
CFLAGS += $(CFLAGS_MQTT)
LDLIBS += $(LDLIBS_MQTT)
SRCS := main.c kernel.c helpers.c control_tcp.c call.c control_udp.c redis.c \
SRCS := main.c kernel.c helpers.c control_tcp.c call.c control_udp.c redis.c redis_sentinel.c \
cookie_cache.c udp_listener.c control_ng_flags_parser.c control_ng.c sdp.strhash.c stun.c rtcp.c \
crypto.c rtp.c call_interfaces.c dtls.c log.c cli.strhash.c graphite.c ice.c \
media_socket.c homer.c recording.c statistics.c cdr.c ssrc.c iptables.c tcp_listener.c \

@ -33,6 +33,7 @@
#include "call.h"
#include "kernel.h"
#include "redis.h"
#include "redis_sentinel.h"
#include "sdp.h"
#include "dtls.h"
#include "call_interfaces.h"
@ -570,14 +571,19 @@ static bool if_addr_parse(intf_config_q *q, char *s, struct ifaddrs *ifas) {
/**
* Sets hostname and auth.
* With the keyword "sentinel" instead of IP:PORT, sets *sentinel and leaves
* the endpoint and hostname unset.
*/
static int redis_ep_parse(endpoint_t *ep, int *db, char **hostname, char **auth, const char *auth_env, char *s) {
static int redis_ep_parse(endpoint_t *ep, int *db, char **hostname, char **auth, const char *auth_env,
gboolean *sentinel, char *s)
{
char *sl, *sp;
long l;
/* ensure to have pointers nulled */
*auth = NULL;
*hostname = NULL;
*sentinel = false;
sl = strrchr(s, '@');
if (sl) {
@ -602,6 +608,11 @@ static int redis_ep_parse(endpoint_t *ep, int *db, char **hostname, char **auth,
*db = l;
}
if (!strcmp(s, "sentinel")) {
*sentinel = true;
return 0;
}
/* copy for the case with re-resolve during re-connections */
sp = strrchr(s, ':'); /* make sure to not take port into the value of hostname */
if (sp)
@ -719,6 +730,9 @@ static void options(int *argc, char ***argv, charp_ht templates) {
g_autoptr(char) redisps = NULL;
g_autoptr(char) redisps_write = NULL;
g_autoptr(char) redisps_subscribe = NULL;
g_autoptr(char) redis_sentinels = NULL;
g_autoptr(char) redis_sentinel_master = NULL;
int redis_sentinel_interval = -1;
g_autoptr(char) log_facility_cdr_s = NULL;
g_autoptr(char) log_facility_rtcp_s = NULL;
g_autoptr(char) log_facility_dtmf_s = NULL;
@ -812,6 +826,9 @@ static void options(int *argc, char ***argv, charp_ht templates) {
{ "redis-write",'w', 0, G_OPTION_ARG_STRING, &redisps_write, "Connect to Redis write database", "[PW@]IP:PORT/INT" },
{ "redis-subscribe", 0, 0, G_OPTION_ARG_STRING, &redisps_subscribe, "Connect to Redis subscribe database", "[PW@]IP:PORT[/INT]" },
{ "redis-resolve-on-reconnect", 0,0, G_OPTION_ARG_NONE, &rtpe_config.redis_resolve_on_reconnect, "Re-resolve given FQDN on each re-connect to the redis server.", NULL },
{ "redis-sentinel", 0, 0, G_OPTION_ARG_STRING, &redis_sentinels, "Redis Sentinels to ask for the master, for the roles given as 'sentinel'", "[PW@]IP:PORT[,IP:PORT...]" },
{ "redis-sentinel-master", 0, 0, G_OPTION_ARG_STRING, &redis_sentinel_master, "Name of the master monitored by Redis Sentinel", "STRING" },
{ "redis-sentinel-interval", 0, 0, G_OPTION_ARG_INT, &redis_sentinel_interval, "How often to ask Redis Sentinel for the master, in milliseconds", "INT" },
{ "redis-num-threads", 0, 0, G_OPTION_ARG_INT, &rtpe_config.redis_num_threads, "Number of Redis restore threads", "INT" },
{ "redis-expires", 0, 0, G_OPTION_ARG_INT, &rtpe_config.redis_expires_secs, "Expire time in seconds for redis keys", "INT" },
{ "no-redis-required", 'q', 0, G_OPTION_ARG_NONE, &rtpe_config.no_redis_required, "Start no matter of redis connection state", NULL },
@ -1294,7 +1311,7 @@ static void options(int *argc, char ***argv, charp_ht templates) {
if (redisps) {
if (redis_ep_parse(&rtpe_config.redis_ep, &rtpe_config.redis_db, &rtpe_config.redis_hostname,
&rtpe_config.redis_auth, "RTPENGINE_REDIS_AUTH_PW", redisps))
&rtpe_config.redis_auth, "RTPENGINE_REDIS_AUTH_PW", &rtpe_config.redis_via_sentinel, redisps))
{
die("Invalid Redis endpoint [IP:PORT/INT] '%s' (--redis)", redisps);
}
@ -1302,7 +1319,8 @@ static void options(int *argc, char ***argv, charp_ht templates) {
if (redisps_write) {
if (redis_ep_parse(&rtpe_config.redis_write_ep, &rtpe_config.redis_write_db, &rtpe_config.redis_write_hostname,
&rtpe_config.redis_write_auth, "RTPENGINE_REDIS_WRITE_AUTH_PW", redisps_write))
&rtpe_config.redis_write_auth, "RTPENGINE_REDIS_WRITE_AUTH_PW", &rtpe_config.redis_write_via_sentinel,
redisps_write))
{
die("Invalid Redis endpoint [IP:PORT/INT] '%s' (--redis-write)", redisps_write);
}
@ -1310,14 +1328,38 @@ static void options(int *argc, char ***argv, charp_ht templates) {
if (redisps_subscribe)
if (redis_ep_parse(&rtpe_config.redis_subscribe_ep, &rtpe_config.redis_subscribe_db, &rtpe_config.redis_subscribe_hostname,
&rtpe_config.redis_subscribe_auth,"RTPENGINE_REDIS_SUBSCRIBE_AUTH_PW", redisps_subscribe))
&rtpe_config.redis_subscribe_auth,"RTPENGINE_REDIS_SUBSCRIBE_AUTH_PW", &rtpe_config.redis_subscribe_via_sentinel,
redisps_subscribe))
{
rtpe_config.redis_subscribe_db = -1;
if (redis_ep_parse(&rtpe_config.redis_subscribe_ep, NULL, &rtpe_config.redis_subscribe_hostname,
&rtpe_config.redis_subscribe_auth,"RTPENGINE_REDIS_SUBSCRIBE_AUTH_PW", redisps_subscribe))
&rtpe_config.redis_subscribe_auth,"RTPENGINE_REDIS_SUBSCRIBE_AUTH_PW", &rtpe_config.redis_subscribe_via_sentinel,
redisps_subscribe))
die("Invalid Redis endpoint [IP:PORT/INT] '%s' (--redis-subscribe)", redisps_subscribe);
}
if (rtpe_config.redis_via_sentinel || rtpe_config.redis_write_via_sentinel
|| rtpe_config.redis_subscribe_via_sentinel)
{
if (!redis_sentinels)
die("A Redis endpoint is 'sentinel', but --redis-sentinel is not set");
if (!redis_sentinel_master)
die("--redis-sentinel-master must be set with --redis-sentinel");
if (redis_sentinel_interval == -1)
redis_sentinel_interval = 1000;
if (redis_sentinel_interval <= 0)
die("Invalid --redis-sentinel-interval (must be a positive number of milliseconds)");
rtpe_config.redis_sentinel_interval = redis_sentinel_interval;
rtpe_redis_sentinel = redis_sentinel_new(redis_sentinels, redis_sentinel_master,
"RTPENGINE_REDIS_SENTINEL_AUTH_PW");
if (!rtpe_redis_sentinel)
die("Invalid Redis Sentinel list [PW@]IP:PORT[,IP:PORT...] '%s' (--redis-sentinel)",
redis_sentinels);
}
else if (redis_sentinels || redis_sentinel_master || redis_sentinel_interval != -1)
die("--redis-sentinel, --redis-sentinel-master or --redis-sentinel-interval is set, "
"but no Redis endpoint is 'sentinel'");
if (rtpe_config.fmt < 0 || rtpe_config.fmt > 2)
die("Invalid XMLRPC format");
@ -1910,28 +1952,51 @@ static void create_everything(void) {
create_listeners(&rtpe_config.ng_tcp_listen_ep, &rtpe_control_ng_tcp, (void *(*)(const endpoint_t *)) control_ng_tcp_new, false, "TCP NG control");
create_listeners(&rtpe_config.cli_listen_ep, &rtpe_cli, (void *(*)(const endpoint_t *)) cli_new, false, "CLI");
if (!is_addr_unspecified(&rtpe_config.redis_write_ep.address)) {
// the roles that use Sentinel require a master, and never re-resolve
bool sent_r = rtpe_config.redis_via_sentinel, sent_w = rtpe_config.redis_write_via_sentinel,
sent_s = rtpe_config.redis_subscribe_via_sentinel;
if (rtpe_redis_sentinel) {
// the master's password, for the role check on the Sentinels' answer
redis_sentinel_set_probe_auth(rtpe_redis_sentinel,
sent_w ? rtpe_config.redis_write_auth
: sent_r ? rtpe_config.redis_auth
: rtpe_config.redis_subscribe_auth);
ilog(LOG_INFO, "Using Redis Sentinel for the master of '%s'",
redis_sentinel_master_name(rtpe_redis_sentinel));
if (!rtpe_config.redis_cmd_timeout)
ilog(LOG_WARN, "Redis Sentinel is used without --redis-cmd-timeout: a master that "
"stops answering holds Redis operations until it answers again");
if (!redis_sentinel_discover(rtpe_redis_sentinel) && !rtpe_config.no_redis_required)
die("Cannot start up without a master of '%s' from Redis Sentinel! "
"See also NO_REDIS_REQUIRED parameter.",
redis_sentinel_master_name(rtpe_redis_sentinel));
}
if (!is_addr_unspecified(&rtpe_config.redis_write_ep.address) || sent_w) {
rtpe_redis_write = redis_new(&rtpe_config.redis_write_ep,
rtpe_config.redis_write_db,
rtpe_config.redis_write_hostname,
rtpe_config.redis_write_auth,
MASTER_REDIS_ROLE,
rtpe_config.no_redis_required,
rtpe_config.redis_resolve_on_reconnect);
rtpe_config.redis_resolve_on_reconnect && !sent_w,
sent_w ? rtpe_redis_sentinel : NULL);
if (!rtpe_redis_write)
die("Cannot start up without running Redis %s write database! See also NO_REDIS_REQUIRED parameter.",
endpoint_print_buf(&rtpe_config.redis_write_ep));
}
if (!is_addr_unspecified(&rtpe_config.redis_subscribe_ep.address)) {
if (!is_addr_unspecified(&rtpe_config.redis_subscribe_ep.address) || sent_s) {
rtpe_redis_notify = redis_new(&rtpe_config.redis_subscribe_ep,
rtpe_config.redis_subscribe_db,
rtpe_config.redis_subscribe_hostname,
rtpe_config.redis_subscribe_auth,
ANY_REDIS_ROLE,
(sent_s ? MASTER_REDIS_ROLE : ANY_REDIS_ROLE),
rtpe_config.no_redis_required,
rtpe_config.redis_resolve_on_reconnect);
rtpe_config.redis_resolve_on_reconnect && !sent_s,
sent_s ? rtpe_redis_sentinel : NULL);
if (!rtpe_redis_notify)
die("Cannot start up without running Redis %s subscribe database! See also NO_REDIS_REQUIRED parameter.",
@ -1942,14 +2007,15 @@ static void create_everything(void) {
}
}
if (!is_addr_unspecified(&rtpe_config.redis_ep.address)) {
if (!is_addr_unspecified(&rtpe_config.redis_ep.address) || sent_r) {
rtpe_redis = redis_new(&rtpe_config.redis_ep,
rtpe_config.redis_db,
rtpe_config.redis_hostname,
rtpe_config.redis_auth,
(rtpe_redis_write ? ANY_REDIS_ROLE : MASTER_REDIS_ROLE),
((rtpe_redis_write && !sent_r) ? ANY_REDIS_ROLE : MASTER_REDIS_ROLE),
rtpe_config.no_redis_required,
rtpe_config.redis_resolve_on_reconnect);
rtpe_config.redis_resolve_on_reconnect && !sent_r,
sent_r ? rtpe_redis_sentinel : NULL);
if (!rtpe_redis)
die("Cannot start up without running Redis %s database! "
@ -1961,9 +2027,10 @@ static void create_everything(void) {
rtpe_config.redis_db,
rtpe_config.redis_hostname,
rtpe_config.redis_auth,
(rtpe_redis_write ? ANY_REDIS_ROLE : MASTER_REDIS_ROLE),
((rtpe_redis_write && !sent_r) ? ANY_REDIS_ROLE : MASTER_REDIS_ROLE),
rtpe_config.no_redis_required,
rtpe_config.redis_resolve_on_reconnect);
rtpe_config.redis_resolve_on_reconnect && !sent_r,
sent_r ? rtpe_redis_sentinel : NULL);
if (!rtpe_redis_notify)
die("Cannot start up without running notification Redis %s database! "
@ -2130,12 +2197,15 @@ int main(int argc, char **argv) {
thread_create_looper(media_player_expire, rtpe_config.idle_scheduling,
rtpe_config.idle_priority, "cache expiry", rtpe_config.expiry_timer * 1000000LL);
if (!is_addr_unspecified(&rtpe_config.redis_ep.address) && initial_rtpe_config.redis_delete_async)
if (rtpe_redis && initial_rtpe_config.redis_delete_async)
thread_create_detach(redis_delete_async_loop, NULL, "redis async");
if (rtpe_redis_notify)
thread_create_detach(redis_notify_loop, NULL, "redis notify");
if (rtpe_redis_sentinel)
thread_create_detach(redis_sentinel_loop, rtpe_redis_sentinel, "redis sentinel");
do_redis_restore();
if (graphite_is_enabled())
@ -2193,7 +2263,7 @@ int main(int argc, char **argv) {
websocket_stop();
if (!is_addr_unspecified(&rtpe_config.redis_ep.address) && initial_rtpe_config.redis_delete_async)
if (rtpe_redis && initial_rtpe_config.redis_delete_async)
redis_async_event_base_action(rtpe_redis_write, EVENT_BASE_LOOPBREAK);
if (rtpe_redis_notify)
@ -2201,10 +2271,10 @@ int main(int argc, char **argv) {
threads_join_all(true);
if (!is_addr_unspecified(&rtpe_config.redis_ep.address) && initial_rtpe_config.redis_delete_async)
if (rtpe_redis && initial_rtpe_config.redis_delete_async)
redis_async_event_base_action(rtpe_redis_write, EVENT_BASE_FREE);
if (!is_addr_unspecified(&rtpe_config.redis_ep.address) && rtpe_redis_notify)
if (rtpe_redis && rtpe_redis_notify)
redis_async_event_base_action(rtpe_redis_notify, EVENT_BASE_FREE);
ilog(LOG_INFO, "Version %s shutting down", RTPENGINE_VERSION);
@ -2230,6 +2300,8 @@ int main(int argc, char **argv) {
if (rtpe_redis_write != rtpe_redis)
redis_close(rtpe_redis_write);
redis_close(rtpe_redis_notify);
redis_sentinel_free(rtpe_redis_sentinel);
rtpe_redis_sentinel = NULL;
free_prefix();
log_free();

@ -34,6 +34,7 @@
#include "codec.h"
#include "sdp.h"
#include "statistics.h"
#include "redis_sentinel.h"
#define REDIS_CONNECT_ROLE -2 /* redis_connect(): the server has the wrong role */
@ -97,8 +98,6 @@ static int redisCommandNR(redisContext *r, const char *fmt, ...)
#define REDIS_FMT(x) (int) (x)->len, (x)->str
#define rlog(l, x...) ilog(l | LOG_FLAG_RESTORE, x)
// To protect against a restore race condition: Keyspace notifications are set up
@ -277,8 +276,12 @@ static int redis_consume(struct redis *r) {
redis_write_result(r, rp);
if (redis_is_role_error(rp))
role_error = true;
else if (r->sentinel && rp->type == REDIS_REPLY_ERROR && !strncmp(rp->str, "NOREPLICAS", 10))
redis_sentinel_wake(r->sentinel, true); // the master may have been replaced
freeReplyObject(rp);
}
if (role_error && r->sentinel)
redis_sentinel_wake(r->sentinel, false);
if (role_error && r->role == MASTER_REDIS_ROLE) {
rlog(LOG_WARN, "Redis %s is not a master: dropping the connection",
endpoint_print_buf(&r->endpoint));
@ -308,6 +311,36 @@ int redis_set_cmd_timeout(struct redis *r, int64_t timeout) {
return redis_set_timeout(r, timeout);
}
/* called with r->lock held
* With Sentinel, takes the master that the Sentinel thread published, if it
* is a new one, and drops the connection to the old one: true if so. Not
* while a reconnect is under way: the next check follows. */
static bool redis_sentinel_follow(struct redis *r) {
if (!r->sentinel || r->reconnecting || redis_sentinel_gen(r->sentinel) == r->gen)
return false;
endpoint_t ep;
uint64_t gen = redis_sentinel_get(r->sentinel, &ep);
bool moved = !r->gen || !endpoint_eq(&ep, &r->endpoint);
r->gen = gen;
if (!moved)
return false;
rlog(LOG_NOTICE, "Redis: following the master of '%s' from %s to %s",
redis_sentinel_master_name(r->sentinel),
r->ctx ? endpoint_print_buf(&r->endpoint) : "none", endpoint_print_buf(&ep));
r->endpoint = ep;
sockaddr_print(&ep.address, r->host, sizeof(r->host));
// try the new master at once, whatever happened with the old one
r->restore_tick_us = 0;
r->consecutive_errors = 0;
if (r->ctx)
redisFree(r->ctx);
r->ctx = NULL;
r->state = REDIS_STATE_DISCONNECTED;
return true;
}
int redis_reconnect(struct redis* r) {
int rval;
LOCK(&r->lock);
@ -317,6 +350,10 @@ int redis_reconnect(struct redis* r) {
rlog(LOG_WARN, "Redis %s: a reconnect is already in progress", endpoint_print_buf(&r->endpoint));
return -1;
}
redis_sentinel_follow(r);
if (r->sentinel && !r->gen)
return -1;
rval = redis_connect(r, 0, r->update_resolve);
if (rval)
r->state = REDIS_STATE_DISCONNECTED;
@ -680,6 +717,7 @@ int redis_async_event_base_action(struct redis *r, enum event_base_action action
rlog(LOG_ERR, "redis_async_event_base_action: NULL r");
return -1;
} else {
LOCK(&r->lock); // (Sentinel moves the endpoint)
rlog(LOG_DEBUG, "redis_async_event_base_action: Use Redis %s", endpoint_print_buf(&r->endpoint));
}
@ -992,7 +1030,8 @@ void redis_notify_loop(void *d) {
}
struct redis *redis_new(const endpoint_t *ep, int db, const char *hostname, const char *auth,
enum redis_role role, int no_redis_required, bool update_resolve) {
enum redis_role role, int no_redis_required, bool update_resolve,
struct redis_sentinel *sentinel) {
struct redis *r;
r = g_new0(struct redis, 1);
@ -1007,9 +1046,23 @@ struct redis *redis_new(const endpoint_t *ep, int db, const char *hostname, cons
r->restore_tick_us = 0;
r->consecutive_errors = 0;
r->update_resolve = update_resolve;
r->sentinel = sentinel;
mutex_init(&r->lock);
cond_init(&r->reconnect_cond);
if (sentinel) {
r->gen = redis_sentinel_get(sentinel, &r->endpoint);
sockaddr_print(&r->endpoint.address, r->host, sizeof(r->host));
if (!r->gen) {
if (r->no_redis_required) {
rlog(LOG_WARN, "Starting with no master of '%s' known from Redis Sentinel !",
redis_sentinel_master_name(sentinel));
return r;
}
goto err;
}
}
if (redis_connect(r, 10, false)) {
if (r->no_redis_required) {
rlog(LOG_WARN, "Starting with no initial connection to Redis %s !",
@ -1038,7 +1091,8 @@ struct redis *redis_dup(const struct redis *r, int db) {
r->auth,
r->role,
r->no_redis_required,
r->update_resolve);
r->update_resolve,
r->sentinel);
}
void redis_close(struct redis *r) {
@ -1090,6 +1144,18 @@ static int redis_check_conn(struct redis *r) {
rtpe_now = now_us();
}
bool moved = redis_sentinel_follow(r);
if (r->sentinel && !r->gen) {
// no master known yet
r->skipped++;
if (!r->skipped_logged)
rlog(LOG_WARNING, "No master of '%s' known from Redis Sentinel yet",
redis_sentinel_master_name(r->sentinel));
r->skipped_logged = true;
redis_sentinel_wake(r->sentinel, false);
return REDIS_STATE_DISCONNECTED;
}
if ((r->state == REDIS_STATE_DISCONNECTED) && r->restore_tick_us > rtpe_now) {
// every Redis operation ends up here: say it once
r->skipped++;
@ -1101,8 +1167,9 @@ static int redis_check_conn(struct redis *r) {
}
if (r->state == REDIS_STATE_DISCONNECTED) {
ilog(LOG_INFO, "RE-Establishing connection for Redis server %s (%u operations skipped)",
endpoint_print_buf(&r->endpoint), r->skipped);
if (!moved)
ilog(LOG_INFO, "RE-Establishing connection for Redis server %s (%u operations skipped)",
endpoint_print_buf(&r->endpoint), r->skipped);
r->skipped = 0;
r->skipped_logged = false;
}
@ -1121,7 +1188,7 @@ static int redis_check_conn(struct redis *r) {
// redis is disconnected
if (r->state == REDIS_STATE_CONNECTED) {
rlog(LOG_ERR, "Lost connection to Redis '%s'",
r->hostname);
r->hostname ? : r->host);
r->state = REDIS_STATE_DISCONNECTED;
}
@ -1139,6 +1206,8 @@ static int redis_check_conn(struct redis *r) {
if (ret) {
// redis is disconnected
redis_count_err_and_disable(r);
if (r->sentinel && ret == REDIS_CONNECT_ROLE)
redis_sentinel_wake(r->sentinel, false);
// After the wrong role, or a slow failure (a timeout), pause (one
// second by default): a failover takes longer than that, so the
// next attempt would fail the same way. A quick failure

@ -0,0 +1,401 @@
#include "redis_sentinel.h"
#include <glib.h>
#include <inttypes.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <hiredis/hiredis.h>
#include "helpers.h"
#include "log_d.h"
#include "main.h"
#include "redis.h"
struct sentinel_node {
char *host; /* as configured, for --redis-resolve-on-reconnect */
endpoint_t ep;
bool failing; /* last query failed (logged once) */
};
struct redis_sentinel {
struct sentinel_node *nodes;
unsigned int num_nodes;
char *auth;
char *master_name;
const char *probe_auth; /* data node password, for ROLE */
/* published master: written by the thread, read by the connections */
mutex_t lock;
cond_t cond;
bool wake;
endpoint_t master;
uint64_t epoch;
atomic64 gen; /* 0: no master known yet */
atomic64 limited_wake_us;
/* thread only */
bool none_answered; /* logged once */
bool epoch_back; /* logged once */
struct thread_waker waker;
};
struct redis_sentinel *rtpe_redis_sentinel;
/* [PW@]HOST:PORT[,HOST:PORT...] */
struct redis_sentinel *redis_sentinel_new(const char *list, const char *master_name,
const char *auth_env)
{
if (!list || !*list || !master_name || !*master_name)
return NULL;
struct redis_sentinel *s = g_new0(struct redis_sentinel, 1);
mutex_init(&s->lock);
cond_init(&s->cond);
g_autofree char *dup = g_strdup(list);
g_auto(GStrv) items = NULL;
char *hosts = dup;
char *at = strrchr(hosts, '@');
if (at) {
*at = '\0';
if (!*hosts)
goto err; // "@host:port": an empty password is a mistake
s->auth = g_strdup(hosts);
hosts = at + 1;
}
else {
const char *env = getenv(auth_env);
if (env && *env)
s->auth = g_strdup(env);
}
items = g_strsplit(hosts, ",", -1);
s->num_nodes = g_strv_length(items);
if (!s->num_nodes)
goto err;
s->nodes = g_new0(struct sentinel_node, s->num_nodes);
for (unsigned int i = 0; i < s->num_nodes; i++) {
struct sentinel_node *n = &s->nodes[i];
char *item = g_strstrip(items[i]);
char *colon = strrchr(item, ':');
if (!colon)
goto err;
if (!endpoint_parse_any_getaddrinfo_full(&n->ep, item) || !n->ep.port)
goto err;
n->host = g_strndup(item, colon - item);
}
s->master_name = g_strdup(master_name);
return s;
err:
redis_sentinel_free(s);
return NULL;
}
void redis_sentinel_free(struct redis_sentinel *s) {
if (!s)
return;
for (unsigned int i = 0; s->nodes && i < s->num_nodes; i++)
g_free(s->nodes[i].host);
g_free(s->nodes);
g_free(s->auth);
g_free(s->master_name);
mutex_destroy(&s->lock);
g_free(s);
}
void redis_sentinel_set_probe_auth(struct redis_sentinel *s, const char *auth) {
s->probe_auth = auth;
}
const char *redis_sentinel_master_name(const struct redis_sentinel *s) {
return s->master_name;
}
uint64_t redis_sentinel_gen(struct redis_sentinel *s) {
return atomic64_get(&s->gen);
}
uint64_t redis_sentinel_get(struct redis_sentinel *s, endpoint_t *ep) {
LOCK(&s->lock);
*ep = s->master;
return atomic64_get_na(&s->gen);
}
/* "limited": at most once per poll interval (NOREPLICAS lasts as long as the
* master lacks replicas) */
void redis_sentinel_wake(struct redis_sentinel *s, bool limited) {
if (limited) {
int64_t now = now_us();
int64_t last = atomic64_get_na(&s->limited_wake_us);
if (last && now >= last && now - last < rtpe_config.redis_sentinel_interval * 1000LL)
return;
if (!atomic64_set_if(&s->limited_wake_us, now, last))
return;
}
LOCK(&s->lock);
s->wake = true;
cond_signal(&s->cond);
}
static redisContext *sentinel_connect(const endpoint_t *ep, const char *auth) {
char host[64];
sockaddr_print(&ep->address, host, sizeof(host));
int64_t connect_timeout = atomic_get_na(&rtpe_config.redis_connect_timeout) * 1000LL;
// never without a command timeout ("rediscmdtimeout 0"): a server that
// accepts but never answers would stop the thread for good
int64_t cmd_timeout = atomic_get_na(&rtpe_config.redis_cmd_timeout) * 1000LL ? : 1000000;
redisContext *ctx = redisConnectWithTimeout(host, ep->port, timeval_from_us(connect_timeout));
if (!ctx)
return NULL;
const char *err = NULL;
redisReply *rp = NULL;
if (ctx->err || redisSetTimeout(ctx, timeval_from_us(cmd_timeout)))
err = ctx->errstr;
else if (auth && !(rp = redisCommand(ctx, "AUTH %s", auth)))
err = ctx->errstr;
else if (rp && rp->type == REDIS_REPLY_ERROR)
err = "authentication failed";
if (rp)
freeReplyObject(rp);
if (!err)
return ctx;
rlog(LOG_WARN | LOG_FLAG_LIMIT, "Redis Sentinel: cannot connect to %s: %s",
endpoint_print_buf(ep), err);
redisFree(ctx);
return NULL;
}
static const char *reply_field(const redisReply *rp, const char *name) {
for (size_t i = 0; i + 1 < rp->elements; i += 2) {
const redisReply *k = rp->element[i], *v = rp->element[i + 1];
if (k->type == REDIS_REPLY_STRING && v->type == REDIS_REPLY_STRING && !strcmp(k->str, name))
return v->str;
}
return NULL;
}
static const char *sentinel_parse(const redisReply *rp, const redisReply *ra,
endpoint_t *master, uint64_t *epoch)
{
if (rp->type == REDIS_REPLY_ERROR)
return rp->str;
if (ra->type == REDIS_REPLY_ERROR)
return ra->str;
const char *epoch_s = rp->type == REDIS_REPLY_ARRAY ? reply_field(rp, "config-epoch") : NULL;
if (!epoch_s || ra->type != REDIS_REPLY_ARRAY || ra->elements != 2
|| ra->element[0]->type != REDIS_REPLY_STRING
|| ra->element[1]->type != REDIS_REPLY_STRING)
return "unexpected reply";
char *end;
*epoch = strtoull(epoch_s, &end, 10);
if (end == epoch_s || *end)
return "unexpected reply";
unsigned long port = strtoul(ra->element[1]->str, &end, 10);
if (end == ra->element[1]->str || *end || !port || port > 0xffff)
return "unexpected reply";
// a host name (announce-hostnames) is resolved here, in the thread
if (!sockaddr_getaddrinfo(&master->address, ra->element[0]->str))
return "cannot resolve the master's address";
master->port = port;
return NULL;
}
/* Epoch first, then address, so that a failover between the two replies can
* only pair an older epoch with the newer address. (The "ip" of SENTINEL
* MASTER switches seconds after the epoch.) */
static bool sentinel_query(struct redis_sentinel *s, struct sentinel_node *n,
endpoint_t *master, uint64_t *epoch)
{
if (rtpe_config.redis_resolve_on_reconnect) {
sockaddr_t a;
if (sockaddr_getaddrinfo(&a, n->host))
n->ep.address = a;
}
redisContext *ctx = sentinel_connect(&n->ep, s->auth);
if (!ctx) {
n->failing = true;
return false;
}
redisReply *rp = NULL, *ra = NULL;
const char *err;
redisAppendCommand(ctx, "SENTINEL MASTER %s", s->master_name);
redisAppendCommand(ctx, "SENTINEL GET-MASTER-ADDR-BY-NAME %s", s->master_name);
if (redisGetReply(ctx, (void **) &rp) != REDIS_OK || redisGetReply(ctx, (void **) &ra) != REDIS_OK)
err = ctx->errstr;
else
err = sentinel_parse(rp, ra, master, epoch);
if (err && !n->failing)
rlog(LOG_WARN, "Redis Sentinel %s: cannot get the master of '%s': %s",
endpoint_print_buf(&n->ep), s->master_name, err);
else if (!err && n->failing)
rlog(LOG_NOTICE, "Redis Sentinel %s answers again", endpoint_print_buf(&n->ep));
n->failing = err != NULL;
if (rp)
freeReplyObject(rp);
if (ra)
freeReplyObject(ra);
redisFree(ctx);
return !err;
}
static bool sentinel_is_master(struct redis_sentinel *s, const endpoint_t *ep, uint64_t epoch) {
redisContext *ctx = sentinel_connect(ep, s->probe_auth);
if (!ctx)
return false;
bool ret = false;
redisReply *rp = redisCommand(ctx, "ROLE");
if (rp && rp->type == REDIS_REPLY_ARRAY && rp->elements && rp->element[0]->type == REDIS_REPLY_STRING) {
ret = !strcmp(rp->element[0]->str, "master");
if (!ret)
rlog(LOG_WARN | LOG_FLAG_LIMIT, "Redis Sentinel names %s as master of '%s' "
"(config epoch %" PRIu64 "), but it is a %s: not used",
endpoint_print_buf(ep), s->master_name, epoch, rp->element[0]->str);
}
else
rlog(LOG_WARN | LOG_FLAG_LIMIT, "Redis Sentinel: cannot check the role of %s: %s",
endpoint_print_buf(ep), !rp ? ctx->errstr
: rp->type == REDIS_REPLY_ERROR ? rp->str : "unexpected reply");
if (rp)
freeReplyObject(rp);
redisFree(ctx);
return ret;
}
static void sentinel_publish(struct redis_sentinel *s, const endpoint_t *m, uint64_t epoch) {
{
LOCK(&s->lock);
s->master = *m;
s->epoch = epoch;
atomic64_inc(&s->gen);
}
rlog(LOG_NOTICE, "Redis Sentinel: master of '%s' is now %s (config epoch %" PRIu64 ")",
s->master_name, endpoint_print_buf(m), epoch);
}
/* true if the candidate is not a master: ask again soon */
static bool sentinel_round(struct redis_sentinel *s) {
endpoint_t best = {0}, pub;
uint64_t best_epoch = 0, pub_epoch, pub_gen;
bool found = false;
{
LOCK(&s->lock);
pub = s->master;
pub_epoch = s->epoch;
pub_gen = atomic64_get_na(&s->gen);
}
for (unsigned int i = 0; i < s->num_nodes; i++) {
endpoint_t m = {0};
uint64_t e = 0;
if (!sentinel_query(s, &s->nodes[i], &m, &e))
continue;
// at the same epoch, keep the master in use (no flapping while
// the Sentinels disagree)
if (!found || e > best_epoch || (e == best_epoch && pub_gen && endpoint_eq(&m, &pub))) {
best = m;
best_epoch = e;
found = true;
}
}
if (!found) {
if (!s->none_answered)
rlog(LOG_ERR, "Redis Sentinel: no Sentinel answered for '%s'%s%s",
s->master_name,
pub_gen ? "; keeping " : "",
pub_gen ? endpoint_print_buf(&pub) : "");
s->none_answered = true;
return false;
}
s->none_answered = false;
// Never back to an older epoch: a demoted master still says "master"
// until Sentinel reconfigures it. The master in use is kept for as long
// as it answers as a master (its connection checks the role).
if (pub_gen && best_epoch < pub_epoch) {
if (!s->epoch_back)
rlog(LOG_ERR, "Redis Sentinel: every Sentinel that answers names %s as master of '%s' "
"at config epoch %" PRIu64 ", older than %" PRIu64 " (%s, in use): not "
"followed. If the Sentinels were redeployed without their state, their "
"epochs went back, and rtpengine must be restarted",
endpoint_print_buf(&best), s->master_name, best_epoch, pub_epoch,
endpoint_print_buf(&pub));
s->epoch_back = true;
return false;
}
s->epoch_back = false;
if (pub_gen && endpoint_eq(&best, &pub)) {
LOCK(&s->lock);
s->epoch = best_epoch;
return false;
}
if (!sentinel_is_master(s, &best, best_epoch))
return true;
sentinel_publish(s, &best, best_epoch);
return false;
}
/* at startup: as for a data node, fails at once if nothing answers, and asks
* again once a second (11 times) while the candidate is not a master */
bool redis_sentinel_discover(struct redis_sentinel *s) {
for (int i = 0; i < 11; i++) {
if (i)
usleep(1000000);
bool refused = sentinel_round(s);
if (redis_sentinel_gen(s))
return true;
if (!refused)
break;
}
return false;
}
void redis_sentinel_loop(struct redis_sentinel *s) {
int64_t interval_us = rtpe_config.redis_sentinel_interval * 1000LL;
int64_t next = now_us() + interval_us, last = 0;
// a round is bounded by the timeouts; at shutdown, the waker ends the wait
s->waker.lock = &s->lock;
s->waker.cond = &s->cond;
thread_waker_add(&s->waker);
while (!rtpe_shutdown) {
{
LOCK(&s->lock);
// a wake-up comes at least 250 ms after the last round
while (!rtpe_shutdown) {
int64_t until = s->wake ? MIN(next, last + 250000) : next;
if (now_us() >= until)
break;
cond_timedwait(&s->cond, &s->lock, until);
}
s->wake = false;
}
if (rtpe_shutdown)
break;
bool refused = sentinel_round(s);
last = now_us();
next = last + (refused ? MIN(250000, interval_us) : interval_us);
}
thread_waker_del(&s->waker);
}

@ -560,6 +560,11 @@ call to inject-DTMF won't be sent to __\-\-dtmf-log-dest=__ or __\-\-listen-tcp-
When this option is given, __rtpengine__ will delay startup until the
Redis database adopts the master role (but see below).
Instead of *IP*:*PORT*, the keyword __sentinel__ can be given, for
example *foobar@sentinel/12*: the address is then the current master of
a Redis Sentinel group, see __\-\-redis-sentinel__. This is also
possible for __\-\-redis-write__ and __\-\-redis-subscribe__.
- __-w__, __\-\-redis-write=__\[*PW*@\]*IP*:*PORT*/*INT*
Configures a second Redis database for write operations.
@ -590,7 +595,10 @@ call to inject-DTMF won't be sent to __\-\-dtmf-log-dest=__ or __\-\-listen-tcp-
This option takes precedence over __\-\-redis__, if configured.
When __\-\-subscribe-keyspace__ is also configured, the keyspace part of
__\-\-redis-subscribe=__ is not used, the former takes precedence.
The keyspace number can also be omitted altogether.
The keyspace number can also be omitted altogether. With __sentinel__
instead of *IP*:*PORT* (see __\-\-redis-sentinel__), the subscription is
made on the current master: a replica is not used, even though
notifications could be read from one.
For password protected Redis servers, the environment variable for the
password is __RTPENGINE\_REDIS\_SUBSCRIBE\_AUTH\_PW__.
@ -718,6 +726,40 @@ call to inject-DTMF won't be sent to __\-\-dtmf-log-dest=__ or __\-\-listen-tcp-
remote redis server try to re-resolve if the redis hostname was an FQDN
and not IP address. It is a boolean value (either __true__ or __false__).
- __\-\-redis-sentinel=__\[*PW*@\]*IP*:*PORT*\[,*IP*:*PORT*...\]
The Redis Sentinels to ask for the master, for every Redis option given
with __sentinel__ instead of *IP*:*PORT*; requires
__\-\-redis-sentinel-master__. Their password, usually not the one of
the data nodes, is given before the `@` symbol or in
__RTPENGINE\_REDIS\_SENTINEL\_AUTH\_PW__. Host names are resolved at
startup, and before each query with __\-\-redis-resolve-on-reconnect__.
The Sentinels are asked every __\-\-redis-sentinel-interval__ and when a
connection finds a replica. The highest configuration epoch wins, and
never an older one than the master in use: if all are older (Sentinels
redeployed without their state), an error is logged and __rtpengine__
must be restarted. The master is checked with __ROLE__, then every
connection through Sentinel moves to it at its next operation. These
connections require the master role, as the write connection does. Set
__\-\-redis-cmd-timeout__ (for example to 1000): without it, a master
that stops answering holds Redis operations until it answers again.
If no Sentinel answers at startup, __rtpengine__ does not start unless
__\-\-no-redis-required__ is given; if none answers later, the last
master is kept.
- __\-\-redis-sentinel-master=__*STRING*
The name of the master that the Redis Sentinels monitor.
- __\-\-redis-sentinel-interval=__*INT*
How often, in milliseconds, the Redis Sentinels are asked for the master
when nothing prompted a query. The default value is 1000.
__\-\-redis-sentinel-master__ and __\-\-redis-sentinel-interval__ are
refused without a Redis option that uses __sentinel__.
- __\-\-redis-format=bencode__\|__JSON__
Selects the format for serialised call data written to Redis or KeyDB. The
@ -2150,6 +2192,10 @@ IPv6.
specified, in which case the one specified in __\-\-redis__ will be used for
read operations only.
- __RTPENGINE\_REDIS\_SENTINEL\_AUTH\_PW__
Redis Sentinel password, for __\-\-redis-sentinel__.
## FILES
- `/etc/rtpengine/rtpengine.conf`

@ -84,6 +84,10 @@ recording-method = proc
# redis-reconnect-pause = 1000
# redis-reconnect-slow = 100
# redis-resolve-on-reconnect = false
# redis-write = password@sentinel/42
# redis-sentinel = password@10.0.0.11:26379,10.0.0.12:26379,10.0.0.13:26379
# redis-sentinel-master = mymaster
# redis-sentinel-interval = 1000
# redis-tcp-keepalive-time = 1
# redis-tcp-keepalive-intvl = 1
# redis-tcp-keepalive-probes = 3

@ -62,6 +62,7 @@ enum endpoint_learning {
X(redis_tcp_keepalive_time) \
X(redis_tcp_keepalive_intvl) \
X(redis_tcp_keepalive_probes) \
X(redis_sentinel_interval) \
X(num_threads) \
X(media_num_threads) \
X(codec_num_threads) \
@ -139,6 +140,9 @@ enum endpoint_learning {
X(player_cache) \
X(poller_per_thread) \
X(redis_resolve_on_reconnect) \
X(redis_via_sentinel) \
X(redis_write_via_sentinel) \
X(redis_subscribe_via_sentinel) \
X(measure_rtp) \
X(amr_cn_dtx) \
X(evs_cn_dtx) \

@ -15,8 +15,13 @@
#include "crypto.h"
#include "dtls.h"
struct redis_sentinel;
#define REDIS_RESTORE_NUM_THREADS 4
// also logged while calls are being restored
#define rlog(l, x...) ilog(l | LOG_FLAG_RESTORE, x)
enum redis_role {
MASTER_REDIS_ROLE = 0,
SLAVE_REDIS_ROLE = 1,
@ -76,6 +81,9 @@ struct redis {
int64_t async_last;
bool update_resolve;
struct redis_sentinel *sentinel; /* address from Redis Sentinel, or NULL */
uint64_t gen; /* generation of the master in endpoint */
};
struct redis_hash {
@ -113,7 +121,8 @@ void redis_notify_loop(void *d);
void redis_delete_async_loop(void *d);
struct redis *redis_new(const endpoint_t *, int, const char *, const char *, enum redis_role, int, bool);
struct redis *redis_new(const endpoint_t *, int, const char *, const char *, enum redis_role, int, bool,
struct redis_sentinel *);
struct redis *redis_dup(const struct redis *r, int db);
void redis_close(struct redis *r);
int redis_restore(struct redis *, bool foreign, int db);

@ -0,0 +1,32 @@
#ifndef __REDIS_SENTINEL_H__
#define __REDIS_SENTINEL_H__
#include <stdbool.h>
#include <stdint.h>
#include "socket.h"
/*
* A thread asks the Redis Sentinels for the master and publishes it with a
* generation number; the connections that use it compare the generation (one
* atomic read) and reconnect when it changes. All Sentinel I/O is done in
* that thread.
*/
struct redis_sentinel;
extern struct redis_sentinel *rtpe_redis_sentinel;
struct redis_sentinel *redis_sentinel_new(const char *list, const char *master_name,
const char *auth_env);
void redis_sentinel_set_probe_auth(struct redis_sentinel *, const char *auth);
bool redis_sentinel_discover(struct redis_sentinel *);
void redis_sentinel_loop(struct redis_sentinel *);
void redis_sentinel_free(struct redis_sentinel *);
uint64_t redis_sentinel_gen(struct redis_sentinel *);
uint64_t redis_sentinel_get(struct redis_sentinel *, endpoint_t *);
void redis_sentinel_wake(struct redis_sentinel *, bool limited);
const char *redis_sentinel_master_name(const struct redis_sentinel *);
#endif

@ -2,12 +2,15 @@ package NGCP::Rtpengine::FakeRedis;
# Fake Redis servers for the daemon tests: listeners on test addresses (made
# local by tests-preload.so), and helpers that read the commands that
# rtpengine sends (RESP arrays of bulk strings) and answer them.
# rtpengine sends (RESP arrays of bulk strings) and answer them. Fake Redis
# Sentinels run in a child process (sentinels_start()), so that they answer
# while the test script handles the data nodes.
use strict;
use warnings;
use Socket qw(AF_INET SOCK_STREAM sockaddr_in inet_aton);
use File::Temp;
use IO::Handle;
use Test::More;
use Time::HiRes qw(time sleep);
use IO::Select;
@ -18,9 +21,18 @@ use Exporter 'import';
our @EXPORT = qw(redis_listen redis_accept redis_conn redis_read redis_send redis_expect
redis_expect_set redis_expect_del redis_close redis_closed readable info_reply
redis_write_errors wait_log
redis_serve redis_sets_seen redis_record_after redis_serve_next_get);
redis_serve redis_sets_seen redis_record_after redis_serve_next_get
%role @probes wait_probe serve_probes
sentinels_start sentinels_set sentinel_log sentinel_queries);
my %rbuf; # what was read from each socket and not parsed yet
my %node; # IP of each listener, by fileno
# Role probes of the Sentinel thread (AUTH, then ROLE) on the data nodes are
# answered from %role (by IP, "master" by default), and recorded in @probes
# as [IP, password, role answered].
our %role;
our @probes;
sub redis_listen {
@ -29,6 +41,7 @@ sub redis_listen {
socket($l, AF_INET, SOCK_STREAM, 0) or die;
bind($l, sockaddr_in($port // 6379, inet_aton($ip))) or die "bind $ip: $!";
listen($l, 10) or die;
$node{fileno($l)} = $ip;
return $l;
}
@ -132,14 +145,40 @@ sub redis_accept {
return $fd;
}
sub role_reply {
my $role = shift;
return "*3\r\n\$6\r\nmaster\r\n:0\r\n*0\r\n" if $role eq 'master';
return "*5\r\n\$5\r\nslave\r\n\$12\r\n203.0.113.44\r\n:6379\r\n\$9\r\nconnected\r\n:0\r\n";
}
# after the AUTH of a probe: answer its ROLE
sub probe_answer {
my ($fd, $ip, $auth) = @_;
my $r = $role{$ip} // 'master';
redis_send($fd, role_reply($r));
push(@probes, [$ip, $auth, $r]);
redis_close($fd);
}
# A new data connection: AUTH, SELECT, INFO answered with the role (default
# "master"), then TYPE unless "refused" (the role is not the one wanted).
# Options: auth, db, role, short, replid, refused.
# Options: auth, db, role, short, replid, refused. Probes that come first
# are answered.
sub redis_conn {
my ($l, $n, %opt) = @_;
my $fd = redis_accept($l);
redis_expect($fd, ['AUTH', $opt{auth} // 'auth'], "+OK\r\n", "$n AUTH");
redis_expect($fd, ['SELECT', $opt{db} // 2], "+OK\r\n", "$n SELECT");
my ($fd, $auth, $c);
while (1) {
$fd = redis_accept($l);
$auth = redis_read($fd);
# (a probe can give up before it is answered)
redis_send($fd, "+OK\r\n") if $auth;
$c = $auth && redis_read($fd);
last if $c && $c->[0] ne 'ROLE';
$c ? probe_answer($fd, $node{fileno($l)}, $auth->[1]) : redis_close($fd);
}
is(join(' ', @$auth), 'AUTH ' . ($opt{auth} // 'auth'), "$n AUTH");
is(join(' ', @$c), 'SELECT ' . ($opt{db} // 2), "$n SELECT");
redis_send($fd, "+OK\r\n");
redis_expect($fd, ['INFO'], info_reply($opt{role} // 'master', %opt), "$n INFO");
redis_expect($fd, ['TYPE', 'calls'], "+none\r\n", "$n TYPE") unless $opt{refused};
return $fd;
@ -318,4 +357,190 @@ sub wait_log {
die "not in the log: $re\n";
}
# wait for a role probe on this listener
sub wait_probe {
my ($l, $n, $timeout) = @_;
my $fd = redis_accept($l, $timeout);
my $c = redis_expect($fd, ['AUTH', 'auth'], "+OK\r\n", "$n: probe AUTH with the data password");
redis_expect($fd, ['ROLE'], undef, "$n: probe ROLE");
probe_answer($fd, $node{fileno($l)}, $c->[1]);
}
# for a while, answer every probe on these listeners, whatever arrives
sub serve_probes {
my ($secs, @l) = @_;
my $end = time() + $secs;
while ((my $left = $end - time()) > 0) {
my $rin = '';
vec($rin, fileno($_), 1) = 1 for @l;
next unless select(my $rout = $rin, undef, undef, $left) > 0;
for my $l (grep { vec($rout, fileno($_), 1) } @l) {
my $fd = redis_accept($l);
my $a = eval { redis_read($fd, 1) };
redis_send($fd, "+OK\r\n") if $a;
my $c = $a && eval { redis_read($fd, 1) };
$c && $c->[0] eq 'ROLE' ? probe_answer($fd, $node{fileno($l)}, $a->[1]) : redis_close($fd);
}
}
}
# Fake Sentinels. The child re-reads a state file every 50 ms and logs every
# command it gets ("IP:PORT COMMAND ARG..."), plus "applied N" for each new
# state. Each Sentinel (IP, port 26379) is either "down" (not listening) or
# "MASTER_IP MASTER_PORT EPOCH [MONITORED_IP]": it names this master
# (GET-MASTER-ADDR-BY-NAME) at this config epoch (SENTINEL MASTER), whose
# "ip" is MONITORED_IP if given: a Sentinel running a failover reports the
# promoted replica in the former and still the old master in the latter.
my ($sentinel_pid, $sentinel_state, $sentinel_log, $sentinel_password);
my $serial = 0;
sub sentinel_log {
open(my $fh, '<', "$sentinel_log") or die;
local $/;
return <$fh> // '';
}
# how many times this Sentinel was asked for the master
sub sentinel_queries {
my $ip = shift;
my @l = sentinel_log() =~ /^\Q$ip\E:26379 SENTINEL MASTER mymaster$/mg;
return scalar(@l);
}
# set what the Sentinels answer (IP => state), and wait until it is applied
sub sentinels_set {
my %st = @_;
$serial++;
open(my $fh, '>', "$sentinel_state.new") or die;
print $fh "serial $serial\npassword $sentinel_password\n";
print $fh "$_:26379 $st{$_}\n" for sort(keys(%st));
close($fh);
rename("$sentinel_state.new", "$sentinel_state") or die;
return unless $sentinel_pid;
for (1 .. 100) {
return if sentinel_log() =~ /^applied $serial$/m;
sleep(0.05);
}
die "fake Sentinels did not apply state $serial\n";
}
sub sentinels_start {
($sentinel_password, my %st) = @_;
$sentinel_state = File::Temp->new(TEMPLATE => 'rtpe-sentinel-state.XXXXXXXXXX', TMPDIR => 1);
$sentinel_log = File::Temp->new(TEMPLATE => 'rtpe-sentinel-log.XXXXXXXXXX', TMPDIR => 1);
sentinels_set(%st);
$sentinel_pid = fork() // die;
if (!$sentinel_pid) {
exec($^X, (map { "-I$_" } grep { !ref($_) } @INC), '-MNGCP::Rtpengine::FakeRedis',
'-e', 'NGCP::Rtpengine::FakeRedis::sentinel_main(@ARGV)',
"$sentinel_state", "$sentinel_log") or die;
}
sentinels_set(%st);
}
END {
if ($sentinel_pid) {
kill('TERM', $sentinel_pid);
waitpid($sentinel_pid, 0);
}
}
# the child
sub sentinel_main {
my ($statefile, $logfile) = @_;
open(my $log, '>>', $logfile) or die;
$log->autoflush(1);
my $done = 0;
$SIG{TERM} = sub { $done = 1 };
$SIG{PIPE} = 'IGNORE';
my (%sentinels, %clients, $password);
my $applied = -1;
my $read_state = sub {
open(my $fh, '<', $statefile) or return;
my ($ser, %want);
while (my $l = <$fh>) {
my @f = split(' ', $l);
if ($f[0] eq 'serial') { $ser = $f[1] }
elsif ($f[0] eq 'password') { $password = $f[1] }
else { $want{$f[0]} = [ @f[1 .. $#f] ] }
}
close($fh);
for my $s (keys(%want)) {
my $st = $sentinels{$s} //= { };
$st->{state} = $want{$s};
if ($want{$s}[0] eq 'down') {
close(delete($st->{listener})) if $st->{listener};
}
elsif (!$st->{listener}) {
my ($ip, $port) = split(/:/, $s);
$st->{listener} = redis_listen($ip, $port);
}
}
# a Sentinel that goes down closes its clients too
for my $fn (keys(%clients)) {
next if $sentinels{$clients{$fn}{sentinel}}{listener};
close($clients{$fn}{fh});
delete($clients{$fn});
}
if (defined($ser) && $ser != $applied) {
$applied = $ser;
print $log "applied $ser\n";
}
};
my $reply = sub {
my ($c, $args) = @_;
my $st = $sentinels{$c->{sentinel}}{state};
print $log "$c->{sentinel} @$args\n";
my $cmd = uc($args->[0]) . ' ' . uc($args->[1] // '');
if ($cmd =~ /^AUTH /) {
return "-WRONGPASS invalid username-password pair\r\n" if $args->[-1] ne $password;
$c->{authed} = 1;
return "+OK\r\n";
}
return "-NOAUTH Authentication required.\r\n" unless $c->{authed};
return "-ERR No such master with that name\r\n" if ($args->[2] // '') ne 'mymaster';
my ($ip, $port, $epoch, $monitored) = @$st;
return "*2\r\n" . bulk($ip) . bulk($port) if $cmd eq 'SENTINEL GET-MASTER-ADDR-BY-NAME';
my @kv = (name => 'mymaster', ip => $monitored // $ip, port => $port, flags => 'master',
'num-slaves' => 2, 'num-other-sentinels' => 2, quorum => 2, 'config-epoch' => $epoch);
return '*' . scalar(@kv) . "\r\n" . join('', map { bulk($_) } @kv) if $cmd eq 'SENTINEL MASTER';
return "-ERR unknown command\r\n";
};
$read_state->();
while (!$done) {
my $rin = '';
my %l = map { fileno($_->{listener}) => $_ } grep { $_->{listener} } values(%sentinels);
vec($rin, $_, 1) = 1 for (keys(%l), keys(%clients));
if (select(my $rout = $rin, undef, undef, 0.05) > 0) {
for my $fn (grep { vec($rout, $_, 1) } keys(%l)) {
my ($s) = grep { $sentinels{$_} == $l{$fn} } keys(%sentinels);
accept(my $fh, $l{$fn}{listener}) or next;
$clients{fileno($fh)} = { fh => $fh, buf => '', sentinel => $s };
}
for my $fn (grep { vec($rout, $_, 1) } keys(%clients)) {
my $c = $clients{$fn};
my $data;
if (!sysread($c->{fh}, $data, 65536)) {
close($c->{fh});
delete($clients{$fn});
next;
}
$c->{buf} .= $data;
while (my $args = parse_cmd(\$c->{buf})) {
syswrite($c->{fh}, $reply->($c, $args));
}
}
}
$read_state->();
}
exit(0);
}
1;

@ -138,6 +138,8 @@ include ../lib/common.Makefile
daemon-tests-dtx-dtmf \
daemon-tests-ice-force-answer \
daemon-tests-redis-master-only \
daemon-tests-redis-sentinel \
daemon-tests-redis-sentinel-start \
#
TESTS := test-bitstr test-str aes-crypt aead-aes-crypt test-const_str_hash.strhash
@ -238,6 +240,8 @@ daemon-tests: \
daemon-tests-video \
daemon-tests-inject \
daemon-tests-redis-master-only \
daemon-tests-redis-sentinel \
daemon-tests-redis-sentinel-start \
#
daemon-test-deps: tests-preload.so
@ -360,6 +364,12 @@ daemon-tests-redis-reconnect: daemon-test-deps
daemon-tests-redis-master-only: daemon-test-deps
./auto-test-helper "$@" perl -I../perl auto-daemon-tests-redis-master-only.pl
daemon-tests-redis-sentinel: daemon-test-deps
./auto-test-helper "$@" perl -I../perl auto-daemon-tests-redis-sentinel.pl
daemon-tests-redis-sentinel-start: daemon-test-deps
./auto-test-helper "$@" perl -I../perl auto-daemon-tests-redis-sentinel-start.pl
daemon-tests-audio-player: daemon-test-deps
./auto-test-helper "$@" perl -I../perl auto-daemon-tests-audio-player.pl
@ -562,6 +572,7 @@ test-stats: test-stats.o \
../daemon/ng_client.o \
../daemon/recording.o \
../daemon/redis.o \
../daemon/redis_sentinel.o \
../daemon/rtcp.o \
../daemon/rtp.o \
../daemon/sdp.strhash.o \
@ -626,6 +637,7 @@ test-transcode: test-transcode.o \
../daemon/ng_client.o \
../daemon/recording.o \
../daemon/redis.o \
../daemon/redis_sentinel.o \
../daemon/rtcp.o \
../daemon/rtp.o \
../daemon/sdp.strhash.o \

@ -136,8 +136,10 @@ $NGCP::Rtpengine::req_cb = sub {
offer('write after MASTERDOWN', { }, $sdp);
# the read connection was never touched
# the read connection was never touched, and without Sentinel options
# nothing of Sentinel is set up
ok(!readable($read_fd, 0), 'nothing on the read connection');
unlike(rtpe_log(), qr/Sentinel/, 'no Sentinel');
undef($NGCP::Rtpengine::req_cb);

@ -0,0 +1,197 @@
#!/usr/bin/perl
# Redis Sentinel, started with no Sentinel reachable (--no-redis-required),
# and polled rarely (--redis-sentinel-interval=60000), so that every query
# seen here was triggered by rtpengine itself: by an operation with no master
# known, by a NOREPLICAS refusal, by a READONLY refusal, or by a reconnect
# that finds a replica. The Sentinel password comes from the environment.
use strict;
use warnings;
use NGCP::Rtpengine::Test;
use NGCP::Rtpengine::AutoTest;
use NGCP::Rtpengine::FakeRedis;
use Test::More;
use Time::HiRes qw(sleep);
my %l = map { $_ => redis_listen("203.0.113.$_") } (42, 44);
sentinels_start('spw2', '203.0.113.43' => 'down');
# a missing reply fails the test with the daemon's last log lines
local $SIG{ALRM} = sub { die "timeout\n" };
# a role probe can give up before it is answered
local $SIG{PIPE} = 'IGNORE';
my $sdp = <<SDP;
v=0
o=- 1545997027 1 IN IP4 198.51.101.40
s=tester
t=0 0
m=audio 3000 RTP/AVP 0 8
c=IN IP4 198.51.100.1
----------------------------------
v=0
o=- 1545997027 1 IN IP4 198.51.101.40
s=tester
t=0 0
m=audio PORT RTP/AVP 0 8
c=IN IP4 203.0.113.1
a=rtpmap:0 PCMU/8000
a=rtpmap:8 PCMA/8000
a=sendrecv
a=rtcp:PORT
SDP
# the queries since the last call
my $queries_seen = 0;
sub new_queries {
my $n = sentinel_queries('203.0.113.43');
my $ret = $n - $queries_seen;
$queries_seen = $n;
return $ret;
}
sub wait_query {
my $n = shift;
for (1 .. 60) {
return pass($n) if new_queries();
sleep(0.05);
}
fail($n);
}
# a write refused with $err
sub refused_write {
my ($fd, $err, $n) = @_;
new_call;
$NGCP::Rtpengine::req_cb = sub { redis_expect_set($fd, cid(), "-$err\r\n", $n) };
offer($n, { }, $sdp);
}
local $ENV{RTPENGINE_REDIS_SENTINEL_AUTH_PW} = 'spw2';
autotest_start(qw(--config-file=none -t -1 -i 203.0.113.1
-n 2223 -f -L 7 -E
--redis=auth@sentinel/2
--redis-sentinel=203.0.113.43:26379
--redis-sentinel-master=mymaster
--redis-sentinel-interval=60000
--no-redis-required
))
or die;
like(rtpe_log(), qr/Starting with no master of 'mymaster' known from Redis Sentinel/,
'started without a master');
like(rtpe_log(), qr/Redis Sentinel is used without --redis-cmd-timeout/,
'no command timeout: warned, not changed');
ok(!readable($l{42}, 0), 'no data connection at startup');
# no master known: the update is skipped, nothing is connected
new_call;
$NGCP::Rtpengine::req_cb = sub {
ok(!readable($l{42}, 0.3), 'no master known: no connection');
};
offer('write with no master known', { }, $sdp);
like(rtpe_log(), qr/No master of 'mymaster' known from Redis Sentinel yet/, 'skipped write logged');
# the Sentinel comes up: the next skipped update wakes the thread, which
# finds the master long before its next poll
sentinels_set('203.0.113.43' => '203.0.113.42 6379 1');
new_queries();
new_call;
undef($NGCP::Rtpengine::req_cb);
offer('write that wakes the Sentinel thread', { }, $sdp);
wait_query('woken by a write with no master known');
like(sentinel_log(), qr/^203\.0\.113\.43:26379 AUTH spw2$/m, 'Sentinel password from the environment');
wait_probe($l{42}, 'first master');
wait_log(qr/master of 'mymaster' is now 203\.0\.113\.42:6379 \(config epoch 1\)/, 'master published');
my $fd;
new_call;
$NGCP::Rtpengine::req_cb = sub {
$fd = redis_conn($l{42}, 'first connection');
redis_expect_set($fd, cid(), "+OK\r\n", 'first write', no_ping => 1);
};
offer('first write', { }, $sdp);
# NOREPLICAS: the connection is kept, and the Sentinels are asked at once;
# they name .44, where the next write goes
sentinels_set('203.0.113.43' => '203.0.113.44 6379 2');
new_queries();
refused_write($fd, 'NOREPLICAS Not enough good replicas to write.', 'NOREPLICAS write');
wait_query('woken by NOREPLICAS');
wait_probe($l{44}, 'after NOREPLICAS');
wait_log(qr/master of 'mymaster' is now 203\.0\.113\.44:6379 \(config epoch 2\)/, '.44 published');
new_call;
$NGCP::Rtpengine::req_cb = sub {
redis_closed($fd, 'connection to .42 closed');
$fd = redis_conn($l{44}, 'after NOREPLICAS');
redis_expect_set($fd, cid(), "+OK\r\n", 'write on .44', no_ping => 1);
};
offer('write on .44', { }, $sdp);
unlike(rtpe_log(), qr/RE-Establishing connection for Redis server 203\.0\.113\.44/,
'the move is logged once');
# NOREPLICAS again, within the poll interval: the connection is kept, and
# the Sentinels are not asked again (a master also says this for as long as
# it lacks replicas: at most one query per interval)
new_queries();
refused_write($fd, 'NOREPLICAS Not enough good replicas to write.', 'NOREPLICAS again');
sleep(0.5);
is(new_queries(), 0, 'NOREPLICAS again: no new query within the interval');
# READONLY: the master role is implied in Sentinel mode, so the connection
# is dropped at once, and the Sentinels are asked
new_queries();
new_call;
$NGCP::Rtpengine::req_cb = sub {
redis_expect_set($fd, cid(), "-READONLY You can't write against a read only replica.\r\n", 'READONLY');
redis_closed($fd, 'READONLY: connection dropped');
};
offer('READONLY write', { }, $sdp);
wait_query('woken by READONLY');
# The Sentinel still names .44 (same address and epoch: no probe). The
# reconnect finds a replica, gives up after one INFO, asks the Sentinels
# again, and makes no new attempt within the next second.
sleep(0.3);
new_queries();
new_call;
$NGCP::Rtpengine::req_cb = sub {
$fd = redis_conn($l{44}, 'replica', role => 'slave', refused => 1);
redis_closed($fd, 'replica: connection closed after one INFO');
};
offer('write while .44 is a replica', { }, $sdp);
wait_query('woken by the role mismatch');
ok(!readable($l{44}, 0.3), 'no probe for an unchanged answer, no reconnect within the pause');
new_call;
$NGCP::Rtpengine::req_cb = sub {
ok(!readable($l{44}, 0.2), 'no reconnect within the pacing interval');
};
offer('write within the pacing interval', { }, $sdp);
sleep(1);
new_call;
$NGCP::Rtpengine::req_cb = sub {
$fd = redis_conn($l{44}, 'master again');
redis_expect_set($fd, cid(), "+OK\r\n", 'master again', no_ping => 1);
};
offer('write after the pacing interval', { }, $sdp);
undef($NGCP::Rtpengine::req_cb);
done_testing();

@ -0,0 +1,213 @@
#!/usr/bin/perl
# Redis Sentinel: --redis=PW@sentinel/DB takes the master's address from the
# Sentinels. Two fake Sentinels (203.0.113.43 and .45) name the master; two
# fake data nodes (203.0.113.42 and .44) are scripted here.
use strict;
use warnings;
use NGCP::Rtpengine::Test;
use NGCP::Rtpengine::AutoTest;
use NGCP::Rtpengine::FakeRedis;
use Test::More;
use Time::HiRes qw(sleep time);
my %l = map { $_ => redis_listen("203.0.113.$_") } (42, 44);
sentinels_start('spw', '203.0.113.43' => '203.0.113.42 6379 1', '203.0.113.45' => '203.0.113.42 6379 1');
# a missing reply fails the test with the daemon's last log lines
local $SIG{ALRM} = sub { die "timeout\n" };
# a role probe can give up before it is answered
local $SIG{PIPE} = 'IGNORE';
my $sdp = <<SDP;
v=0
o=- 1545997027 1 IN IP4 198.51.101.40
s=tester
t=0 0
m=audio 3000 RTP/AVP 0 8
c=IN IP4 198.51.100.1
----------------------------------
v=0
o=- 1545997027 1 IN IP4 198.51.101.40
s=tester
t=0 0
m=audio PORT RTP/AVP 0 8
c=IN IP4 203.0.113.1
a=rtpmap:0 PCMU/8000
a=rtpmap:8 PCMA/8000
a=sendrecv
a=rtcp:PORT
SDP
sub published {
my $re = shift;
return scalar(() = rtpe_log() =~ /master of 'mymaster' is now $re/g);
}
# wait until this Sentinel has been asked $n more times
sub queried {
my ($ip, $n) = @_;
my $q = sentinel_queries($ip);
for (1 .. 100) {
return 1 if sentinel_queries($ip) >= $q + $n;
sleep(0.05);
}
return 0;
}
sub write_on {
my ($fd, $n) = @_;
new_call;
$NGCP::Rtpengine::req_cb = sub { redis_expect_set($$fd, cid(), "+OK\r\n", $n) };
offer($n, { }, $sdp);
}
# the next write moves to the new master, with nothing sent to the old one
sub write_moves {
my ($fd, $to, $n) = @_;
new_call;
$NGCP::Rtpengine::req_cb = sub {
redis_closed($$fd, "$n: old connection closed without a write");
$$fd = redis_conn($l{$to}, "$n: on .$to");
redis_expect_set($$fd, cid(), "+OK\r\n", $n, no_ping => 1);
};
offer($n, { }, $sdp);
}
my $main_fd;
$NGCP::Rtpengine::AutoTest::launch_cb = sub {
# discovery (Sentinels, then a role probe on .42), then the connection
$main_fd = redis_conn($l{42}, 'startup');
redis_expect($main_fd, ['PING'], "+PONG\r\n", 'restore PING');
redis_expect($main_fd, ['KEYS', '*'], "*0\r\n", 'restore KEYS');
};
autotest_start(qw(--config-file=none -t -1 -i 203.0.113.1
-n 2223 -f -L 7 -E
--redis=auth@sentinel/2
--redis-sentinel-master=mymaster --redis-sentinel-interval=500
--redis-cmd-timeout=1000),
'--redis-sentinel=spw@203.0.113.43:26379,203.0.113.45:26379')
or die;
# startup: both Sentinels asked, with their password; the master they name
# checked with ROLE and the data password before it was used
for my $s ('203.0.113.43', '203.0.113.45') {
like(sentinel_log(), qr/^\Q$s\E:26379 AUTH spw\n\Q$s\E:26379 SENTINEL MASTER mymaster\n\Q$s\E:26379 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster$/m,
"Sentinel $s asked at startup");
}
is_deeply(\@probes, [['203.0.113.42', 'auth', 'master']], 'one role probe at startup, with the data password');
is(published(qr/203\.0\.113\.42:6379 \(config epoch 1\)/), 1, 'master published');
like(rtpe_log(), qr/Using Redis Sentinel for the master of 'mymaster'/, 'Sentinel mode');
write_on(\$main_fd, 'first write on .42');
# Failover to .44. The Sentinel running it (.45) has raised the epoch and
# names the promoted replica in GET-MASTER-ADDR-BY-NAME, while the "ip" of
# its SENTINEL MASTER is still the old master; the other Sentinel (.43) has
# not heard of it yet. The new master is taken at once, once.
sentinels_set('203.0.113.43' => '203.0.113.42 6379 1', '203.0.113.45' => '203.0.113.44 6379 2 203.0.113.42');
wait_probe($l{44}, 'failover, leader alone at the new epoch');
wait_log(qr/master of 'mymaster' is now 203\.0\.113\.44:6379 \(config epoch 2\)/, 'new master published');
sentinels_set('203.0.113.43' => '203.0.113.44 6379 2', '203.0.113.45' => '203.0.113.44 6379 2');
ok(queried('203.0.113.43', 2), 'Sentinels asked again');
ok(!readable($l{44}, 0), 'failover: no second probe');
is(published(qr/203\.0\.113\.44/), 1, 'failover: published once');
write_moves(\$main_fd, 44, 'first write after the failover');
like(rtpe_log(), qr/following the master of 'mymaster' from 203\.0\.113\.42:6379 to 203\.0\.113\.44:6379/,
'move logged');
write_on(\$main_fd, 'next write on .44');
# a Sentinel that missed the failover names .42 with the old epoch: not used
# while the other one names the new master
sentinels_set('203.0.113.43' => '203.0.113.42 6379 1', '203.0.113.45' => '203.0.113.44 6379 2');
ok(queried('203.0.113.43', 2), 'stale Sentinel asked');
ok(!readable($l{42}, 0), 'stale Sentinel: no probe and no connection to .42');
unlike(rtpe_log(), qr/epochs went back/, 'stale Sentinel, the other one up to date: no error');
# Every Sentinel that answers names an older epoch: never followed, an error
# is logged once, and the master in use is kept. First the stale Sentinel
# alone, then (after a normal state) a Sentinel set redeployed without its
# state, whose epochs start again.
my $epoch_back = qr/every Sentinel that answers names 203\.0\.113\.42:6379 as master of 'mymaster' at config epoch 1, older than 2 \(203\.0\.113\.44:6379, in use\): not followed\. If the Sentinels were redeployed without their state, their epochs went back, and rtpengine must be restarted/;
sentinels_set('203.0.113.43' => '203.0.113.42 6379 1', '203.0.113.45' => 'down');
wait_log($epoch_back, 'older epoch from every Sentinel that answers: error');
ok(queried('203.0.113.43', 2), 'asked again');
is(scalar(() = rtpe_log() =~ /$epoch_back/g), 1, 'logged once');
sentinels_set('203.0.113.43' => '203.0.113.44 6379 2', '203.0.113.45' => '203.0.113.44 6379 2');
ok(queried('203.0.113.43', 2), 'back to the epoch in use');
sentinels_set('203.0.113.43' => '203.0.113.42 6379 1', '203.0.113.45' => '203.0.113.42 6379 1');
ok(queried('203.0.113.43', 2), 'redeployed Sentinels asked');
is(scalar(() = rtpe_log() =~ /$epoch_back/g), 2, 'logged again after the normal state');
ok(!readable($l{42}, 0), 'older epoch: no probe and no connection to .42');
is(published(qr/203\.0\.113\.42:6379 \(config epoch 1\)/), 1, 'older epoch: .42 not published again');
write_on(\$main_fd, 'write on the master in use');
# a Sentinel names a node at a newer epoch, but it is a replica: not used,
# and asked again soon (before the next poll); then the Sentinels agree on
# .44 at that epoch (same address: no probe)
$role{'203.0.113.42'} = 'slave';
sentinels_set('203.0.113.43' => '203.0.113.42 6379 3', '203.0.113.45' => 'down');
wait_probe($l{42}, 'replica candidate');
my $t = time();
wait_probe($l{42}, 'replica candidate again');
ok(time() - $t < 0.45, 'replica candidate asked again before the next poll');
sentinels_set('203.0.113.43' => '203.0.113.44 6379 3', '203.0.113.45' => '203.0.113.44 6379 3');
serve_probes(1, $l{42});
ok(!readable($l{44}, 0), 'same address at a newer epoch: no probe');
is(scalar(() = rtpe_log() =~ /names 203\.0\.113\.42:6379 as master of 'mymaster' \(config epoch 3\), but it is a slave: not used/g),
1, 'replica candidate refused, logged once');
is(published(qr/203\.0\.113\.42:6379 \(config epoch 3\)/), 0, 'replica never published');
write_on(\$main_fd, 'write after a replica candidate');
# no Sentinel answers: the last master is kept, logged once
sentinels_set('203.0.113.43' => 'down', '203.0.113.45' => 'down');
wait_log(qr/no Sentinel answered for 'mymaster'; keeping 203\.0\.113\.44:6379/, 'no Sentinel answering: logged');
sleep(1.2);
is(scalar(() = rtpe_log() =~ /no Sentinel answered/g), 1, 'logged once');
write_on(\$main_fd, 'write without Sentinels');
sentinels_set('203.0.113.43' => '203.0.113.44 6379 3', '203.0.113.45' => '203.0.113.44 6379 3');
wait_log(qr/Redis Sentinel 203\.0\.113\.43:26379 answers again/, 'Sentinel back');
# a second failover, back to .42, seen first from its leader (.45), then
# from both: one move, at once, no flip back
$role{'203.0.113.42'} = 'master';
sentinels_set('203.0.113.43' => '203.0.113.44 6379 3', '203.0.113.45' => '203.0.113.42 6379 4 203.0.113.44');
wait_probe($l{42}, 'second failover');
wait_log(qr/master of 'mymaster' is now 203\.0\.113\.42:6379 \(config epoch 4\)/, 'new master at once');
sentinels_set('203.0.113.43' => '203.0.113.42 6379 4', '203.0.113.45' => '203.0.113.42 6379 4');
ok(queried('203.0.113.43', 2), 'Sentinels asked again');
is(published(qr/203\.0\.113\.44:6379 \(config epoch 4\)/), 0, 'no flip back to the old master');
ok(!readable($l{44}, 0), 'no probe of the old master');
write_moves(\$main_fd, 42, 'first write on the new master');
# throughout: the Sentinels only got their own password and their two
# queries, and every role probe used the data password
is(scalar(grep { !/^(applied \d+|\S+ (AUTH spw|SENTINEL MASTER mymaster|SENTINEL GET-MASTER-ADDR-BY-NAME mymaster))$/ } split(/\n/, sentinel_log())), 0,
'Sentinels get nothing else');
is(scalar(grep { $_->[1] ne 'auth' } @probes), 0, 'role probes only use the data password');
undef($NGCP::Rtpengine::req_cb);
done_testing();
Loading…
Cancel
Save