diff --git a/daemon/Makefile b/daemon/Makefile index 42d8c91e1..2d97fe922 100644 --- a/daemon/Makefile +++ b/daemon/Makefile @@ -85,7 +85,7 @@ endif CFLAGS += $(CFLAGS_MQTT) LDLIBS += $(LDLIBS_MQTT) -SRCS := main.c kernel.c helpers.c control_tcp.c call.c control_udp.c redis.c \ +SRCS := main.c kernel.c helpers.c control_tcp.c call.c control_udp.c redis.c redis_sentinel.c \ cookie_cache.c udp_listener.c control_ng_flags_parser.c control_ng.c sdp.strhash.c stun.c rtcp.c \ crypto.c rtp.c call_interfaces.c dtls.c log.c cli.strhash.c graphite.c ice.c \ media_socket.c homer.c recording.c statistics.c cdr.c ssrc.c iptables.c tcp_listener.c \ diff --git a/daemon/main.c b/daemon/main.c index b88deb850..bac095c2f 100644 --- a/daemon/main.c +++ b/daemon/main.c @@ -33,6 +33,7 @@ #include "call.h" #include "kernel.h" #include "redis.h" +#include "redis_sentinel.h" #include "sdp.h" #include "dtls.h" #include "call_interfaces.h" @@ -570,14 +571,19 @@ static bool if_addr_parse(intf_config_q *q, char *s, struct ifaddrs *ifas) { /** * Sets hostname and auth. + * With the keyword "sentinel" instead of IP:PORT, sets *sentinel and leaves + * the endpoint and hostname unset. */ -static int redis_ep_parse(endpoint_t *ep, int *db, char **hostname, char **auth, const char *auth_env, char *s) { +static int redis_ep_parse(endpoint_t *ep, int *db, char **hostname, char **auth, const char *auth_env, + gboolean *sentinel, char *s) +{ char *sl, *sp; long l; /* ensure to have pointers nulled */ *auth = NULL; *hostname = NULL; + *sentinel = false; sl = strrchr(s, '@'); if (sl) { @@ -602,6 +608,11 @@ static int redis_ep_parse(endpoint_t *ep, int *db, char **hostname, char **auth, *db = l; } + if (!strcmp(s, "sentinel")) { + *sentinel = true; + return 0; + } + /* copy for the case with re-resolve during re-connections */ sp = strrchr(s, ':'); /* make sure to not take port into the value of hostname */ if (sp) @@ -719,6 +730,9 @@ static void options(int *argc, char ***argv, charp_ht templates) { g_autoptr(char) redisps = NULL; g_autoptr(char) redisps_write = NULL; g_autoptr(char) redisps_subscribe = NULL; + g_autoptr(char) redis_sentinels = NULL; + g_autoptr(char) redis_sentinel_master = NULL; + int redis_sentinel_interval = -1; g_autoptr(char) log_facility_cdr_s = NULL; g_autoptr(char) log_facility_rtcp_s = NULL; g_autoptr(char) log_facility_dtmf_s = NULL; @@ -812,6 +826,9 @@ static void options(int *argc, char ***argv, charp_ht templates) { { "redis-write",'w', 0, G_OPTION_ARG_STRING, &redisps_write, "Connect to Redis write database", "[PW@]IP:PORT/INT" }, { "redis-subscribe", 0, 0, G_OPTION_ARG_STRING, &redisps_subscribe, "Connect to Redis subscribe database", "[PW@]IP:PORT[/INT]" }, { "redis-resolve-on-reconnect", 0,0, G_OPTION_ARG_NONE, &rtpe_config.redis_resolve_on_reconnect, "Re-resolve given FQDN on each re-connect to the redis server.", NULL }, + { "redis-sentinel", 0, 0, G_OPTION_ARG_STRING, &redis_sentinels, "Redis Sentinels to ask for the master, for the roles given as 'sentinel'", "[PW@]IP:PORT[,IP:PORT...]" }, + { "redis-sentinel-master", 0, 0, G_OPTION_ARG_STRING, &redis_sentinel_master, "Name of the master monitored by Redis Sentinel", "STRING" }, + { "redis-sentinel-interval", 0, 0, G_OPTION_ARG_INT, &redis_sentinel_interval, "How often to ask Redis Sentinel for the master, in milliseconds", "INT" }, { "redis-num-threads", 0, 0, G_OPTION_ARG_INT, &rtpe_config.redis_num_threads, "Number of Redis restore threads", "INT" }, { "redis-expires", 0, 0, G_OPTION_ARG_INT, &rtpe_config.redis_expires_secs, "Expire time in seconds for redis keys", "INT" }, { "no-redis-required", 'q', 0, G_OPTION_ARG_NONE, &rtpe_config.no_redis_required, "Start no matter of redis connection state", NULL }, @@ -1294,7 +1311,7 @@ static void options(int *argc, char ***argv, charp_ht templates) { if (redisps) { if (redis_ep_parse(&rtpe_config.redis_ep, &rtpe_config.redis_db, &rtpe_config.redis_hostname, - &rtpe_config.redis_auth, "RTPENGINE_REDIS_AUTH_PW", redisps)) + &rtpe_config.redis_auth, "RTPENGINE_REDIS_AUTH_PW", &rtpe_config.redis_via_sentinel, redisps)) { die("Invalid Redis endpoint [IP:PORT/INT] '%s' (--redis)", redisps); } @@ -1302,7 +1319,8 @@ static void options(int *argc, char ***argv, charp_ht templates) { if (redisps_write) { if (redis_ep_parse(&rtpe_config.redis_write_ep, &rtpe_config.redis_write_db, &rtpe_config.redis_write_hostname, - &rtpe_config.redis_write_auth, "RTPENGINE_REDIS_WRITE_AUTH_PW", redisps_write)) + &rtpe_config.redis_write_auth, "RTPENGINE_REDIS_WRITE_AUTH_PW", &rtpe_config.redis_write_via_sentinel, + redisps_write)) { die("Invalid Redis endpoint [IP:PORT/INT] '%s' (--redis-write)", redisps_write); } @@ -1310,14 +1328,38 @@ static void options(int *argc, char ***argv, charp_ht templates) { if (redisps_subscribe) if (redis_ep_parse(&rtpe_config.redis_subscribe_ep, &rtpe_config.redis_subscribe_db, &rtpe_config.redis_subscribe_hostname, - &rtpe_config.redis_subscribe_auth,"RTPENGINE_REDIS_SUBSCRIBE_AUTH_PW", redisps_subscribe)) + &rtpe_config.redis_subscribe_auth,"RTPENGINE_REDIS_SUBSCRIBE_AUTH_PW", &rtpe_config.redis_subscribe_via_sentinel, + redisps_subscribe)) { rtpe_config.redis_subscribe_db = -1; if (redis_ep_parse(&rtpe_config.redis_subscribe_ep, NULL, &rtpe_config.redis_subscribe_hostname, - &rtpe_config.redis_subscribe_auth,"RTPENGINE_REDIS_SUBSCRIBE_AUTH_PW", redisps_subscribe)) + &rtpe_config.redis_subscribe_auth,"RTPENGINE_REDIS_SUBSCRIBE_AUTH_PW", &rtpe_config.redis_subscribe_via_sentinel, + redisps_subscribe)) die("Invalid Redis endpoint [IP:PORT/INT] '%s' (--redis-subscribe)", redisps_subscribe); } + if (rtpe_config.redis_via_sentinel || rtpe_config.redis_write_via_sentinel + || rtpe_config.redis_subscribe_via_sentinel) + { + if (!redis_sentinels) + die("A Redis endpoint is 'sentinel', but --redis-sentinel is not set"); + if (!redis_sentinel_master) + die("--redis-sentinel-master must be set with --redis-sentinel"); + if (redis_sentinel_interval == -1) + redis_sentinel_interval = 1000; + if (redis_sentinel_interval <= 0) + die("Invalid --redis-sentinel-interval (must be a positive number of milliseconds)"); + rtpe_config.redis_sentinel_interval = redis_sentinel_interval; + rtpe_redis_sentinel = redis_sentinel_new(redis_sentinels, redis_sentinel_master, + "RTPENGINE_REDIS_SENTINEL_AUTH_PW"); + if (!rtpe_redis_sentinel) + die("Invalid Redis Sentinel list [PW@]IP:PORT[,IP:PORT...] '%s' (--redis-sentinel)", + redis_sentinels); + } + else if (redis_sentinels || redis_sentinel_master || redis_sentinel_interval != -1) + die("--redis-sentinel, --redis-sentinel-master or --redis-sentinel-interval is set, " + "but no Redis endpoint is 'sentinel'"); + if (rtpe_config.fmt < 0 || rtpe_config.fmt > 2) die("Invalid XMLRPC format"); @@ -1910,28 +1952,51 @@ static void create_everything(void) { create_listeners(&rtpe_config.ng_tcp_listen_ep, &rtpe_control_ng_tcp, (void *(*)(const endpoint_t *)) control_ng_tcp_new, false, "TCP NG control"); create_listeners(&rtpe_config.cli_listen_ep, &rtpe_cli, (void *(*)(const endpoint_t *)) cli_new, false, "CLI"); - if (!is_addr_unspecified(&rtpe_config.redis_write_ep.address)) { + // the roles that use Sentinel require a master, and never re-resolve + bool sent_r = rtpe_config.redis_via_sentinel, sent_w = rtpe_config.redis_write_via_sentinel, + sent_s = rtpe_config.redis_subscribe_via_sentinel; + + if (rtpe_redis_sentinel) { + // the master's password, for the role check on the Sentinels' answer + redis_sentinel_set_probe_auth(rtpe_redis_sentinel, + sent_w ? rtpe_config.redis_write_auth + : sent_r ? rtpe_config.redis_auth + : rtpe_config.redis_subscribe_auth); + ilog(LOG_INFO, "Using Redis Sentinel for the master of '%s'", + redis_sentinel_master_name(rtpe_redis_sentinel)); + if (!rtpe_config.redis_cmd_timeout) + ilog(LOG_WARN, "Redis Sentinel is used without --redis-cmd-timeout: a master that " + "stops answering holds Redis operations until it answers again"); + if (!redis_sentinel_discover(rtpe_redis_sentinel) && !rtpe_config.no_redis_required) + die("Cannot start up without a master of '%s' from Redis Sentinel! " + "See also NO_REDIS_REQUIRED parameter.", + redis_sentinel_master_name(rtpe_redis_sentinel)); + } + + if (!is_addr_unspecified(&rtpe_config.redis_write_ep.address) || sent_w) { rtpe_redis_write = redis_new(&rtpe_config.redis_write_ep, rtpe_config.redis_write_db, rtpe_config.redis_write_hostname, rtpe_config.redis_write_auth, MASTER_REDIS_ROLE, rtpe_config.no_redis_required, - rtpe_config.redis_resolve_on_reconnect); + rtpe_config.redis_resolve_on_reconnect && !sent_w, + sent_w ? rtpe_redis_sentinel : NULL); if (!rtpe_redis_write) die("Cannot start up without running Redis %s write database! See also NO_REDIS_REQUIRED parameter.", endpoint_print_buf(&rtpe_config.redis_write_ep)); } - if (!is_addr_unspecified(&rtpe_config.redis_subscribe_ep.address)) { + if (!is_addr_unspecified(&rtpe_config.redis_subscribe_ep.address) || sent_s) { rtpe_redis_notify = redis_new(&rtpe_config.redis_subscribe_ep, rtpe_config.redis_subscribe_db, rtpe_config.redis_subscribe_hostname, rtpe_config.redis_subscribe_auth, - ANY_REDIS_ROLE, + (sent_s ? MASTER_REDIS_ROLE : ANY_REDIS_ROLE), rtpe_config.no_redis_required, - rtpe_config.redis_resolve_on_reconnect); + rtpe_config.redis_resolve_on_reconnect && !sent_s, + sent_s ? rtpe_redis_sentinel : NULL); if (!rtpe_redis_notify) die("Cannot start up without running Redis %s subscribe database! See also NO_REDIS_REQUIRED parameter.", @@ -1942,14 +2007,15 @@ static void create_everything(void) { } } - if (!is_addr_unspecified(&rtpe_config.redis_ep.address)) { + if (!is_addr_unspecified(&rtpe_config.redis_ep.address) || sent_r) { rtpe_redis = redis_new(&rtpe_config.redis_ep, rtpe_config.redis_db, rtpe_config.redis_hostname, rtpe_config.redis_auth, - (rtpe_redis_write ? ANY_REDIS_ROLE : MASTER_REDIS_ROLE), + ((rtpe_redis_write && !sent_r) ? ANY_REDIS_ROLE : MASTER_REDIS_ROLE), rtpe_config.no_redis_required, - rtpe_config.redis_resolve_on_reconnect); + rtpe_config.redis_resolve_on_reconnect && !sent_r, + sent_r ? rtpe_redis_sentinel : NULL); if (!rtpe_redis) die("Cannot start up without running Redis %s database! " @@ -1961,9 +2027,10 @@ static void create_everything(void) { rtpe_config.redis_db, rtpe_config.redis_hostname, rtpe_config.redis_auth, - (rtpe_redis_write ? ANY_REDIS_ROLE : MASTER_REDIS_ROLE), + ((rtpe_redis_write && !sent_r) ? ANY_REDIS_ROLE : MASTER_REDIS_ROLE), rtpe_config.no_redis_required, - rtpe_config.redis_resolve_on_reconnect); + rtpe_config.redis_resolve_on_reconnect && !sent_r, + sent_r ? rtpe_redis_sentinel : NULL); if (!rtpe_redis_notify) die("Cannot start up without running notification Redis %s database! " @@ -2130,12 +2197,15 @@ int main(int argc, char **argv) { thread_create_looper(media_player_expire, rtpe_config.idle_scheduling, rtpe_config.idle_priority, "cache expiry", rtpe_config.expiry_timer * 1000000LL); - if (!is_addr_unspecified(&rtpe_config.redis_ep.address) && initial_rtpe_config.redis_delete_async) + if (rtpe_redis && initial_rtpe_config.redis_delete_async) thread_create_detach(redis_delete_async_loop, NULL, "redis async"); if (rtpe_redis_notify) thread_create_detach(redis_notify_loop, NULL, "redis notify"); + if (rtpe_redis_sentinel) + thread_create_detach(redis_sentinel_loop, rtpe_redis_sentinel, "redis sentinel"); + do_redis_restore(); if (graphite_is_enabled()) @@ -2193,7 +2263,7 @@ int main(int argc, char **argv) { websocket_stop(); - if (!is_addr_unspecified(&rtpe_config.redis_ep.address) && initial_rtpe_config.redis_delete_async) + if (rtpe_redis && initial_rtpe_config.redis_delete_async) redis_async_event_base_action(rtpe_redis_write, EVENT_BASE_LOOPBREAK); if (rtpe_redis_notify) @@ -2201,10 +2271,10 @@ int main(int argc, char **argv) { threads_join_all(true); - if (!is_addr_unspecified(&rtpe_config.redis_ep.address) && initial_rtpe_config.redis_delete_async) + if (rtpe_redis && initial_rtpe_config.redis_delete_async) redis_async_event_base_action(rtpe_redis_write, EVENT_BASE_FREE); - if (!is_addr_unspecified(&rtpe_config.redis_ep.address) && rtpe_redis_notify) + if (rtpe_redis && rtpe_redis_notify) redis_async_event_base_action(rtpe_redis_notify, EVENT_BASE_FREE); ilog(LOG_INFO, "Version %s shutting down", RTPENGINE_VERSION); @@ -2230,6 +2300,8 @@ int main(int argc, char **argv) { if (rtpe_redis_write != rtpe_redis) redis_close(rtpe_redis_write); redis_close(rtpe_redis_notify); + redis_sentinel_free(rtpe_redis_sentinel); + rtpe_redis_sentinel = NULL; free_prefix(); log_free(); diff --git a/daemon/redis.c b/daemon/redis.c index 2a391015b..372755c51 100644 --- a/daemon/redis.c +++ b/daemon/redis.c @@ -34,6 +34,7 @@ #include "codec.h" #include "sdp.h" #include "statistics.h" +#include "redis_sentinel.h" #define REDIS_CONNECT_ROLE -2 /* redis_connect(): the server has the wrong role */ @@ -97,8 +98,6 @@ static int redisCommandNR(redisContext *r, const char *fmt, ...) #define REDIS_FMT(x) (int) (x)->len, (x)->str -#define rlog(l, x...) ilog(l | LOG_FLAG_RESTORE, x) - // To protect against a restore race condition: Keyspace notifications are set up @@ -277,8 +276,12 @@ static int redis_consume(struct redis *r) { redis_write_result(r, rp); if (redis_is_role_error(rp)) role_error = true; + else if (r->sentinel && rp->type == REDIS_REPLY_ERROR && !strncmp(rp->str, "NOREPLICAS", 10)) + redis_sentinel_wake(r->sentinel, true); // the master may have been replaced freeReplyObject(rp); } + if (role_error && r->sentinel) + redis_sentinel_wake(r->sentinel, false); if (role_error && r->role == MASTER_REDIS_ROLE) { rlog(LOG_WARN, "Redis %s is not a master: dropping the connection", endpoint_print_buf(&r->endpoint)); @@ -308,6 +311,36 @@ int redis_set_cmd_timeout(struct redis *r, int64_t timeout) { return redis_set_timeout(r, timeout); } +/* called with r->lock held + * With Sentinel, takes the master that the Sentinel thread published, if it + * is a new one, and drops the connection to the old one: true if so. Not + * while a reconnect is under way: the next check follows. */ +static bool redis_sentinel_follow(struct redis *r) { + if (!r->sentinel || r->reconnecting || redis_sentinel_gen(r->sentinel) == r->gen) + return false; + + endpoint_t ep; + uint64_t gen = redis_sentinel_get(r->sentinel, &ep); + bool moved = !r->gen || !endpoint_eq(&ep, &r->endpoint); + r->gen = gen; + if (!moved) + return false; + + rlog(LOG_NOTICE, "Redis: following the master of '%s' from %s to %s", + redis_sentinel_master_name(r->sentinel), + r->ctx ? endpoint_print_buf(&r->endpoint) : "none", endpoint_print_buf(&ep)); + r->endpoint = ep; + sockaddr_print(&ep.address, r->host, sizeof(r->host)); + // try the new master at once, whatever happened with the old one + r->restore_tick_us = 0; + r->consecutive_errors = 0; + if (r->ctx) + redisFree(r->ctx); + r->ctx = NULL; + r->state = REDIS_STATE_DISCONNECTED; + return true; +} + int redis_reconnect(struct redis* r) { int rval; LOCK(&r->lock); @@ -317,6 +350,10 @@ int redis_reconnect(struct redis* r) { rlog(LOG_WARN, "Redis %s: a reconnect is already in progress", endpoint_print_buf(&r->endpoint)); return -1; } + redis_sentinel_follow(r); + if (r->sentinel && !r->gen) + return -1; + rval = redis_connect(r, 0, r->update_resolve); if (rval) r->state = REDIS_STATE_DISCONNECTED; @@ -680,6 +717,7 @@ int redis_async_event_base_action(struct redis *r, enum event_base_action action rlog(LOG_ERR, "redis_async_event_base_action: NULL r"); return -1; } else { + LOCK(&r->lock); // (Sentinel moves the endpoint) rlog(LOG_DEBUG, "redis_async_event_base_action: Use Redis %s", endpoint_print_buf(&r->endpoint)); } @@ -992,7 +1030,8 @@ void redis_notify_loop(void *d) { } struct redis *redis_new(const endpoint_t *ep, int db, const char *hostname, const char *auth, - enum redis_role role, int no_redis_required, bool update_resolve) { + enum redis_role role, int no_redis_required, bool update_resolve, + struct redis_sentinel *sentinel) { struct redis *r; r = g_new0(struct redis, 1); @@ -1007,9 +1046,23 @@ struct redis *redis_new(const endpoint_t *ep, int db, const char *hostname, cons r->restore_tick_us = 0; r->consecutive_errors = 0; r->update_resolve = update_resolve; + r->sentinel = sentinel; mutex_init(&r->lock); cond_init(&r->reconnect_cond); + if (sentinel) { + r->gen = redis_sentinel_get(sentinel, &r->endpoint); + sockaddr_print(&r->endpoint.address, r->host, sizeof(r->host)); + if (!r->gen) { + if (r->no_redis_required) { + rlog(LOG_WARN, "Starting with no master of '%s' known from Redis Sentinel !", + redis_sentinel_master_name(sentinel)); + return r; + } + goto err; + } + } + if (redis_connect(r, 10, false)) { if (r->no_redis_required) { rlog(LOG_WARN, "Starting with no initial connection to Redis %s !", @@ -1038,7 +1091,8 @@ struct redis *redis_dup(const struct redis *r, int db) { r->auth, r->role, r->no_redis_required, - r->update_resolve); + r->update_resolve, + r->sentinel); } void redis_close(struct redis *r) { @@ -1090,6 +1144,18 @@ static int redis_check_conn(struct redis *r) { rtpe_now = now_us(); } + bool moved = redis_sentinel_follow(r); + if (r->sentinel && !r->gen) { + // no master known yet + r->skipped++; + if (!r->skipped_logged) + rlog(LOG_WARNING, "No master of '%s' known from Redis Sentinel yet", + redis_sentinel_master_name(r->sentinel)); + r->skipped_logged = true; + redis_sentinel_wake(r->sentinel, false); + return REDIS_STATE_DISCONNECTED; + } + if ((r->state == REDIS_STATE_DISCONNECTED) && r->restore_tick_us > rtpe_now) { // every Redis operation ends up here: say it once r->skipped++; @@ -1101,8 +1167,9 @@ static int redis_check_conn(struct redis *r) { } if (r->state == REDIS_STATE_DISCONNECTED) { - ilog(LOG_INFO, "RE-Establishing connection for Redis server %s (%u operations skipped)", - endpoint_print_buf(&r->endpoint), r->skipped); + if (!moved) + ilog(LOG_INFO, "RE-Establishing connection for Redis server %s (%u operations skipped)", + endpoint_print_buf(&r->endpoint), r->skipped); r->skipped = 0; r->skipped_logged = false; } @@ -1121,7 +1188,7 @@ static int redis_check_conn(struct redis *r) { // redis is disconnected if (r->state == REDIS_STATE_CONNECTED) { rlog(LOG_ERR, "Lost connection to Redis '%s'", - r->hostname); + r->hostname ? : r->host); r->state = REDIS_STATE_DISCONNECTED; } @@ -1139,6 +1206,8 @@ static int redis_check_conn(struct redis *r) { if (ret) { // redis is disconnected redis_count_err_and_disable(r); + if (r->sentinel && ret == REDIS_CONNECT_ROLE) + redis_sentinel_wake(r->sentinel, false); // After the wrong role, or a slow failure (a timeout), pause (one // second by default): a failover takes longer than that, so the // next attempt would fail the same way. A quick failure diff --git a/daemon/redis_sentinel.c b/daemon/redis_sentinel.c new file mode 100644 index 000000000..b924ce29f --- /dev/null +++ b/daemon/redis_sentinel.c @@ -0,0 +1,401 @@ +#include "redis_sentinel.h" + +#include +#include +#include +#include +#include +#include + +#include "helpers.h" +#include "log_d.h" +#include "main.h" +#include "redis.h" + +struct sentinel_node { + char *host; /* as configured, for --redis-resolve-on-reconnect */ + endpoint_t ep; + bool failing; /* last query failed (logged once) */ +}; + +struct redis_sentinel { + struct sentinel_node *nodes; + unsigned int num_nodes; + char *auth; + char *master_name; + const char *probe_auth; /* data node password, for ROLE */ + + /* published master: written by the thread, read by the connections */ + mutex_t lock; + cond_t cond; + bool wake; + endpoint_t master; + uint64_t epoch; + atomic64 gen; /* 0: no master known yet */ + atomic64 limited_wake_us; + + /* thread only */ + bool none_answered; /* logged once */ + bool epoch_back; /* logged once */ + struct thread_waker waker; +}; + +struct redis_sentinel *rtpe_redis_sentinel; + + +/* [PW@]HOST:PORT[,HOST:PORT...] */ +struct redis_sentinel *redis_sentinel_new(const char *list, const char *master_name, + const char *auth_env) +{ + if (!list || !*list || !master_name || !*master_name) + return NULL; + + struct redis_sentinel *s = g_new0(struct redis_sentinel, 1); + mutex_init(&s->lock); + cond_init(&s->cond); + g_autofree char *dup = g_strdup(list); + g_auto(GStrv) items = NULL; + char *hosts = dup; + + char *at = strrchr(hosts, '@'); + if (at) { + *at = '\0'; + if (!*hosts) + goto err; // "@host:port": an empty password is a mistake + s->auth = g_strdup(hosts); + hosts = at + 1; + } + else { + const char *env = getenv(auth_env); + if (env && *env) + s->auth = g_strdup(env); + } + + items = g_strsplit(hosts, ",", -1); + s->num_nodes = g_strv_length(items); + if (!s->num_nodes) + goto err; + s->nodes = g_new0(struct sentinel_node, s->num_nodes); + + for (unsigned int i = 0; i < s->num_nodes; i++) { + struct sentinel_node *n = &s->nodes[i]; + char *item = g_strstrip(items[i]); + char *colon = strrchr(item, ':'); + if (!colon) + goto err; + if (!endpoint_parse_any_getaddrinfo_full(&n->ep, item) || !n->ep.port) + goto err; + n->host = g_strndup(item, colon - item); + } + + s->master_name = g_strdup(master_name); + return s; + +err: + redis_sentinel_free(s); + return NULL; +} + +void redis_sentinel_free(struct redis_sentinel *s) { + if (!s) + return; + for (unsigned int i = 0; s->nodes && i < s->num_nodes; i++) + g_free(s->nodes[i].host); + g_free(s->nodes); + g_free(s->auth); + g_free(s->master_name); + mutex_destroy(&s->lock); + g_free(s); +} + +void redis_sentinel_set_probe_auth(struct redis_sentinel *s, const char *auth) { + s->probe_auth = auth; +} + +const char *redis_sentinel_master_name(const struct redis_sentinel *s) { + return s->master_name; +} + + +uint64_t redis_sentinel_gen(struct redis_sentinel *s) { + return atomic64_get(&s->gen); +} + +uint64_t redis_sentinel_get(struct redis_sentinel *s, endpoint_t *ep) { + LOCK(&s->lock); + *ep = s->master; + return atomic64_get_na(&s->gen); +} + +/* "limited": at most once per poll interval (NOREPLICAS lasts as long as the + * master lacks replicas) */ +void redis_sentinel_wake(struct redis_sentinel *s, bool limited) { + if (limited) { + int64_t now = now_us(); + int64_t last = atomic64_get_na(&s->limited_wake_us); + if (last && now >= last && now - last < rtpe_config.redis_sentinel_interval * 1000LL) + return; + if (!atomic64_set_if(&s->limited_wake_us, now, last)) + return; + } + LOCK(&s->lock); + s->wake = true; + cond_signal(&s->cond); +} + + +static redisContext *sentinel_connect(const endpoint_t *ep, const char *auth) { + char host[64]; + sockaddr_print(&ep->address, host, sizeof(host)); + int64_t connect_timeout = atomic_get_na(&rtpe_config.redis_connect_timeout) * 1000LL; + // never without a command timeout ("rediscmdtimeout 0"): a server that + // accepts but never answers would stop the thread for good + int64_t cmd_timeout = atomic_get_na(&rtpe_config.redis_cmd_timeout) * 1000LL ? : 1000000; + + redisContext *ctx = redisConnectWithTimeout(host, ep->port, timeval_from_us(connect_timeout)); + if (!ctx) + return NULL; + const char *err = NULL; + redisReply *rp = NULL; + if (ctx->err || redisSetTimeout(ctx, timeval_from_us(cmd_timeout))) + err = ctx->errstr; + else if (auth && !(rp = redisCommand(ctx, "AUTH %s", auth))) + err = ctx->errstr; + else if (rp && rp->type == REDIS_REPLY_ERROR) + err = "authentication failed"; + if (rp) + freeReplyObject(rp); + if (!err) + return ctx; + rlog(LOG_WARN | LOG_FLAG_LIMIT, "Redis Sentinel: cannot connect to %s: %s", + endpoint_print_buf(ep), err); + redisFree(ctx); + return NULL; +} + +static const char *reply_field(const redisReply *rp, const char *name) { + for (size_t i = 0; i + 1 < rp->elements; i += 2) { + const redisReply *k = rp->element[i], *v = rp->element[i + 1]; + if (k->type == REDIS_REPLY_STRING && v->type == REDIS_REPLY_STRING && !strcmp(k->str, name)) + return v->str; + } + return NULL; +} + +static const char *sentinel_parse(const redisReply *rp, const redisReply *ra, + endpoint_t *master, uint64_t *epoch) +{ + if (rp->type == REDIS_REPLY_ERROR) + return rp->str; + if (ra->type == REDIS_REPLY_ERROR) + return ra->str; + const char *epoch_s = rp->type == REDIS_REPLY_ARRAY ? reply_field(rp, "config-epoch") : NULL; + if (!epoch_s || ra->type != REDIS_REPLY_ARRAY || ra->elements != 2 + || ra->element[0]->type != REDIS_REPLY_STRING + || ra->element[1]->type != REDIS_REPLY_STRING) + return "unexpected reply"; + char *end; + *epoch = strtoull(epoch_s, &end, 10); + if (end == epoch_s || *end) + return "unexpected reply"; + unsigned long port = strtoul(ra->element[1]->str, &end, 10); + if (end == ra->element[1]->str || *end || !port || port > 0xffff) + return "unexpected reply"; + // a host name (announce-hostnames) is resolved here, in the thread + if (!sockaddr_getaddrinfo(&master->address, ra->element[0]->str)) + return "cannot resolve the master's address"; + master->port = port; + return NULL; +} + +/* Epoch first, then address, so that a failover between the two replies can + * only pair an older epoch with the newer address. (The "ip" of SENTINEL + * MASTER switches seconds after the epoch.) */ +static bool sentinel_query(struct redis_sentinel *s, struct sentinel_node *n, + endpoint_t *master, uint64_t *epoch) +{ + if (rtpe_config.redis_resolve_on_reconnect) { + sockaddr_t a; + if (sockaddr_getaddrinfo(&a, n->host)) + n->ep.address = a; + } + + redisContext *ctx = sentinel_connect(&n->ep, s->auth); + if (!ctx) { + n->failing = true; + return false; + } + + redisReply *rp = NULL, *ra = NULL; + const char *err; + redisAppendCommand(ctx, "SENTINEL MASTER %s", s->master_name); + redisAppendCommand(ctx, "SENTINEL GET-MASTER-ADDR-BY-NAME %s", s->master_name); + if (redisGetReply(ctx, (void **) &rp) != REDIS_OK || redisGetReply(ctx, (void **) &ra) != REDIS_OK) + err = ctx->errstr; + else + err = sentinel_parse(rp, ra, master, epoch); + + if (err && !n->failing) + rlog(LOG_WARN, "Redis Sentinel %s: cannot get the master of '%s': %s", + endpoint_print_buf(&n->ep), s->master_name, err); + else if (!err && n->failing) + rlog(LOG_NOTICE, "Redis Sentinel %s answers again", endpoint_print_buf(&n->ep)); + n->failing = err != NULL; + + if (rp) + freeReplyObject(rp); + if (ra) + freeReplyObject(ra); + redisFree(ctx); + return !err; +} + +static bool sentinel_is_master(struct redis_sentinel *s, const endpoint_t *ep, uint64_t epoch) { + redisContext *ctx = sentinel_connect(ep, s->probe_auth); + if (!ctx) + return false; + + bool ret = false; + redisReply *rp = redisCommand(ctx, "ROLE"); + if (rp && rp->type == REDIS_REPLY_ARRAY && rp->elements && rp->element[0]->type == REDIS_REPLY_STRING) { + ret = !strcmp(rp->element[0]->str, "master"); + if (!ret) + rlog(LOG_WARN | LOG_FLAG_LIMIT, "Redis Sentinel names %s as master of '%s' " + "(config epoch %" PRIu64 "), but it is a %s: not used", + endpoint_print_buf(ep), s->master_name, epoch, rp->element[0]->str); + } + else + rlog(LOG_WARN | LOG_FLAG_LIMIT, "Redis Sentinel: cannot check the role of %s: %s", + endpoint_print_buf(ep), !rp ? ctx->errstr + : rp->type == REDIS_REPLY_ERROR ? rp->str : "unexpected reply"); + + if (rp) + freeReplyObject(rp); + redisFree(ctx); + return ret; +} + +static void sentinel_publish(struct redis_sentinel *s, const endpoint_t *m, uint64_t epoch) { + { + LOCK(&s->lock); + s->master = *m; + s->epoch = epoch; + atomic64_inc(&s->gen); + } + rlog(LOG_NOTICE, "Redis Sentinel: master of '%s' is now %s (config epoch %" PRIu64 ")", + s->master_name, endpoint_print_buf(m), epoch); +} + +/* true if the candidate is not a master: ask again soon */ +static bool sentinel_round(struct redis_sentinel *s) { + endpoint_t best = {0}, pub; + uint64_t best_epoch = 0, pub_epoch, pub_gen; + bool found = false; + + { + LOCK(&s->lock); + pub = s->master; + pub_epoch = s->epoch; + pub_gen = atomic64_get_na(&s->gen); + } + + for (unsigned int i = 0; i < s->num_nodes; i++) { + endpoint_t m = {0}; + uint64_t e = 0; + if (!sentinel_query(s, &s->nodes[i], &m, &e)) + continue; + // at the same epoch, keep the master in use (no flapping while + // the Sentinels disagree) + if (!found || e > best_epoch || (e == best_epoch && pub_gen && endpoint_eq(&m, &pub))) { + best = m; + best_epoch = e; + found = true; + } + } + + if (!found) { + if (!s->none_answered) + rlog(LOG_ERR, "Redis Sentinel: no Sentinel answered for '%s'%s%s", + s->master_name, + pub_gen ? "; keeping " : "", + pub_gen ? endpoint_print_buf(&pub) : ""); + s->none_answered = true; + return false; + } + s->none_answered = false; + + // Never back to an older epoch: a demoted master still says "master" + // until Sentinel reconfigures it. The master in use is kept for as long + // as it answers as a master (its connection checks the role). + if (pub_gen && best_epoch < pub_epoch) { + if (!s->epoch_back) + rlog(LOG_ERR, "Redis Sentinel: every Sentinel that answers names %s as master of '%s' " + "at config epoch %" PRIu64 ", older than %" PRIu64 " (%s, in use): not " + "followed. If the Sentinels were redeployed without their state, their " + "epochs went back, and rtpengine must be restarted", + endpoint_print_buf(&best), s->master_name, best_epoch, pub_epoch, + endpoint_print_buf(&pub)); + s->epoch_back = true; + return false; + } + s->epoch_back = false; + + if (pub_gen && endpoint_eq(&best, &pub)) { + LOCK(&s->lock); + s->epoch = best_epoch; + return false; + } + + if (!sentinel_is_master(s, &best, best_epoch)) + return true; + + sentinel_publish(s, &best, best_epoch); + return false; +} + +/* at startup: as for a data node, fails at once if nothing answers, and asks + * again once a second (11 times) while the candidate is not a master */ +bool redis_sentinel_discover(struct redis_sentinel *s) { + for (int i = 0; i < 11; i++) { + if (i) + usleep(1000000); + bool refused = sentinel_round(s); + if (redis_sentinel_gen(s)) + return true; + if (!refused) + break; + } + return false; +} + +void redis_sentinel_loop(struct redis_sentinel *s) { + int64_t interval_us = rtpe_config.redis_sentinel_interval * 1000LL; + int64_t next = now_us() + interval_us, last = 0; + + // a round is bounded by the timeouts; at shutdown, the waker ends the wait + s->waker.lock = &s->lock; + s->waker.cond = &s->cond; + thread_waker_add(&s->waker); + + while (!rtpe_shutdown) { + { + LOCK(&s->lock); + // a wake-up comes at least 250 ms after the last round + while (!rtpe_shutdown) { + int64_t until = s->wake ? MIN(next, last + 250000) : next; + if (now_us() >= until) + break; + cond_timedwait(&s->cond, &s->lock, until); + } + s->wake = false; + } + if (rtpe_shutdown) + break; + + bool refused = sentinel_round(s); + last = now_us(); + next = last + (refused ? MIN(250000, interval_us) : interval_us); + } + + thread_waker_del(&s->waker); +} diff --git a/docs/rtpengine.md b/docs/rtpengine.md index a822083b5..ea607645f 100644 --- a/docs/rtpengine.md +++ b/docs/rtpengine.md @@ -560,6 +560,11 @@ call to inject-DTMF won't be sent to __\-\-dtmf-log-dest=__ or __\-\-listen-tcp- When this option is given, __rtpengine__ will delay startup until the Redis database adopts the master role (but see below). + Instead of *IP*:*PORT*, the keyword __sentinel__ can be given, for + example *foobar@sentinel/12*: the address is then the current master of + a Redis Sentinel group, see __\-\-redis-sentinel__. This is also + possible for __\-\-redis-write__ and __\-\-redis-subscribe__. + - __-w__, __\-\-redis-write=__\[*PW*@\]*IP*:*PORT*/*INT* Configures a second Redis database for write operations. @@ -590,7 +595,10 @@ call to inject-DTMF won't be sent to __\-\-dtmf-log-dest=__ or __\-\-listen-tcp- This option takes precedence over __\-\-redis__, if configured. When __\-\-subscribe-keyspace__ is also configured, the keyspace part of __\-\-redis-subscribe=__ is not used, the former takes precedence. - The keyspace number can also be omitted altogether. + The keyspace number can also be omitted altogether. With __sentinel__ + instead of *IP*:*PORT* (see __\-\-redis-sentinel__), the subscription is + made on the current master: a replica is not used, even though + notifications could be read from one. For password protected Redis servers, the environment variable for the password is __RTPENGINE\_REDIS\_SUBSCRIBE\_AUTH\_PW__. @@ -718,6 +726,40 @@ call to inject-DTMF won't be sent to __\-\-dtmf-log-dest=__ or __\-\-listen-tcp- remote redis server try to re-resolve if the redis hostname was an FQDN and not IP address. It is a boolean value (either __true__ or __false__). +- __\-\-redis-sentinel=__\[*PW*@\]*IP*:*PORT*\[,*IP*:*PORT*...\] + + The Redis Sentinels to ask for the master, for every Redis option given + with __sentinel__ instead of *IP*:*PORT*; requires + __\-\-redis-sentinel-master__. Their password, usually not the one of + the data nodes, is given before the `@` symbol or in + __RTPENGINE\_REDIS\_SENTINEL\_AUTH\_PW__. Host names are resolved at + startup, and before each query with __\-\-redis-resolve-on-reconnect__. + + The Sentinels are asked every __\-\-redis-sentinel-interval__ and when a + connection finds a replica. The highest configuration epoch wins, and + never an older one than the master in use: if all are older (Sentinels + redeployed without their state), an error is logged and __rtpengine__ + must be restarted. The master is checked with __ROLE__, then every + connection through Sentinel moves to it at its next operation. These + connections require the master role, as the write connection does. Set + __\-\-redis-cmd-timeout__ (for example to 1000): without it, a master + that stops answering holds Redis operations until it answers again. + + If no Sentinel answers at startup, __rtpengine__ does not start unless + __\-\-no-redis-required__ is given; if none answers later, the last + master is kept. + +- __\-\-redis-sentinel-master=__*STRING* + + The name of the master that the Redis Sentinels monitor. + +- __\-\-redis-sentinel-interval=__*INT* + + How often, in milliseconds, the Redis Sentinels are asked for the master + when nothing prompted a query. The default value is 1000. + __\-\-redis-sentinel-master__ and __\-\-redis-sentinel-interval__ are + refused without a Redis option that uses __sentinel__. + - __\-\-redis-format=bencode__\|__JSON__ Selects the format for serialised call data written to Redis or KeyDB. The @@ -2150,6 +2192,10 @@ IPv6. specified, in which case the one specified in __\-\-redis__ will be used for read operations only. +- __RTPENGINE\_REDIS\_SENTINEL\_AUTH\_PW__ + + Redis Sentinel password, for __\-\-redis-sentinel__. + ## FILES - `/etc/rtpengine/rtpengine.conf` diff --git a/etc/rtpengine.conf b/etc/rtpengine.conf index d5dc2e429..f137a9f5b 100644 --- a/etc/rtpengine.conf +++ b/etc/rtpengine.conf @@ -84,6 +84,10 @@ recording-method = proc # redis-reconnect-pause = 1000 # redis-reconnect-slow = 100 # redis-resolve-on-reconnect = false +# redis-write = password@sentinel/42 +# redis-sentinel = password@10.0.0.11:26379,10.0.0.12:26379,10.0.0.13:26379 +# redis-sentinel-master = mymaster +# redis-sentinel-interval = 1000 # redis-tcp-keepalive-time = 1 # redis-tcp-keepalive-intvl = 1 # redis-tcp-keepalive-probes = 3 diff --git a/include/main.h b/include/main.h index 861e09fa9..6abd5f5bf 100644 --- a/include/main.h +++ b/include/main.h @@ -62,6 +62,7 @@ enum endpoint_learning { X(redis_tcp_keepalive_time) \ X(redis_tcp_keepalive_intvl) \ X(redis_tcp_keepalive_probes) \ + X(redis_sentinel_interval) \ X(num_threads) \ X(media_num_threads) \ X(codec_num_threads) \ @@ -139,6 +140,9 @@ enum endpoint_learning { X(player_cache) \ X(poller_per_thread) \ X(redis_resolve_on_reconnect) \ + X(redis_via_sentinel) \ + X(redis_write_via_sentinel) \ + X(redis_subscribe_via_sentinel) \ X(measure_rtp) \ X(amr_cn_dtx) \ X(evs_cn_dtx) \ diff --git a/include/redis.h b/include/redis.h index 1c2a2b4e4..c29fb2bc5 100644 --- a/include/redis.h +++ b/include/redis.h @@ -15,8 +15,13 @@ #include "crypto.h" #include "dtls.h" +struct redis_sentinel; + #define REDIS_RESTORE_NUM_THREADS 4 +// also logged while calls are being restored +#define rlog(l, x...) ilog(l | LOG_FLAG_RESTORE, x) + enum redis_role { MASTER_REDIS_ROLE = 0, SLAVE_REDIS_ROLE = 1, @@ -76,6 +81,9 @@ struct redis { int64_t async_last; bool update_resolve; + + struct redis_sentinel *sentinel; /* address from Redis Sentinel, or NULL */ + uint64_t gen; /* generation of the master in endpoint */ }; struct redis_hash { @@ -113,7 +121,8 @@ void redis_notify_loop(void *d); void redis_delete_async_loop(void *d); -struct redis *redis_new(const endpoint_t *, int, const char *, const char *, enum redis_role, int, bool); +struct redis *redis_new(const endpoint_t *, int, const char *, const char *, enum redis_role, int, bool, + struct redis_sentinel *); struct redis *redis_dup(const struct redis *r, int db); void redis_close(struct redis *r); int redis_restore(struct redis *, bool foreign, int db); diff --git a/include/redis_sentinel.h b/include/redis_sentinel.h new file mode 100644 index 000000000..0a31e1838 --- /dev/null +++ b/include/redis_sentinel.h @@ -0,0 +1,32 @@ +#ifndef __REDIS_SENTINEL_H__ +#define __REDIS_SENTINEL_H__ + +#include +#include + +#include "socket.h" + +/* + * A thread asks the Redis Sentinels for the master and publishes it with a + * generation number; the connections that use it compare the generation (one + * atomic read) and reconnect when it changes. All Sentinel I/O is done in + * that thread. + */ + +struct redis_sentinel; + +extern struct redis_sentinel *rtpe_redis_sentinel; + +struct redis_sentinel *redis_sentinel_new(const char *list, const char *master_name, + const char *auth_env); +void redis_sentinel_set_probe_auth(struct redis_sentinel *, const char *auth); +bool redis_sentinel_discover(struct redis_sentinel *); +void redis_sentinel_loop(struct redis_sentinel *); +void redis_sentinel_free(struct redis_sentinel *); + +uint64_t redis_sentinel_gen(struct redis_sentinel *); +uint64_t redis_sentinel_get(struct redis_sentinel *, endpoint_t *); +void redis_sentinel_wake(struct redis_sentinel *, bool limited); +const char *redis_sentinel_master_name(const struct redis_sentinel *); + +#endif diff --git a/perl/NGCP/Rtpengine/FakeRedis.pm b/perl/NGCP/Rtpengine/FakeRedis.pm index bc15b3f8e..552d409fc 100644 --- a/perl/NGCP/Rtpengine/FakeRedis.pm +++ b/perl/NGCP/Rtpengine/FakeRedis.pm @@ -2,12 +2,15 @@ package NGCP::Rtpengine::FakeRedis; # Fake Redis servers for the daemon tests: listeners on test addresses (made # local by tests-preload.so), and helpers that read the commands that -# rtpengine sends (RESP arrays of bulk strings) and answer them. +# rtpengine sends (RESP arrays of bulk strings) and answer them. Fake Redis +# Sentinels run in a child process (sentinels_start()), so that they answer +# while the test script handles the data nodes. use strict; use warnings; use Socket qw(AF_INET SOCK_STREAM sockaddr_in inet_aton); use File::Temp; +use IO::Handle; use Test::More; use Time::HiRes qw(time sleep); use IO::Select; @@ -18,9 +21,18 @@ use Exporter 'import'; our @EXPORT = qw(redis_listen redis_accept redis_conn redis_read redis_send redis_expect redis_expect_set redis_expect_del redis_close redis_closed readable info_reply redis_write_errors wait_log - redis_serve redis_sets_seen redis_record_after redis_serve_next_get); + redis_serve redis_sets_seen redis_record_after redis_serve_next_get + %role @probes wait_probe serve_probes + sentinels_start sentinels_set sentinel_log sentinel_queries); my %rbuf; # what was read from each socket and not parsed yet +my %node; # IP of each listener, by fileno + +# Role probes of the Sentinel thread (AUTH, then ROLE) on the data nodes are +# answered from %role (by IP, "master" by default), and recorded in @probes +# as [IP, password, role answered]. +our %role; +our @probes; sub redis_listen { @@ -29,6 +41,7 @@ sub redis_listen { socket($l, AF_INET, SOCK_STREAM, 0) or die; bind($l, sockaddr_in($port // 6379, inet_aton($ip))) or die "bind $ip: $!"; listen($l, 10) or die; + $node{fileno($l)} = $ip; return $l; } @@ -132,14 +145,40 @@ sub redis_accept { return $fd; } +sub role_reply { + my $role = shift; + return "*3\r\n\$6\r\nmaster\r\n:0\r\n*0\r\n" if $role eq 'master'; + return "*5\r\n\$5\r\nslave\r\n\$12\r\n203.0.113.44\r\n:6379\r\n\$9\r\nconnected\r\n:0\r\n"; +} + +# after the AUTH of a probe: answer its ROLE +sub probe_answer { + my ($fd, $ip, $auth) = @_; + my $r = $role{$ip} // 'master'; + redis_send($fd, role_reply($r)); + push(@probes, [$ip, $auth, $r]); + redis_close($fd); +} + # A new data connection: AUTH, SELECT, INFO answered with the role (default # "master"), then TYPE unless "refused" (the role is not the one wanted). -# Options: auth, db, role, short, replid, refused. +# Options: auth, db, role, short, replid, refused. Probes that come first +# are answered. sub redis_conn { my ($l, $n, %opt) = @_; - my $fd = redis_accept($l); - redis_expect($fd, ['AUTH', $opt{auth} // 'auth'], "+OK\r\n", "$n AUTH"); - redis_expect($fd, ['SELECT', $opt{db} // 2], "+OK\r\n", "$n SELECT"); + my ($fd, $auth, $c); + while (1) { + $fd = redis_accept($l); + $auth = redis_read($fd); + # (a probe can give up before it is answered) + redis_send($fd, "+OK\r\n") if $auth; + $c = $auth && redis_read($fd); + last if $c && $c->[0] ne 'ROLE'; + $c ? probe_answer($fd, $node{fileno($l)}, $auth->[1]) : redis_close($fd); + } + is(join(' ', @$auth), 'AUTH ' . ($opt{auth} // 'auth'), "$n AUTH"); + is(join(' ', @$c), 'SELECT ' . ($opt{db} // 2), "$n SELECT"); + redis_send($fd, "+OK\r\n"); redis_expect($fd, ['INFO'], info_reply($opt{role} // 'master', %opt), "$n INFO"); redis_expect($fd, ['TYPE', 'calls'], "+none\r\n", "$n TYPE") unless $opt{refused}; return $fd; @@ -318,4 +357,190 @@ sub wait_log { die "not in the log: $re\n"; } + + +# wait for a role probe on this listener +sub wait_probe { + my ($l, $n, $timeout) = @_; + my $fd = redis_accept($l, $timeout); + my $c = redis_expect($fd, ['AUTH', 'auth'], "+OK\r\n", "$n: probe AUTH with the data password"); + redis_expect($fd, ['ROLE'], undef, "$n: probe ROLE"); + probe_answer($fd, $node{fileno($l)}, $c->[1]); +} + +# for a while, answer every probe on these listeners, whatever arrives +sub serve_probes { + my ($secs, @l) = @_; + my $end = time() + $secs; + while ((my $left = $end - time()) > 0) { + my $rin = ''; + vec($rin, fileno($_), 1) = 1 for @l; + next unless select(my $rout = $rin, undef, undef, $left) > 0; + for my $l (grep { vec($rout, fileno($_), 1) } @l) { + my $fd = redis_accept($l); + my $a = eval { redis_read($fd, 1) }; + redis_send($fd, "+OK\r\n") if $a; + my $c = $a && eval { redis_read($fd, 1) }; + $c && $c->[0] eq 'ROLE' ? probe_answer($fd, $node{fileno($l)}, $a->[1]) : redis_close($fd); + } + } +} + + +# Fake Sentinels. The child re-reads a state file every 50 ms and logs every +# command it gets ("IP:PORT COMMAND ARG..."), plus "applied N" for each new +# state. Each Sentinel (IP, port 26379) is either "down" (not listening) or +# "MASTER_IP MASTER_PORT EPOCH [MONITORED_IP]": it names this master +# (GET-MASTER-ADDR-BY-NAME) at this config epoch (SENTINEL MASTER), whose +# "ip" is MONITORED_IP if given: a Sentinel running a failover reports the +# promoted replica in the former and still the old master in the latter. + +my ($sentinel_pid, $sentinel_state, $sentinel_log, $sentinel_password); +my $serial = 0; + +sub sentinel_log { + open(my $fh, '<', "$sentinel_log") or die; + local $/; + return <$fh> // ''; +} + +# how many times this Sentinel was asked for the master +sub sentinel_queries { + my $ip = shift; + my @l = sentinel_log() =~ /^\Q$ip\E:26379 SENTINEL MASTER mymaster$/mg; + return scalar(@l); +} + +# set what the Sentinels answer (IP => state), and wait until it is applied +sub sentinels_set { + my %st = @_; + $serial++; + open(my $fh, '>', "$sentinel_state.new") or die; + print $fh "serial $serial\npassword $sentinel_password\n"; + print $fh "$_:26379 $st{$_}\n" for sort(keys(%st)); + close($fh); + rename("$sentinel_state.new", "$sentinel_state") or die; + return unless $sentinel_pid; + for (1 .. 100) { + return if sentinel_log() =~ /^applied $serial$/m; + sleep(0.05); + } + die "fake Sentinels did not apply state $serial\n"; +} + +sub sentinels_start { + ($sentinel_password, my %st) = @_; + $sentinel_state = File::Temp->new(TEMPLATE => 'rtpe-sentinel-state.XXXXXXXXXX', TMPDIR => 1); + $sentinel_log = File::Temp->new(TEMPLATE => 'rtpe-sentinel-log.XXXXXXXXXX', TMPDIR => 1); + sentinels_set(%st); + $sentinel_pid = fork() // die; + if (!$sentinel_pid) { + exec($^X, (map { "-I$_" } grep { !ref($_) } @INC), '-MNGCP::Rtpengine::FakeRedis', + '-e', 'NGCP::Rtpengine::FakeRedis::sentinel_main(@ARGV)', + "$sentinel_state", "$sentinel_log") or die; + } + sentinels_set(%st); +} + +END { + if ($sentinel_pid) { + kill('TERM', $sentinel_pid); + waitpid($sentinel_pid, 0); + } +} + +# the child +sub sentinel_main { + my ($statefile, $logfile) = @_; + open(my $log, '>>', $logfile) or die; + $log->autoflush(1); + my $done = 0; + $SIG{TERM} = sub { $done = 1 }; + $SIG{PIPE} = 'IGNORE'; + my (%sentinels, %clients, $password); + my $applied = -1; + + my $read_state = sub { + open(my $fh, '<', $statefile) or return; + my ($ser, %want); + while (my $l = <$fh>) { + my @f = split(' ', $l); + if ($f[0] eq 'serial') { $ser = $f[1] } + elsif ($f[0] eq 'password') { $password = $f[1] } + else { $want{$f[0]} = [ @f[1 .. $#f] ] } + } + close($fh); + for my $s (keys(%want)) { + my $st = $sentinels{$s} //= { }; + $st->{state} = $want{$s}; + if ($want{$s}[0] eq 'down') { + close(delete($st->{listener})) if $st->{listener}; + } + elsif (!$st->{listener}) { + my ($ip, $port) = split(/:/, $s); + $st->{listener} = redis_listen($ip, $port); + } + } + # a Sentinel that goes down closes its clients too + for my $fn (keys(%clients)) { + next if $sentinels{$clients{$fn}{sentinel}}{listener}; + close($clients{$fn}{fh}); + delete($clients{$fn}); + } + if (defined($ser) && $ser != $applied) { + $applied = $ser; + print $log "applied $ser\n"; + } + }; + + my $reply = sub { + my ($c, $args) = @_; + my $st = $sentinels{$c->{sentinel}}{state}; + print $log "$c->{sentinel} @$args\n"; + my $cmd = uc($args->[0]) . ' ' . uc($args->[1] // ''); + if ($cmd =~ /^AUTH /) { + return "-WRONGPASS invalid username-password pair\r\n" if $args->[-1] ne $password; + $c->{authed} = 1; + return "+OK\r\n"; + } + return "-NOAUTH Authentication required.\r\n" unless $c->{authed}; + return "-ERR No such master with that name\r\n" if ($args->[2] // '') ne 'mymaster'; + my ($ip, $port, $epoch, $monitored) = @$st; + return "*2\r\n" . bulk($ip) . bulk($port) if $cmd eq 'SENTINEL GET-MASTER-ADDR-BY-NAME'; + my @kv = (name => 'mymaster', ip => $monitored // $ip, port => $port, flags => 'master', + 'num-slaves' => 2, 'num-other-sentinels' => 2, quorum => 2, 'config-epoch' => $epoch); + return '*' . scalar(@kv) . "\r\n" . join('', map { bulk($_) } @kv) if $cmd eq 'SENTINEL MASTER'; + return "-ERR unknown command\r\n"; + }; + + $read_state->(); + while (!$done) { + my $rin = ''; + my %l = map { fileno($_->{listener}) => $_ } grep { $_->{listener} } values(%sentinels); + vec($rin, $_, 1) = 1 for (keys(%l), keys(%clients)); + if (select(my $rout = $rin, undef, undef, 0.05) > 0) { + for my $fn (grep { vec($rout, $_, 1) } keys(%l)) { + my ($s) = grep { $sentinels{$_} == $l{$fn} } keys(%sentinels); + accept(my $fh, $l{$fn}{listener}) or next; + $clients{fileno($fh)} = { fh => $fh, buf => '', sentinel => $s }; + } + for my $fn (grep { vec($rout, $_, 1) } keys(%clients)) { + my $c = $clients{$fn}; + my $data; + if (!sysread($c->{fh}, $data, 65536)) { + close($c->{fh}); + delete($clients{$fn}); + next; + } + $c->{buf} .= $data; + while (my $args = parse_cmd(\$c->{buf})) { + syswrite($c->{fh}, $reply->($c, $args)); + } + } + } + $read_state->(); + } + exit(0); +} + 1; diff --git a/t/Makefile b/t/Makefile index f344c73ed..5772183d5 100644 --- a/t/Makefile +++ b/t/Makefile @@ -138,6 +138,8 @@ include ../lib/common.Makefile daemon-tests-dtx-dtmf \ daemon-tests-ice-force-answer \ daemon-tests-redis-master-only \ + daemon-tests-redis-sentinel \ + daemon-tests-redis-sentinel-start \ # TESTS := test-bitstr test-str aes-crypt aead-aes-crypt test-const_str_hash.strhash @@ -238,6 +240,8 @@ daemon-tests: \ daemon-tests-video \ daemon-tests-inject \ daemon-tests-redis-master-only \ + daemon-tests-redis-sentinel \ + daemon-tests-redis-sentinel-start \ # daemon-test-deps: tests-preload.so @@ -360,6 +364,12 @@ daemon-tests-redis-reconnect: daemon-test-deps daemon-tests-redis-master-only: daemon-test-deps ./auto-test-helper "$@" perl -I../perl auto-daemon-tests-redis-master-only.pl +daemon-tests-redis-sentinel: daemon-test-deps + ./auto-test-helper "$@" perl -I../perl auto-daemon-tests-redis-sentinel.pl + +daemon-tests-redis-sentinel-start: daemon-test-deps + ./auto-test-helper "$@" perl -I../perl auto-daemon-tests-redis-sentinel-start.pl + daemon-tests-audio-player: daemon-test-deps ./auto-test-helper "$@" perl -I../perl auto-daemon-tests-audio-player.pl @@ -562,6 +572,7 @@ test-stats: test-stats.o \ ../daemon/ng_client.o \ ../daemon/recording.o \ ../daemon/redis.o \ + ../daemon/redis_sentinel.o \ ../daemon/rtcp.o \ ../daemon/rtp.o \ ../daemon/sdp.strhash.o \ @@ -626,6 +637,7 @@ test-transcode: test-transcode.o \ ../daemon/ng_client.o \ ../daemon/recording.o \ ../daemon/redis.o \ + ../daemon/redis_sentinel.o \ ../daemon/rtcp.o \ ../daemon/rtp.o \ ../daemon/sdp.strhash.o \ diff --git a/t/auto-daemon-tests-redis-master-only.pl b/t/auto-daemon-tests-redis-master-only.pl index 4e9603d93..7752a0b4d 100755 --- a/t/auto-daemon-tests-redis-master-only.pl +++ b/t/auto-daemon-tests-redis-master-only.pl @@ -136,8 +136,10 @@ $NGCP::Rtpengine::req_cb = sub { offer('write after MASTERDOWN', { }, $sdp); -# the read connection was never touched +# the read connection was never touched, and without Sentinel options +# nothing of Sentinel is set up ok(!readable($read_fd, 0), 'nothing on the read connection'); +unlike(rtpe_log(), qr/Sentinel/, 'no Sentinel'); undef($NGCP::Rtpengine::req_cb); diff --git a/t/auto-daemon-tests-redis-sentinel-start.pl b/t/auto-daemon-tests-redis-sentinel-start.pl new file mode 100755 index 000000000..0d574e8fa --- /dev/null +++ b/t/auto-daemon-tests-redis-sentinel-start.pl @@ -0,0 +1,197 @@ +#!/usr/bin/perl + +# Redis Sentinel, started with no Sentinel reachable (--no-redis-required), +# and polled rarely (--redis-sentinel-interval=60000), so that every query +# seen here was triggered by rtpengine itself: by an operation with no master +# known, by a NOREPLICAS refusal, by a READONLY refusal, or by a reconnect +# that finds a replica. The Sentinel password comes from the environment. + +use strict; +use warnings; +use NGCP::Rtpengine::Test; +use NGCP::Rtpengine::AutoTest; +use NGCP::Rtpengine::FakeRedis; +use Test::More; +use Time::HiRes qw(sleep); + + +my %l = map { $_ => redis_listen("203.0.113.$_") } (42, 44); + +sentinels_start('spw2', '203.0.113.43' => 'down'); + +# a missing reply fails the test with the daemon's last log lines +local $SIG{ALRM} = sub { die "timeout\n" }; +# a role probe can give up before it is answered +local $SIG{PIPE} = 'IGNORE'; + +my $sdp = < '203.0.113.42 6379 1'); +new_queries(); +new_call; +undef($NGCP::Rtpengine::req_cb); +offer('write that wakes the Sentinel thread', { }, $sdp); +wait_query('woken by a write with no master known'); +like(sentinel_log(), qr/^203\.0\.113\.43:26379 AUTH spw2$/m, 'Sentinel password from the environment'); +wait_probe($l{42}, 'first master'); +wait_log(qr/master of 'mymaster' is now 203\.0\.113\.42:6379 \(config epoch 1\)/, 'master published'); + +my $fd; +new_call; +$NGCP::Rtpengine::req_cb = sub { + $fd = redis_conn($l{42}, 'first connection'); + redis_expect_set($fd, cid(), "+OK\r\n", 'first write', no_ping => 1); +}; +offer('first write', { }, $sdp); + + +# NOREPLICAS: the connection is kept, and the Sentinels are asked at once; +# they name .44, where the next write goes + +sentinels_set('203.0.113.43' => '203.0.113.44 6379 2'); +new_queries(); +refused_write($fd, 'NOREPLICAS Not enough good replicas to write.', 'NOREPLICAS write'); +wait_query('woken by NOREPLICAS'); +wait_probe($l{44}, 'after NOREPLICAS'); +wait_log(qr/master of 'mymaster' is now 203\.0\.113\.44:6379 \(config epoch 2\)/, '.44 published'); + +new_call; +$NGCP::Rtpengine::req_cb = sub { + redis_closed($fd, 'connection to .42 closed'); + $fd = redis_conn($l{44}, 'after NOREPLICAS'); + redis_expect_set($fd, cid(), "+OK\r\n", 'write on .44', no_ping => 1); +}; +offer('write on .44', { }, $sdp); +unlike(rtpe_log(), qr/RE-Establishing connection for Redis server 203\.0\.113\.44/, + 'the move is logged once'); + +# NOREPLICAS again, within the poll interval: the connection is kept, and +# the Sentinels are not asked again (a master also says this for as long as +# it lacks replicas: at most one query per interval) + +new_queries(); +refused_write($fd, 'NOREPLICAS Not enough good replicas to write.', 'NOREPLICAS again'); +sleep(0.5); +is(new_queries(), 0, 'NOREPLICAS again: no new query within the interval'); + + +# READONLY: the master role is implied in Sentinel mode, so the connection +# is dropped at once, and the Sentinels are asked + +new_queries(); +new_call; +$NGCP::Rtpengine::req_cb = sub { + redis_expect_set($fd, cid(), "-READONLY You can't write against a read only replica.\r\n", 'READONLY'); + redis_closed($fd, 'READONLY: connection dropped'); +}; +offer('READONLY write', { }, $sdp); +wait_query('woken by READONLY'); + +# The Sentinel still names .44 (same address and epoch: no probe). The +# reconnect finds a replica, gives up after one INFO, asks the Sentinels +# again, and makes no new attempt within the next second. + +sleep(0.3); +new_queries(); +new_call; +$NGCP::Rtpengine::req_cb = sub { + $fd = redis_conn($l{44}, 'replica', role => 'slave', refused => 1); + redis_closed($fd, 'replica: connection closed after one INFO'); +}; +offer('write while .44 is a replica', { }, $sdp); +wait_query('woken by the role mismatch'); +ok(!readable($l{44}, 0.3), 'no probe for an unchanged answer, no reconnect within the pause'); + +new_call; +$NGCP::Rtpengine::req_cb = sub { + ok(!readable($l{44}, 0.2), 'no reconnect within the pacing interval'); +}; +offer('write within the pacing interval', { }, $sdp); + +sleep(1); +new_call; +$NGCP::Rtpengine::req_cb = sub { + $fd = redis_conn($l{44}, 'master again'); + redis_expect_set($fd, cid(), "+OK\r\n", 'master again', no_ping => 1); +}; +offer('write after the pacing interval', { }, $sdp); + + +undef($NGCP::Rtpengine::req_cb); + +done_testing(); diff --git a/t/auto-daemon-tests-redis-sentinel.pl b/t/auto-daemon-tests-redis-sentinel.pl new file mode 100755 index 000000000..534776b79 --- /dev/null +++ b/t/auto-daemon-tests-redis-sentinel.pl @@ -0,0 +1,213 @@ +#!/usr/bin/perl + +# Redis Sentinel: --redis=PW@sentinel/DB takes the master's address from the +# Sentinels. Two fake Sentinels (203.0.113.43 and .45) name the master; two +# fake data nodes (203.0.113.42 and .44) are scripted here. + +use strict; +use warnings; +use NGCP::Rtpengine::Test; +use NGCP::Rtpengine::AutoTest; +use NGCP::Rtpengine::FakeRedis; +use Test::More; +use Time::HiRes qw(sleep time); + + +my %l = map { $_ => redis_listen("203.0.113.$_") } (42, 44); + +sentinels_start('spw', '203.0.113.43' => '203.0.113.42 6379 1', '203.0.113.45' => '203.0.113.42 6379 1'); + +# a missing reply fails the test with the daemon's last log lines +local $SIG{ALRM} = sub { die "timeout\n" }; +# a role probe can give up before it is answered +local $SIG{PIPE} = 'IGNORE'; + +my $sdp = <= $q + $n; + sleep(0.05); + } + return 0; +} +sub write_on { + my ($fd, $n) = @_; + new_call; + $NGCP::Rtpengine::req_cb = sub { redis_expect_set($$fd, cid(), "+OK\r\n", $n) }; + offer($n, { }, $sdp); +} +# the next write moves to the new master, with nothing sent to the old one +sub write_moves { + my ($fd, $to, $n) = @_; + new_call; + $NGCP::Rtpengine::req_cb = sub { + redis_closed($$fd, "$n: old connection closed without a write"); + $$fd = redis_conn($l{$to}, "$n: on .$to"); + redis_expect_set($$fd, cid(), "+OK\r\n", $n, no_ping => 1); + }; + offer($n, { }, $sdp); +} + + +my $main_fd; + +$NGCP::Rtpengine::AutoTest::launch_cb = sub { + # discovery (Sentinels, then a role probe on .42), then the connection + $main_fd = redis_conn($l{42}, 'startup'); + redis_expect($main_fd, ['PING'], "+PONG\r\n", 'restore PING'); + redis_expect($main_fd, ['KEYS', '*'], "*0\r\n", 'restore KEYS'); +}; + +autotest_start(qw(--config-file=none -t -1 -i 203.0.113.1 + -n 2223 -f -L 7 -E + --redis=auth@sentinel/2 + --redis-sentinel-master=mymaster --redis-sentinel-interval=500 + --redis-cmd-timeout=1000), + '--redis-sentinel=spw@203.0.113.43:26379,203.0.113.45:26379') + or die; + + +# startup: both Sentinels asked, with their password; the master they name +# checked with ROLE and the data password before it was used + +for my $s ('203.0.113.43', '203.0.113.45') { + like(sentinel_log(), qr/^\Q$s\E:26379 AUTH spw\n\Q$s\E:26379 SENTINEL MASTER mymaster\n\Q$s\E:26379 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster$/m, + "Sentinel $s asked at startup"); +} +is_deeply(\@probes, [['203.0.113.42', 'auth', 'master']], 'one role probe at startup, with the data password'); +is(published(qr/203\.0\.113\.42:6379 \(config epoch 1\)/), 1, 'master published'); +like(rtpe_log(), qr/Using Redis Sentinel for the master of 'mymaster'/, 'Sentinel mode'); +write_on(\$main_fd, 'first write on .42'); + + +# Failover to .44. The Sentinel running it (.45) has raised the epoch and +# names the promoted replica in GET-MASTER-ADDR-BY-NAME, while the "ip" of +# its SENTINEL MASTER is still the old master; the other Sentinel (.43) has +# not heard of it yet. The new master is taken at once, once. + +sentinels_set('203.0.113.43' => '203.0.113.42 6379 1', '203.0.113.45' => '203.0.113.44 6379 2 203.0.113.42'); +wait_probe($l{44}, 'failover, leader alone at the new epoch'); +wait_log(qr/master of 'mymaster' is now 203\.0\.113\.44:6379 \(config epoch 2\)/, 'new master published'); +sentinels_set('203.0.113.43' => '203.0.113.44 6379 2', '203.0.113.45' => '203.0.113.44 6379 2'); +ok(queried('203.0.113.43', 2), 'Sentinels asked again'); +ok(!readable($l{44}, 0), 'failover: no second probe'); +is(published(qr/203\.0\.113\.44/), 1, 'failover: published once'); + +write_moves(\$main_fd, 44, 'first write after the failover'); +like(rtpe_log(), qr/following the master of 'mymaster' from 203\.0\.113\.42:6379 to 203\.0\.113\.44:6379/, + 'move logged'); +write_on(\$main_fd, 'next write on .44'); + + +# a Sentinel that missed the failover names .42 with the old epoch: not used +# while the other one names the new master + +sentinels_set('203.0.113.43' => '203.0.113.42 6379 1', '203.0.113.45' => '203.0.113.44 6379 2'); +ok(queried('203.0.113.43', 2), 'stale Sentinel asked'); +ok(!readable($l{42}, 0), 'stale Sentinel: no probe and no connection to .42'); +unlike(rtpe_log(), qr/epochs went back/, 'stale Sentinel, the other one up to date: no error'); + + +# Every Sentinel that answers names an older epoch: never followed, an error +# is logged once, and the master in use is kept. First the stale Sentinel +# alone, then (after a normal state) a Sentinel set redeployed without its +# state, whose epochs start again. + +my $epoch_back = qr/every Sentinel that answers names 203\.0\.113\.42:6379 as master of 'mymaster' at config epoch 1, older than 2 \(203\.0\.113\.44:6379, in use\): not followed\. If the Sentinels were redeployed without their state, their epochs went back, and rtpengine must be restarted/; +sentinels_set('203.0.113.43' => '203.0.113.42 6379 1', '203.0.113.45' => 'down'); +wait_log($epoch_back, 'older epoch from every Sentinel that answers: error'); +ok(queried('203.0.113.43', 2), 'asked again'); +is(scalar(() = rtpe_log() =~ /$epoch_back/g), 1, 'logged once'); + +sentinels_set('203.0.113.43' => '203.0.113.44 6379 2', '203.0.113.45' => '203.0.113.44 6379 2'); +ok(queried('203.0.113.43', 2), 'back to the epoch in use'); +sentinels_set('203.0.113.43' => '203.0.113.42 6379 1', '203.0.113.45' => '203.0.113.42 6379 1'); +ok(queried('203.0.113.43', 2), 'redeployed Sentinels asked'); +is(scalar(() = rtpe_log() =~ /$epoch_back/g), 2, 'logged again after the normal state'); +ok(!readable($l{42}, 0), 'older epoch: no probe and no connection to .42'); +is(published(qr/203\.0\.113\.42:6379 \(config epoch 1\)/), 1, 'older epoch: .42 not published again'); +write_on(\$main_fd, 'write on the master in use'); + + +# a Sentinel names a node at a newer epoch, but it is a replica: not used, +# and asked again soon (before the next poll); then the Sentinels agree on +# .44 at that epoch (same address: no probe) + +$role{'203.0.113.42'} = 'slave'; +sentinels_set('203.0.113.43' => '203.0.113.42 6379 3', '203.0.113.45' => 'down'); +wait_probe($l{42}, 'replica candidate'); +my $t = time(); +wait_probe($l{42}, 'replica candidate again'); +ok(time() - $t < 0.45, 'replica candidate asked again before the next poll'); +sentinels_set('203.0.113.43' => '203.0.113.44 6379 3', '203.0.113.45' => '203.0.113.44 6379 3'); +serve_probes(1, $l{42}); +ok(!readable($l{44}, 0), 'same address at a newer epoch: no probe'); +is(scalar(() = rtpe_log() =~ /names 203\.0\.113\.42:6379 as master of 'mymaster' \(config epoch 3\), but it is a slave: not used/g), + 1, 'replica candidate refused, logged once'); +is(published(qr/203\.0\.113\.42:6379 \(config epoch 3\)/), 0, 'replica never published'); +write_on(\$main_fd, 'write after a replica candidate'); + + +# no Sentinel answers: the last master is kept, logged once + +sentinels_set('203.0.113.43' => 'down', '203.0.113.45' => 'down'); +wait_log(qr/no Sentinel answered for 'mymaster'; keeping 203\.0\.113\.44:6379/, 'no Sentinel answering: logged'); +sleep(1.2); +is(scalar(() = rtpe_log() =~ /no Sentinel answered/g), 1, 'logged once'); +write_on(\$main_fd, 'write without Sentinels'); + +sentinels_set('203.0.113.43' => '203.0.113.44 6379 3', '203.0.113.45' => '203.0.113.44 6379 3'); +wait_log(qr/Redis Sentinel 203\.0\.113\.43:26379 answers again/, 'Sentinel back'); + + +# a second failover, back to .42, seen first from its leader (.45), then +# from both: one move, at once, no flip back + +$role{'203.0.113.42'} = 'master'; +sentinels_set('203.0.113.43' => '203.0.113.44 6379 3', '203.0.113.45' => '203.0.113.42 6379 4 203.0.113.44'); +wait_probe($l{42}, 'second failover'); +wait_log(qr/master of 'mymaster' is now 203\.0\.113\.42:6379 \(config epoch 4\)/, 'new master at once'); +sentinels_set('203.0.113.43' => '203.0.113.42 6379 4', '203.0.113.45' => '203.0.113.42 6379 4'); +ok(queried('203.0.113.43', 2), 'Sentinels asked again'); +is(published(qr/203\.0\.113\.44:6379 \(config epoch 4\)/), 0, 'no flip back to the old master'); +ok(!readable($l{44}, 0), 'no probe of the old master'); +write_moves(\$main_fd, 42, 'first write on the new master'); + + +# throughout: the Sentinels only got their own password and their two +# queries, and every role probe used the data password + +is(scalar(grep { !/^(applied \d+|\S+ (AUTH spw|SENTINEL MASTER mymaster|SENTINEL GET-MASTER-ADDR-BY-NAME mymaster))$/ } split(/\n/, sentinel_log())), 0, + 'Sentinels get nothing else'); +is(scalar(grep { $_->[1] ne 'auth' } @probes), 0, 'role probes only use the data password'); + + +undef($NGCP::Rtpengine::req_cb); + +done_testing();