mirror of https://github.com/sipwise/rtpengine.git
Closes #2212 Behind Redis Sentinel, rtpengine stays on the address it was given: after a failover its writes go to the old master, which refuses them until it is demoted, then serves them as a replica. New options --redis-sentinel, --redis-sentinel-master and --redis-sentinel-interval; the keyword "sentinel" replaces IP:PORT in --redis, --redis-write or --redis-subscribe. A "redis sentinel" thread asks every Sentinel for the master, takes the highest config epoch (never an older one than in use), checks the node with ROLE and publishes it with a generation number. It also runs at once when a connection meets a replica. redis_check_conn() compares the generation and reconnects on a change; these connections require the master role. Without "sentinel", none of this runs. Tested by auto-daemon-tests-redis-sentinel.pl and -sentinel-start.pl. Change-Id: I7b31b50968fd6a255b9933fbf2c95fc95a8b8408pull/2237/head
parent
f3003b4b9d
commit
2c717d54e7
@ -0,0 +1,401 @@
|
||||
#include "redis_sentinel.h"
|
||||
|
||||
#include <glib.h>
|
||||
#include <inttypes.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <unistd.h>
|
||||
#include <hiredis/hiredis.h>
|
||||
|
||||
#include "helpers.h"
|
||||
#include "log_d.h"
|
||||
#include "main.h"
|
||||
#include "redis.h"
|
||||
|
||||
struct sentinel_node {
|
||||
char *host; /* as configured, for --redis-resolve-on-reconnect */
|
||||
endpoint_t ep;
|
||||
bool failing; /* last query failed (logged once) */
|
||||
};
|
||||
|
||||
struct redis_sentinel {
|
||||
struct sentinel_node *nodes;
|
||||
unsigned int num_nodes;
|
||||
char *auth;
|
||||
char *master_name;
|
||||
const char *probe_auth; /* data node password, for ROLE */
|
||||
|
||||
/* published master: written by the thread, read by the connections */
|
||||
mutex_t lock;
|
||||
cond_t cond;
|
||||
bool wake;
|
||||
endpoint_t master;
|
||||
uint64_t epoch;
|
||||
atomic64 gen; /* 0: no master known yet */
|
||||
atomic64 limited_wake_us;
|
||||
|
||||
/* thread only */
|
||||
bool none_answered; /* logged once */
|
||||
bool epoch_back; /* logged once */
|
||||
struct thread_waker waker;
|
||||
};
|
||||
|
||||
struct redis_sentinel *rtpe_redis_sentinel;
|
||||
|
||||
|
||||
/* [PW@]HOST:PORT[,HOST:PORT...] */
|
||||
struct redis_sentinel *redis_sentinel_new(const char *list, const char *master_name,
|
||||
const char *auth_env)
|
||||
{
|
||||
if (!list || !*list || !master_name || !*master_name)
|
||||
return NULL;
|
||||
|
||||
struct redis_sentinel *s = g_new0(struct redis_sentinel, 1);
|
||||
mutex_init(&s->lock);
|
||||
cond_init(&s->cond);
|
||||
g_autofree char *dup = g_strdup(list);
|
||||
g_auto(GStrv) items = NULL;
|
||||
char *hosts = dup;
|
||||
|
||||
char *at = strrchr(hosts, '@');
|
||||
if (at) {
|
||||
*at = '\0';
|
||||
if (!*hosts)
|
||||
goto err; // "@host:port": an empty password is a mistake
|
||||
s->auth = g_strdup(hosts);
|
||||
hosts = at + 1;
|
||||
}
|
||||
else {
|
||||
const char *env = getenv(auth_env);
|
||||
if (env && *env)
|
||||
s->auth = g_strdup(env);
|
||||
}
|
||||
|
||||
items = g_strsplit(hosts, ",", -1);
|
||||
s->num_nodes = g_strv_length(items);
|
||||
if (!s->num_nodes)
|
||||
goto err;
|
||||
s->nodes = g_new0(struct sentinel_node, s->num_nodes);
|
||||
|
||||
for (unsigned int i = 0; i < s->num_nodes; i++) {
|
||||
struct sentinel_node *n = &s->nodes[i];
|
||||
char *item = g_strstrip(items[i]);
|
||||
char *colon = strrchr(item, ':');
|
||||
if (!colon)
|
||||
goto err;
|
||||
if (!endpoint_parse_any_getaddrinfo_full(&n->ep, item) || !n->ep.port)
|
||||
goto err;
|
||||
n->host = g_strndup(item, colon - item);
|
||||
}
|
||||
|
||||
s->master_name = g_strdup(master_name);
|
||||
return s;
|
||||
|
||||
err:
|
||||
redis_sentinel_free(s);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
void redis_sentinel_free(struct redis_sentinel *s) {
|
||||
if (!s)
|
||||
return;
|
||||
for (unsigned int i = 0; s->nodes && i < s->num_nodes; i++)
|
||||
g_free(s->nodes[i].host);
|
||||
g_free(s->nodes);
|
||||
g_free(s->auth);
|
||||
g_free(s->master_name);
|
||||
mutex_destroy(&s->lock);
|
||||
g_free(s);
|
||||
}
|
||||
|
||||
void redis_sentinel_set_probe_auth(struct redis_sentinel *s, const char *auth) {
|
||||
s->probe_auth = auth;
|
||||
}
|
||||
|
||||
const char *redis_sentinel_master_name(const struct redis_sentinel *s) {
|
||||
return s->master_name;
|
||||
}
|
||||
|
||||
|
||||
uint64_t redis_sentinel_gen(struct redis_sentinel *s) {
|
||||
return atomic64_get(&s->gen);
|
||||
}
|
||||
|
||||
uint64_t redis_sentinel_get(struct redis_sentinel *s, endpoint_t *ep) {
|
||||
LOCK(&s->lock);
|
||||
*ep = s->master;
|
||||
return atomic64_get_na(&s->gen);
|
||||
}
|
||||
|
||||
/* "limited": at most once per poll interval (NOREPLICAS lasts as long as the
|
||||
* master lacks replicas) */
|
||||
void redis_sentinel_wake(struct redis_sentinel *s, bool limited) {
|
||||
if (limited) {
|
||||
int64_t now = now_us();
|
||||
int64_t last = atomic64_get_na(&s->limited_wake_us);
|
||||
if (last && now >= last && now - last < rtpe_config.redis_sentinel_interval * 1000LL)
|
||||
return;
|
||||
if (!atomic64_set_if(&s->limited_wake_us, now, last))
|
||||
return;
|
||||
}
|
||||
LOCK(&s->lock);
|
||||
s->wake = true;
|
||||
cond_signal(&s->cond);
|
||||
}
|
||||
|
||||
|
||||
static redisContext *sentinel_connect(const endpoint_t *ep, const char *auth) {
|
||||
char host[64];
|
||||
sockaddr_print(&ep->address, host, sizeof(host));
|
||||
int64_t connect_timeout = atomic_get_na(&rtpe_config.redis_connect_timeout) * 1000LL;
|
||||
// never without a command timeout ("rediscmdtimeout 0"): a server that
|
||||
// accepts but never answers would stop the thread for good
|
||||
int64_t cmd_timeout = atomic_get_na(&rtpe_config.redis_cmd_timeout) * 1000LL ? : 1000000;
|
||||
|
||||
redisContext *ctx = redisConnectWithTimeout(host, ep->port, timeval_from_us(connect_timeout));
|
||||
if (!ctx)
|
||||
return NULL;
|
||||
const char *err = NULL;
|
||||
redisReply *rp = NULL;
|
||||
if (ctx->err || redisSetTimeout(ctx, timeval_from_us(cmd_timeout)))
|
||||
err = ctx->errstr;
|
||||
else if (auth && !(rp = redisCommand(ctx, "AUTH %s", auth)))
|
||||
err = ctx->errstr;
|
||||
else if (rp && rp->type == REDIS_REPLY_ERROR)
|
||||
err = "authentication failed";
|
||||
if (rp)
|
||||
freeReplyObject(rp);
|
||||
if (!err)
|
||||
return ctx;
|
||||
rlog(LOG_WARN | LOG_FLAG_LIMIT, "Redis Sentinel: cannot connect to %s: %s",
|
||||
endpoint_print_buf(ep), err);
|
||||
redisFree(ctx);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static const char *reply_field(const redisReply *rp, const char *name) {
|
||||
for (size_t i = 0; i + 1 < rp->elements; i += 2) {
|
||||
const redisReply *k = rp->element[i], *v = rp->element[i + 1];
|
||||
if (k->type == REDIS_REPLY_STRING && v->type == REDIS_REPLY_STRING && !strcmp(k->str, name))
|
||||
return v->str;
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static const char *sentinel_parse(const redisReply *rp, const redisReply *ra,
|
||||
endpoint_t *master, uint64_t *epoch)
|
||||
{
|
||||
if (rp->type == REDIS_REPLY_ERROR)
|
||||
return rp->str;
|
||||
if (ra->type == REDIS_REPLY_ERROR)
|
||||
return ra->str;
|
||||
const char *epoch_s = rp->type == REDIS_REPLY_ARRAY ? reply_field(rp, "config-epoch") : NULL;
|
||||
if (!epoch_s || ra->type != REDIS_REPLY_ARRAY || ra->elements != 2
|
||||
|| ra->element[0]->type != REDIS_REPLY_STRING
|
||||
|| ra->element[1]->type != REDIS_REPLY_STRING)
|
||||
return "unexpected reply";
|
||||
char *end;
|
||||
*epoch = strtoull(epoch_s, &end, 10);
|
||||
if (end == epoch_s || *end)
|
||||
return "unexpected reply";
|
||||
unsigned long port = strtoul(ra->element[1]->str, &end, 10);
|
||||
if (end == ra->element[1]->str || *end || !port || port > 0xffff)
|
||||
return "unexpected reply";
|
||||
// a host name (announce-hostnames) is resolved here, in the thread
|
||||
if (!sockaddr_getaddrinfo(&master->address, ra->element[0]->str))
|
||||
return "cannot resolve the master's address";
|
||||
master->port = port;
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* Epoch first, then address, so that a failover between the two replies can
|
||||
* only pair an older epoch with the newer address. (The "ip" of SENTINEL
|
||||
* MASTER switches seconds after the epoch.) */
|
||||
static bool sentinel_query(struct redis_sentinel *s, struct sentinel_node *n,
|
||||
endpoint_t *master, uint64_t *epoch)
|
||||
{
|
||||
if (rtpe_config.redis_resolve_on_reconnect) {
|
||||
sockaddr_t a;
|
||||
if (sockaddr_getaddrinfo(&a, n->host))
|
||||
n->ep.address = a;
|
||||
}
|
||||
|
||||
redisContext *ctx = sentinel_connect(&n->ep, s->auth);
|
||||
if (!ctx) {
|
||||
n->failing = true;
|
||||
return false;
|
||||
}
|
||||
|
||||
redisReply *rp = NULL, *ra = NULL;
|
||||
const char *err;
|
||||
redisAppendCommand(ctx, "SENTINEL MASTER %s", s->master_name);
|
||||
redisAppendCommand(ctx, "SENTINEL GET-MASTER-ADDR-BY-NAME %s", s->master_name);
|
||||
if (redisGetReply(ctx, (void **) &rp) != REDIS_OK || redisGetReply(ctx, (void **) &ra) != REDIS_OK)
|
||||
err = ctx->errstr;
|
||||
else
|
||||
err = sentinel_parse(rp, ra, master, epoch);
|
||||
|
||||
if (err && !n->failing)
|
||||
rlog(LOG_WARN, "Redis Sentinel %s: cannot get the master of '%s': %s",
|
||||
endpoint_print_buf(&n->ep), s->master_name, err);
|
||||
else if (!err && n->failing)
|
||||
rlog(LOG_NOTICE, "Redis Sentinel %s answers again", endpoint_print_buf(&n->ep));
|
||||
n->failing = err != NULL;
|
||||
|
||||
if (rp)
|
||||
freeReplyObject(rp);
|
||||
if (ra)
|
||||
freeReplyObject(ra);
|
||||
redisFree(ctx);
|
||||
return !err;
|
||||
}
|
||||
|
||||
static bool sentinel_is_master(struct redis_sentinel *s, const endpoint_t *ep, uint64_t epoch) {
|
||||
redisContext *ctx = sentinel_connect(ep, s->probe_auth);
|
||||
if (!ctx)
|
||||
return false;
|
||||
|
||||
bool ret = false;
|
||||
redisReply *rp = redisCommand(ctx, "ROLE");
|
||||
if (rp && rp->type == REDIS_REPLY_ARRAY && rp->elements && rp->element[0]->type == REDIS_REPLY_STRING) {
|
||||
ret = !strcmp(rp->element[0]->str, "master");
|
||||
if (!ret)
|
||||
rlog(LOG_WARN | LOG_FLAG_LIMIT, "Redis Sentinel names %s as master of '%s' "
|
||||
"(config epoch %" PRIu64 "), but it is a %s: not used",
|
||||
endpoint_print_buf(ep), s->master_name, epoch, rp->element[0]->str);
|
||||
}
|
||||
else
|
||||
rlog(LOG_WARN | LOG_FLAG_LIMIT, "Redis Sentinel: cannot check the role of %s: %s",
|
||||
endpoint_print_buf(ep), !rp ? ctx->errstr
|
||||
: rp->type == REDIS_REPLY_ERROR ? rp->str : "unexpected reply");
|
||||
|
||||
if (rp)
|
||||
freeReplyObject(rp);
|
||||
redisFree(ctx);
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void sentinel_publish(struct redis_sentinel *s, const endpoint_t *m, uint64_t epoch) {
|
||||
{
|
||||
LOCK(&s->lock);
|
||||
s->master = *m;
|
||||
s->epoch = epoch;
|
||||
atomic64_inc(&s->gen);
|
||||
}
|
||||
rlog(LOG_NOTICE, "Redis Sentinel: master of '%s' is now %s (config epoch %" PRIu64 ")",
|
||||
s->master_name, endpoint_print_buf(m), epoch);
|
||||
}
|
||||
|
||||
/* true if the candidate is not a master: ask again soon */
|
||||
static bool sentinel_round(struct redis_sentinel *s) {
|
||||
endpoint_t best = {0}, pub;
|
||||
uint64_t best_epoch = 0, pub_epoch, pub_gen;
|
||||
bool found = false;
|
||||
|
||||
{
|
||||
LOCK(&s->lock);
|
||||
pub = s->master;
|
||||
pub_epoch = s->epoch;
|
||||
pub_gen = atomic64_get_na(&s->gen);
|
||||
}
|
||||
|
||||
for (unsigned int i = 0; i < s->num_nodes; i++) {
|
||||
endpoint_t m = {0};
|
||||
uint64_t e = 0;
|
||||
if (!sentinel_query(s, &s->nodes[i], &m, &e))
|
||||
continue;
|
||||
// at the same epoch, keep the master in use (no flapping while
|
||||
// the Sentinels disagree)
|
||||
if (!found || e > best_epoch || (e == best_epoch && pub_gen && endpoint_eq(&m, &pub))) {
|
||||
best = m;
|
||||
best_epoch = e;
|
||||
found = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (!found) {
|
||||
if (!s->none_answered)
|
||||
rlog(LOG_ERR, "Redis Sentinel: no Sentinel answered for '%s'%s%s",
|
||||
s->master_name,
|
||||
pub_gen ? "; keeping " : "",
|
||||
pub_gen ? endpoint_print_buf(&pub) : "");
|
||||
s->none_answered = true;
|
||||
return false;
|
||||
}
|
||||
s->none_answered = false;
|
||||
|
||||
// Never back to an older epoch: a demoted master still says "master"
|
||||
// until Sentinel reconfigures it. The master in use is kept for as long
|
||||
// as it answers as a master (its connection checks the role).
|
||||
if (pub_gen && best_epoch < pub_epoch) {
|
||||
if (!s->epoch_back)
|
||||
rlog(LOG_ERR, "Redis Sentinel: every Sentinel that answers names %s as master of '%s' "
|
||||
"at config epoch %" PRIu64 ", older than %" PRIu64 " (%s, in use): not "
|
||||
"followed. If the Sentinels were redeployed without their state, their "
|
||||
"epochs went back, and rtpengine must be restarted",
|
||||
endpoint_print_buf(&best), s->master_name, best_epoch, pub_epoch,
|
||||
endpoint_print_buf(&pub));
|
||||
s->epoch_back = true;
|
||||
return false;
|
||||
}
|
||||
s->epoch_back = false;
|
||||
|
||||
if (pub_gen && endpoint_eq(&best, &pub)) {
|
||||
LOCK(&s->lock);
|
||||
s->epoch = best_epoch;
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!sentinel_is_master(s, &best, best_epoch))
|
||||
return true;
|
||||
|
||||
sentinel_publish(s, &best, best_epoch);
|
||||
return false;
|
||||
}
|
||||
|
||||
/* at startup: as for a data node, fails at once if nothing answers, and asks
|
||||
* again once a second (11 times) while the candidate is not a master */
|
||||
bool redis_sentinel_discover(struct redis_sentinel *s) {
|
||||
for (int i = 0; i < 11; i++) {
|
||||
if (i)
|
||||
usleep(1000000);
|
||||
bool refused = sentinel_round(s);
|
||||
if (redis_sentinel_gen(s))
|
||||
return true;
|
||||
if (!refused)
|
||||
break;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
void redis_sentinel_loop(struct redis_sentinel *s) {
|
||||
int64_t interval_us = rtpe_config.redis_sentinel_interval * 1000LL;
|
||||
int64_t next = now_us() + interval_us, last = 0;
|
||||
|
||||
// a round is bounded by the timeouts; at shutdown, the waker ends the wait
|
||||
s->waker.lock = &s->lock;
|
||||
s->waker.cond = &s->cond;
|
||||
thread_waker_add(&s->waker);
|
||||
|
||||
while (!rtpe_shutdown) {
|
||||
{
|
||||
LOCK(&s->lock);
|
||||
// a wake-up comes at least 250 ms after the last round
|
||||
while (!rtpe_shutdown) {
|
||||
int64_t until = s->wake ? MIN(next, last + 250000) : next;
|
||||
if (now_us() >= until)
|
||||
break;
|
||||
cond_timedwait(&s->cond, &s->lock, until);
|
||||
}
|
||||
s->wake = false;
|
||||
}
|
||||
if (rtpe_shutdown)
|
||||
break;
|
||||
|
||||
bool refused = sentinel_round(s);
|
||||
last = now_us();
|
||||
next = last + (refused ? MIN(250000, interval_us) : interval_us);
|
||||
}
|
||||
|
||||
thread_waker_del(&s->waker);
|
||||
}
|
||||
@ -0,0 +1,32 @@
|
||||
#ifndef __REDIS_SENTINEL_H__
|
||||
#define __REDIS_SENTINEL_H__
|
||||
|
||||
#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#include "socket.h"
|
||||
|
||||
/*
|
||||
* A thread asks the Redis Sentinels for the master and publishes it with a
|
||||
* generation number; the connections that use it compare the generation (one
|
||||
* atomic read) and reconnect when it changes. All Sentinel I/O is done in
|
||||
* that thread.
|
||||
*/
|
||||
|
||||
struct redis_sentinel;
|
||||
|
||||
extern struct redis_sentinel *rtpe_redis_sentinel;
|
||||
|
||||
struct redis_sentinel *redis_sentinel_new(const char *list, const char *master_name,
|
||||
const char *auth_env);
|
||||
void redis_sentinel_set_probe_auth(struct redis_sentinel *, const char *auth);
|
||||
bool redis_sentinel_discover(struct redis_sentinel *);
|
||||
void redis_sentinel_loop(struct redis_sentinel *);
|
||||
void redis_sentinel_free(struct redis_sentinel *);
|
||||
|
||||
uint64_t redis_sentinel_gen(struct redis_sentinel *);
|
||||
uint64_t redis_sentinel_get(struct redis_sentinel *, endpoint_t *);
|
||||
void redis_sentinel_wake(struct redis_sentinel *, bool limited);
|
||||
const char *redis_sentinel_master_name(const struct redis_sentinel *);
|
||||
|
||||
#endif
|
||||
@ -0,0 +1,197 @@
|
||||
#!/usr/bin/perl
|
||||
|
||||
# Redis Sentinel, started with no Sentinel reachable (--no-redis-required),
|
||||
# and polled rarely (--redis-sentinel-interval=60000), so that every query
|
||||
# seen here was triggered by rtpengine itself: by an operation with no master
|
||||
# known, by a NOREPLICAS refusal, by a READONLY refusal, or by a reconnect
|
||||
# that finds a replica. The Sentinel password comes from the environment.
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
use NGCP::Rtpengine::Test;
|
||||
use NGCP::Rtpengine::AutoTest;
|
||||
use NGCP::Rtpengine::FakeRedis;
|
||||
use Test::More;
|
||||
use Time::HiRes qw(sleep);
|
||||
|
||||
|
||||
my %l = map { $_ => redis_listen("203.0.113.$_") } (42, 44);
|
||||
|
||||
sentinels_start('spw2', '203.0.113.43' => 'down');
|
||||
|
||||
# a missing reply fails the test with the daemon's last log lines
|
||||
local $SIG{ALRM} = sub { die "timeout\n" };
|
||||
# a role probe can give up before it is answered
|
||||
local $SIG{PIPE} = 'IGNORE';
|
||||
|
||||
my $sdp = <<SDP;
|
||||
v=0
|
||||
o=- 1545997027 1 IN IP4 198.51.101.40
|
||||
s=tester
|
||||
t=0 0
|
||||
m=audio 3000 RTP/AVP 0 8
|
||||
c=IN IP4 198.51.100.1
|
||||
----------------------------------
|
||||
v=0
|
||||
o=- 1545997027 1 IN IP4 198.51.101.40
|
||||
s=tester
|
||||
t=0 0
|
||||
m=audio PORT RTP/AVP 0 8
|
||||
c=IN IP4 203.0.113.1
|
||||
a=rtpmap:0 PCMU/8000
|
||||
a=rtpmap:8 PCMA/8000
|
||||
a=sendrecv
|
||||
a=rtcp:PORT
|
||||
SDP
|
||||
|
||||
# the queries since the last call
|
||||
my $queries_seen = 0;
|
||||
sub new_queries {
|
||||
my $n = sentinel_queries('203.0.113.43');
|
||||
my $ret = $n - $queries_seen;
|
||||
$queries_seen = $n;
|
||||
return $ret;
|
||||
}
|
||||
sub wait_query {
|
||||
my $n = shift;
|
||||
for (1 .. 60) {
|
||||
return pass($n) if new_queries();
|
||||
sleep(0.05);
|
||||
}
|
||||
fail($n);
|
||||
}
|
||||
# a write refused with $err
|
||||
sub refused_write {
|
||||
my ($fd, $err, $n) = @_;
|
||||
new_call;
|
||||
$NGCP::Rtpengine::req_cb = sub { redis_expect_set($fd, cid(), "-$err\r\n", $n) };
|
||||
offer($n, { }, $sdp);
|
||||
}
|
||||
|
||||
|
||||
local $ENV{RTPENGINE_REDIS_SENTINEL_AUTH_PW} = 'spw2';
|
||||
|
||||
autotest_start(qw(--config-file=none -t -1 -i 203.0.113.1
|
||||
-n 2223 -f -L 7 -E
|
||||
--redis=auth@sentinel/2
|
||||
--redis-sentinel=203.0.113.43:26379
|
||||
--redis-sentinel-master=mymaster
|
||||
--redis-sentinel-interval=60000
|
||||
--no-redis-required
|
||||
))
|
||||
or die;
|
||||
|
||||
like(rtpe_log(), qr/Starting with no master of 'mymaster' known from Redis Sentinel/,
|
||||
'started without a master');
|
||||
like(rtpe_log(), qr/Redis Sentinel is used without --redis-cmd-timeout/,
|
||||
'no command timeout: warned, not changed');
|
||||
ok(!readable($l{42}, 0), 'no data connection at startup');
|
||||
|
||||
|
||||
# no master known: the update is skipped, nothing is connected
|
||||
|
||||
new_call;
|
||||
$NGCP::Rtpengine::req_cb = sub {
|
||||
ok(!readable($l{42}, 0.3), 'no master known: no connection');
|
||||
};
|
||||
offer('write with no master known', { }, $sdp);
|
||||
like(rtpe_log(), qr/No master of 'mymaster' known from Redis Sentinel yet/, 'skipped write logged');
|
||||
|
||||
|
||||
# the Sentinel comes up: the next skipped update wakes the thread, which
|
||||
# finds the master long before its next poll
|
||||
|
||||
sentinels_set('203.0.113.43' => '203.0.113.42 6379 1');
|
||||
new_queries();
|
||||
new_call;
|
||||
undef($NGCP::Rtpengine::req_cb);
|
||||
offer('write that wakes the Sentinel thread', { }, $sdp);
|
||||
wait_query('woken by a write with no master known');
|
||||
like(sentinel_log(), qr/^203\.0\.113\.43:26379 AUTH spw2$/m, 'Sentinel password from the environment');
|
||||
wait_probe($l{42}, 'first master');
|
||||
wait_log(qr/master of 'mymaster' is now 203\.0\.113\.42:6379 \(config epoch 1\)/, 'master published');
|
||||
|
||||
my $fd;
|
||||
new_call;
|
||||
$NGCP::Rtpengine::req_cb = sub {
|
||||
$fd = redis_conn($l{42}, 'first connection');
|
||||
redis_expect_set($fd, cid(), "+OK\r\n", 'first write', no_ping => 1);
|
||||
};
|
||||
offer('first write', { }, $sdp);
|
||||
|
||||
|
||||
# NOREPLICAS: the connection is kept, and the Sentinels are asked at once;
|
||||
# they name .44, where the next write goes
|
||||
|
||||
sentinels_set('203.0.113.43' => '203.0.113.44 6379 2');
|
||||
new_queries();
|
||||
refused_write($fd, 'NOREPLICAS Not enough good replicas to write.', 'NOREPLICAS write');
|
||||
wait_query('woken by NOREPLICAS');
|
||||
wait_probe($l{44}, 'after NOREPLICAS');
|
||||
wait_log(qr/master of 'mymaster' is now 203\.0\.113\.44:6379 \(config epoch 2\)/, '.44 published');
|
||||
|
||||
new_call;
|
||||
$NGCP::Rtpengine::req_cb = sub {
|
||||
redis_closed($fd, 'connection to .42 closed');
|
||||
$fd = redis_conn($l{44}, 'after NOREPLICAS');
|
||||
redis_expect_set($fd, cid(), "+OK\r\n", 'write on .44', no_ping => 1);
|
||||
};
|
||||
offer('write on .44', { }, $sdp);
|
||||
unlike(rtpe_log(), qr/RE-Establishing connection for Redis server 203\.0\.113\.44/,
|
||||
'the move is logged once');
|
||||
|
||||
# NOREPLICAS again, within the poll interval: the connection is kept, and
|
||||
# the Sentinels are not asked again (a master also says this for as long as
|
||||
# it lacks replicas: at most one query per interval)
|
||||
|
||||
new_queries();
|
||||
refused_write($fd, 'NOREPLICAS Not enough good replicas to write.', 'NOREPLICAS again');
|
||||
sleep(0.5);
|
||||
is(new_queries(), 0, 'NOREPLICAS again: no new query within the interval');
|
||||
|
||||
|
||||
# READONLY: the master role is implied in Sentinel mode, so the connection
|
||||
# is dropped at once, and the Sentinels are asked
|
||||
|
||||
new_queries();
|
||||
new_call;
|
||||
$NGCP::Rtpengine::req_cb = sub {
|
||||
redis_expect_set($fd, cid(), "-READONLY You can't write against a read only replica.\r\n", 'READONLY');
|
||||
redis_closed($fd, 'READONLY: connection dropped');
|
||||
};
|
||||
offer('READONLY write', { }, $sdp);
|
||||
wait_query('woken by READONLY');
|
||||
|
||||
# The Sentinel still names .44 (same address and epoch: no probe). The
|
||||
# reconnect finds a replica, gives up after one INFO, asks the Sentinels
|
||||
# again, and makes no new attempt within the next second.
|
||||
|
||||
sleep(0.3);
|
||||
new_queries();
|
||||
new_call;
|
||||
$NGCP::Rtpengine::req_cb = sub {
|
||||
$fd = redis_conn($l{44}, 'replica', role => 'slave', refused => 1);
|
||||
redis_closed($fd, 'replica: connection closed after one INFO');
|
||||
};
|
||||
offer('write while .44 is a replica', { }, $sdp);
|
||||
wait_query('woken by the role mismatch');
|
||||
ok(!readable($l{44}, 0.3), 'no probe for an unchanged answer, no reconnect within the pause');
|
||||
|
||||
new_call;
|
||||
$NGCP::Rtpengine::req_cb = sub {
|
||||
ok(!readable($l{44}, 0.2), 'no reconnect within the pacing interval');
|
||||
};
|
||||
offer('write within the pacing interval', { }, $sdp);
|
||||
|
||||
sleep(1);
|
||||
new_call;
|
||||
$NGCP::Rtpengine::req_cb = sub {
|
||||
$fd = redis_conn($l{44}, 'master again');
|
||||
redis_expect_set($fd, cid(), "+OK\r\n", 'master again', no_ping => 1);
|
||||
};
|
||||
offer('write after the pacing interval', { }, $sdp);
|
||||
|
||||
|
||||
undef($NGCP::Rtpengine::req_cb);
|
||||
|
||||
done_testing();
|
||||
@ -0,0 +1,213 @@
|
||||
#!/usr/bin/perl
|
||||
|
||||
# Redis Sentinel: --redis=PW@sentinel/DB takes the master's address from the
|
||||
# Sentinels. Two fake Sentinels (203.0.113.43 and .45) name the master; two
|
||||
# fake data nodes (203.0.113.42 and .44) are scripted here.
|
||||
|
||||
use strict;
|
||||
use warnings;
|
||||
use NGCP::Rtpengine::Test;
|
||||
use NGCP::Rtpengine::AutoTest;
|
||||
use NGCP::Rtpengine::FakeRedis;
|
||||
use Test::More;
|
||||
use Time::HiRes qw(sleep time);
|
||||
|
||||
|
||||
my %l = map { $_ => redis_listen("203.0.113.$_") } (42, 44);
|
||||
|
||||
sentinels_start('spw', '203.0.113.43' => '203.0.113.42 6379 1', '203.0.113.45' => '203.0.113.42 6379 1');
|
||||
|
||||
# a missing reply fails the test with the daemon's last log lines
|
||||
local $SIG{ALRM} = sub { die "timeout\n" };
|
||||
# a role probe can give up before it is answered
|
||||
local $SIG{PIPE} = 'IGNORE';
|
||||
|
||||
my $sdp = <<SDP;
|
||||
v=0
|
||||
o=- 1545997027 1 IN IP4 198.51.101.40
|
||||
s=tester
|
||||
t=0 0
|
||||
m=audio 3000 RTP/AVP 0 8
|
||||
c=IN IP4 198.51.100.1
|
||||
----------------------------------
|
||||
v=0
|
||||
o=- 1545997027 1 IN IP4 198.51.101.40
|
||||
s=tester
|
||||
t=0 0
|
||||
m=audio PORT RTP/AVP 0 8
|
||||
c=IN IP4 203.0.113.1
|
||||
a=rtpmap:0 PCMU/8000
|
||||
a=rtpmap:8 PCMA/8000
|
||||
a=sendrecv
|
||||
a=rtcp:PORT
|
||||
SDP
|
||||
|
||||
sub published {
|
||||
my $re = shift;
|
||||
return scalar(() = rtpe_log() =~ /master of 'mymaster' is now $re/g);
|
||||
}
|
||||
# wait until this Sentinel has been asked $n more times
|
||||
sub queried {
|
||||
my ($ip, $n) = @_;
|
||||
my $q = sentinel_queries($ip);
|
||||
for (1 .. 100) {
|
||||
return 1 if sentinel_queries($ip) >= $q + $n;
|
||||
sleep(0.05);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
sub write_on {
|
||||
my ($fd, $n) = @_;
|
||||
new_call;
|
||||
$NGCP::Rtpengine::req_cb = sub { redis_expect_set($$fd, cid(), "+OK\r\n", $n) };
|
||||
offer($n, { }, $sdp);
|
||||
}
|
||||
# the next write moves to the new master, with nothing sent to the old one
|
||||
sub write_moves {
|
||||
my ($fd, $to, $n) = @_;
|
||||
new_call;
|
||||
$NGCP::Rtpengine::req_cb = sub {
|
||||
redis_closed($$fd, "$n: old connection closed without a write");
|
||||
$$fd = redis_conn($l{$to}, "$n: on .$to");
|
||||
redis_expect_set($$fd, cid(), "+OK\r\n", $n, no_ping => 1);
|
||||
};
|
||||
offer($n, { }, $sdp);
|
||||
}
|
||||
|
||||
|
||||
my $main_fd;
|
||||
|
||||
$NGCP::Rtpengine::AutoTest::launch_cb = sub {
|
||||
# discovery (Sentinels, then a role probe on .42), then the connection
|
||||
$main_fd = redis_conn($l{42}, 'startup');
|
||||
redis_expect($main_fd, ['PING'], "+PONG\r\n", 'restore PING');
|
||||
redis_expect($main_fd, ['KEYS', '*'], "*0\r\n", 'restore KEYS');
|
||||
};
|
||||
|
||||
autotest_start(qw(--config-file=none -t -1 -i 203.0.113.1
|
||||
-n 2223 -f -L 7 -E
|
||||
--redis=auth@sentinel/2
|
||||
--redis-sentinel-master=mymaster --redis-sentinel-interval=500
|
||||
--redis-cmd-timeout=1000),
|
||||
'--redis-sentinel=spw@203.0.113.43:26379,203.0.113.45:26379')
|
||||
or die;
|
||||
|
||||
|
||||
# startup: both Sentinels asked, with their password; the master they name
|
||||
# checked with ROLE and the data password before it was used
|
||||
|
||||
for my $s ('203.0.113.43', '203.0.113.45') {
|
||||
like(sentinel_log(), qr/^\Q$s\E:26379 AUTH spw\n\Q$s\E:26379 SENTINEL MASTER mymaster\n\Q$s\E:26379 SENTINEL GET-MASTER-ADDR-BY-NAME mymaster$/m,
|
||||
"Sentinel $s asked at startup");
|
||||
}
|
||||
is_deeply(\@probes, [['203.0.113.42', 'auth', 'master']], 'one role probe at startup, with the data password');
|
||||
is(published(qr/203\.0\.113\.42:6379 \(config epoch 1\)/), 1, 'master published');
|
||||
like(rtpe_log(), qr/Using Redis Sentinel for the master of 'mymaster'/, 'Sentinel mode');
|
||||
write_on(\$main_fd, 'first write on .42');
|
||||
|
||||
|
||||
# Failover to .44. The Sentinel running it (.45) has raised the epoch and
|
||||
# names the promoted replica in GET-MASTER-ADDR-BY-NAME, while the "ip" of
|
||||
# its SENTINEL MASTER is still the old master; the other Sentinel (.43) has
|
||||
# not heard of it yet. The new master is taken at once, once.
|
||||
|
||||
sentinels_set('203.0.113.43' => '203.0.113.42 6379 1', '203.0.113.45' => '203.0.113.44 6379 2 203.0.113.42');
|
||||
wait_probe($l{44}, 'failover, leader alone at the new epoch');
|
||||
wait_log(qr/master of 'mymaster' is now 203\.0\.113\.44:6379 \(config epoch 2\)/, 'new master published');
|
||||
sentinels_set('203.0.113.43' => '203.0.113.44 6379 2', '203.0.113.45' => '203.0.113.44 6379 2');
|
||||
ok(queried('203.0.113.43', 2), 'Sentinels asked again');
|
||||
ok(!readable($l{44}, 0), 'failover: no second probe');
|
||||
is(published(qr/203\.0\.113\.44/), 1, 'failover: published once');
|
||||
|
||||
write_moves(\$main_fd, 44, 'first write after the failover');
|
||||
like(rtpe_log(), qr/following the master of 'mymaster' from 203\.0\.113\.42:6379 to 203\.0\.113\.44:6379/,
|
||||
'move logged');
|
||||
write_on(\$main_fd, 'next write on .44');
|
||||
|
||||
|
||||
# a Sentinel that missed the failover names .42 with the old epoch: not used
|
||||
# while the other one names the new master
|
||||
|
||||
sentinels_set('203.0.113.43' => '203.0.113.42 6379 1', '203.0.113.45' => '203.0.113.44 6379 2');
|
||||
ok(queried('203.0.113.43', 2), 'stale Sentinel asked');
|
||||
ok(!readable($l{42}, 0), 'stale Sentinel: no probe and no connection to .42');
|
||||
unlike(rtpe_log(), qr/epochs went back/, 'stale Sentinel, the other one up to date: no error');
|
||||
|
||||
|
||||
# Every Sentinel that answers names an older epoch: never followed, an error
|
||||
# is logged once, and the master in use is kept. First the stale Sentinel
|
||||
# alone, then (after a normal state) a Sentinel set redeployed without its
|
||||
# state, whose epochs start again.
|
||||
|
||||
my $epoch_back = qr/every Sentinel that answers names 203\.0\.113\.42:6379 as master of 'mymaster' at config epoch 1, older than 2 \(203\.0\.113\.44:6379, in use\): not followed\. If the Sentinels were redeployed without their state, their epochs went back, and rtpengine must be restarted/;
|
||||
sentinels_set('203.0.113.43' => '203.0.113.42 6379 1', '203.0.113.45' => 'down');
|
||||
wait_log($epoch_back, 'older epoch from every Sentinel that answers: error');
|
||||
ok(queried('203.0.113.43', 2), 'asked again');
|
||||
is(scalar(() = rtpe_log() =~ /$epoch_back/g), 1, 'logged once');
|
||||
|
||||
sentinels_set('203.0.113.43' => '203.0.113.44 6379 2', '203.0.113.45' => '203.0.113.44 6379 2');
|
||||
ok(queried('203.0.113.43', 2), 'back to the epoch in use');
|
||||
sentinels_set('203.0.113.43' => '203.0.113.42 6379 1', '203.0.113.45' => '203.0.113.42 6379 1');
|
||||
ok(queried('203.0.113.43', 2), 'redeployed Sentinels asked');
|
||||
is(scalar(() = rtpe_log() =~ /$epoch_back/g), 2, 'logged again after the normal state');
|
||||
ok(!readable($l{42}, 0), 'older epoch: no probe and no connection to .42');
|
||||
is(published(qr/203\.0\.113\.42:6379 \(config epoch 1\)/), 1, 'older epoch: .42 not published again');
|
||||
write_on(\$main_fd, 'write on the master in use');
|
||||
|
||||
|
||||
# a Sentinel names a node at a newer epoch, but it is a replica: not used,
|
||||
# and asked again soon (before the next poll); then the Sentinels agree on
|
||||
# .44 at that epoch (same address: no probe)
|
||||
|
||||
$role{'203.0.113.42'} = 'slave';
|
||||
sentinels_set('203.0.113.43' => '203.0.113.42 6379 3', '203.0.113.45' => 'down');
|
||||
wait_probe($l{42}, 'replica candidate');
|
||||
my $t = time();
|
||||
wait_probe($l{42}, 'replica candidate again');
|
||||
ok(time() - $t < 0.45, 'replica candidate asked again before the next poll');
|
||||
sentinels_set('203.0.113.43' => '203.0.113.44 6379 3', '203.0.113.45' => '203.0.113.44 6379 3');
|
||||
serve_probes(1, $l{42});
|
||||
ok(!readable($l{44}, 0), 'same address at a newer epoch: no probe');
|
||||
is(scalar(() = rtpe_log() =~ /names 203\.0\.113\.42:6379 as master of 'mymaster' \(config epoch 3\), but it is a slave: not used/g),
|
||||
1, 'replica candidate refused, logged once');
|
||||
is(published(qr/203\.0\.113\.42:6379 \(config epoch 3\)/), 0, 'replica never published');
|
||||
write_on(\$main_fd, 'write after a replica candidate');
|
||||
|
||||
|
||||
# no Sentinel answers: the last master is kept, logged once
|
||||
|
||||
sentinels_set('203.0.113.43' => 'down', '203.0.113.45' => 'down');
|
||||
wait_log(qr/no Sentinel answered for 'mymaster'; keeping 203\.0\.113\.44:6379/, 'no Sentinel answering: logged');
|
||||
sleep(1.2);
|
||||
is(scalar(() = rtpe_log() =~ /no Sentinel answered/g), 1, 'logged once');
|
||||
write_on(\$main_fd, 'write without Sentinels');
|
||||
|
||||
sentinels_set('203.0.113.43' => '203.0.113.44 6379 3', '203.0.113.45' => '203.0.113.44 6379 3');
|
||||
wait_log(qr/Redis Sentinel 203\.0\.113\.43:26379 answers again/, 'Sentinel back');
|
||||
|
||||
|
||||
# a second failover, back to .42, seen first from its leader (.45), then
|
||||
# from both: one move, at once, no flip back
|
||||
|
||||
$role{'203.0.113.42'} = 'master';
|
||||
sentinels_set('203.0.113.43' => '203.0.113.44 6379 3', '203.0.113.45' => '203.0.113.42 6379 4 203.0.113.44');
|
||||
wait_probe($l{42}, 'second failover');
|
||||
wait_log(qr/master of 'mymaster' is now 203\.0\.113\.42:6379 \(config epoch 4\)/, 'new master at once');
|
||||
sentinels_set('203.0.113.43' => '203.0.113.42 6379 4', '203.0.113.45' => '203.0.113.42 6379 4');
|
||||
ok(queried('203.0.113.43', 2), 'Sentinels asked again');
|
||||
is(published(qr/203\.0\.113\.44:6379 \(config epoch 4\)/), 0, 'no flip back to the old master');
|
||||
ok(!readable($l{44}, 0), 'no probe of the old master');
|
||||
write_moves(\$main_fd, 42, 'first write on the new master');
|
||||
|
||||
|
||||
# throughout: the Sentinels only got their own password and their two
|
||||
# queries, and every role probe used the data password
|
||||
|
||||
is(scalar(grep { !/^(applied \d+|\S+ (AUTH spw|SENTINEL MASTER mymaster|SENTINEL GET-MASTER-ADDR-BY-NAME mymaster))$/ } split(/\n/, sentinel_log())), 0,
|
||||
'Sentinels get nothing else');
|
||||
is(scalar(grep { $_->[1] ne 'auth' } @probes), 0, 'role probes only use the data password');
|
||||
|
||||
|
||||
undef($NGCP::Rtpengine::req_cb);
|
||||
|
||||
done_testing();
|
||||
Loading…
Reference in new issue