You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
rtpengine/daemon/redis_sentinel.c

405 lines
12 KiB

#include "redis_sentinel.h"
#include <glib.h>
#include <inttypes.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <hiredis/hiredis.h>
#include "helpers.h"
#include "log_d.h"
#include "main.h"
#include "redis.h"
struct sentinel_node {
char *host; /* as configured, for --redis-resolve-on-reconnect */
endpoint_t ep;
bool failing; /* last query failed (logged once) */
};
struct redis_sentinel {
struct sentinel_node *nodes;
unsigned int num_nodes;
char *auth;
char *master_name;
const char *probe_auth; /* data node password, for ROLE */
/* published master: written by the thread, read by the connections */
mutex_t lock;
cond_t cond;
bool wake;
endpoint_t master;
uint64_t epoch;
atomic64 gen; /* 0: no master known yet */
atomic64 limited_wake_us;
/* thread only */
bool none_answered; /* logged once */
bool epoch_back; /* logged once */
struct thread_waker waker;
};
struct redis_sentinel *rtpe_redis_sentinel;
/* [PW@]HOST:PORT[,HOST:PORT...] */
struct redis_sentinel *redis_sentinel_new(const char *list, const char *master_name,
const char *auth_env)
{
if (!list || !*list || !master_name || !*master_name)
return NULL;
struct redis_sentinel *s = g_new0(struct redis_sentinel, 1);
mutex_init(&s->lock);
cond_init(&s->cond);
g_autofree char *dup = g_strdup(list);
g_auto(GStrv) items = NULL;
char *hosts = dup;
char *at = strrchr(hosts, '@');
if (at) {
*at = '\0';
if (!*hosts)
goto err; // "@host:port": an empty password is a mistake
s->auth = g_strdup(hosts);
hosts = at + 1;
}
else {
const char *env = getenv(auth_env);
if (env && *env)
s->auth = g_strdup(env);
}
items = g_strsplit(hosts, ",", -1);
s->num_nodes = g_strv_length(items);
if (!s->num_nodes)
goto err;
s->nodes = g_new0(struct sentinel_node, s->num_nodes);
for (unsigned int i = 0; i < s->num_nodes; i++) {
struct sentinel_node *n = &s->nodes[i];
char *item = g_strstrip(items[i]);
char *colon = strrchr(item, ':');
if (!colon)
goto err;
if (!endpoint_parse_any_getaddrinfo_full(&n->ep, item) || !n->ep.port)
goto err;
n->host = g_strndup(item, colon - item);
}
s->master_name = g_strdup(master_name);
return s;
err:
redis_sentinel_free(s);
return NULL;
}
void redis_sentinel_free(struct redis_sentinel *s) {
if (!s)
return;
for (unsigned int i = 0; s->nodes && i < s->num_nodes; i++)
g_free(s->nodes[i].host);
g_free(s->nodes);
g_free(s->auth);
g_free(s->master_name);
mutex_destroy(&s->lock);
g_free(s);
}
void redis_sentinel_set_probe_auth(struct redis_sentinel *s, const char *auth) {
s->probe_auth = auth;
}
const char *redis_sentinel_master_name(const struct redis_sentinel *s) {
return s->master_name;
}
uint64_t redis_sentinel_gen(struct redis_sentinel *s) {
return atomic64_get(&s->gen);
}
uint64_t redis_sentinel_get(struct redis_sentinel *s, endpoint_t *ep) {
LOCK(&s->lock);
*ep = s->master;
return atomic64_get_na(&s->gen);
}
/* "limited": at most once per poll interval (NOREPLICAS lasts as long as the
* master lacks replicas) */
void redis_sentinel_wake(struct redis_sentinel *s, bool limited) {
if (limited) {
int64_t now = now_us();
int64_t last = atomic64_get_na(&s->limited_wake_us);
if (last && now >= last && now - last < rtpe_config.redis_sentinel_interval * 1000LL)
return;
if (!atomic64_set_if(&s->limited_wake_us, now, last))
return;
}
LOCK(&s->lock);
s->wake = true;
cond_signal(&s->cond);
}
/* "name": what the TLS certificate must match, if not the address */
static redisContext *sentinel_connect(const endpoint_t *ep, const char *name, const char *auth) {
char host[64];
sockaddr_print(&ep->address, host, sizeof(host));
int64_t connect_timeout = atomic_get_na(&rtpe_config.redis_connect_timeout) * 1000LL;
// never without a command timeout ("rediscmdtimeout 0"): a server that
// accepts but never answers would stop the thread for good
int64_t cmd_timeout = atomic_get_na(&rtpe_config.redis_cmd_timeout) * 1000LL ? : 1000000;
redisContext *ctx = redisConnectWithTimeout(host, ep->port, timeval_from_us(connect_timeout));
if (!ctx)
return NULL;
const char *err = NULL;
redisReply *rp = NULL;
if (ctx->err || redisSetTimeout(ctx, timeval_from_us(cmd_timeout))
|| redis_tls_start(ctx, name ? : host, 0, NULL))
err = ctx->errstr;
else if (auth && !(rp = redisCommand(ctx, "AUTH %s", auth)))
err = ctx->errstr;
else if (rp && rp->type == REDIS_REPLY_ERROR)
err = "authentication failed";
if (rp)
freeReplyObject(rp);
if (!err)
return ctx;
rlog(LOG_WARN | LOG_FLAG_LIMIT, "Redis Sentinel: cannot connect to %s: %s",
endpoint_print_buf(ep), err);
redisFree(ctx);
return NULL;
}
static const char *reply_field(const redisReply *rp, const char *name) {
for (size_t i = 0; i + 1 < rp->elements; i += 2) {
const redisReply *k = rp->element[i], *v = rp->element[i + 1];
if (k->type == REDIS_REPLY_STRING && v->type == REDIS_REPLY_STRING && !strcmp(k->str, name))
return v->str;
}
return NULL;
}
static const char *sentinel_parse(const redisReply *rp, const redisReply *ra,
endpoint_t *master, uint64_t *epoch)
{
if (rp->type == REDIS_REPLY_ERROR)
return rp->str;
if (ra->type == REDIS_REPLY_ERROR)
return ra->str;
const char *epoch_s = rp->type == REDIS_REPLY_ARRAY ? reply_field(rp, "config-epoch") : NULL;
if (!epoch_s || ra->type != REDIS_REPLY_ARRAY || ra->elements != 2
|| ra->element[0]->type != REDIS_REPLY_STRING
|| ra->element[1]->type != REDIS_REPLY_STRING)
return "unexpected reply";
char *end;
*epoch = strtoull(epoch_s, &end, 10);
if (end == epoch_s || *end)
return "unexpected reply";
unsigned long port = strtoul(ra->element[1]->str, &end, 10);
if (end == ra->element[1]->str || *end || !port || port > 0xffff)
return "unexpected reply";
// a host name (announce-hostnames) is resolved here, in the thread
if (!sockaddr_getaddrinfo(&master->address, ra->element[0]->str))
return "cannot resolve the master's address";
master->port = port;
return NULL;
}
/* Epoch first, then address, so that a failover between the two replies can
* only pair an older epoch with the newer address. (The "ip" of SENTINEL
* MASTER switches seconds after the epoch.) */
static bool sentinel_query(struct redis_sentinel *s, struct sentinel_node *n,
endpoint_t *master, uint64_t *epoch)
{
if (rtpe_config.redis_resolve_on_reconnect) {
sockaddr_t a;
if (sockaddr_getaddrinfo(&a, n->host))
n->ep.address = a;
}
redisContext *ctx = sentinel_connect(&n->ep, n->host, s->auth);
if (!ctx) {
n->failing = true;
return false;
}
redisReply *rp = NULL, *ra = NULL;
const char *err;
redisAppendCommand(ctx, "SENTINEL MASTER %s", s->master_name);
redisAppendCommand(ctx, "SENTINEL GET-MASTER-ADDR-BY-NAME %s", s->master_name);
if (redisGetReply(ctx, (void **) &rp) != REDIS_OK || redisGetReply(ctx, (void **) &ra) != REDIS_OK)
err = ctx->errstr;
else
err = sentinel_parse(rp, ra, master, epoch);
if (err && !n->failing)
rlog(LOG_WARN, "Redis Sentinel %s: cannot get the master of '%s': %s",
endpoint_print_buf(&n->ep), s->master_name, err);
else if (!err && n->failing)
rlog(LOG_NOTICE, "Redis Sentinel %s answers again", endpoint_print_buf(&n->ep));
n->failing = err != NULL;
if (rp)
freeReplyObject(rp);
if (ra)
freeReplyObject(ra);
redisFree(ctx);
return !err;
}
static bool sentinel_is_master(struct redis_sentinel *s, const endpoint_t *ep, uint64_t epoch) {
// a data node: --redis-tls-server-name applies, as for the data connections
redisContext *ctx = sentinel_connect(ep, rtpe_config.redis_tls_server_name, s->probe_auth);
if (!ctx)
return false;
bool ret = false;
redisReply *rp = redisCommand(ctx, "ROLE");
if (rp && rp->type == REDIS_REPLY_ARRAY && rp->elements && rp->element[0]->type == REDIS_REPLY_STRING) {
ret = !strcmp(rp->element[0]->str, "master");
if (!ret)
rlog(LOG_WARN | LOG_FLAG_LIMIT, "Redis Sentinel names %s as master of '%s' "
"(config epoch %" PRIu64 "), but it is a %s: not used",
endpoint_print_buf(ep), s->master_name, epoch, rp->element[0]->str);
}
else
rlog(LOG_WARN | LOG_FLAG_LIMIT, "Redis Sentinel: cannot check the role of %s: %s",
endpoint_print_buf(ep), !rp ? ctx->errstr
: rp->type == REDIS_REPLY_ERROR ? rp->str : "unexpected reply");
if (rp)
freeReplyObject(rp);
redisFree(ctx);
return ret;
}
static void sentinel_publish(struct redis_sentinel *s, const endpoint_t *m, uint64_t epoch) {
{
LOCK(&s->lock);
s->master = *m;
s->epoch = epoch;
atomic64_inc(&s->gen);
}
rlog(LOG_NOTICE, "Redis Sentinel: master of '%s' is now %s (config epoch %" PRIu64 ")",
s->master_name, endpoint_print_buf(m), epoch);
}
/* true if the candidate is not a master: ask again soon */
static bool sentinel_round(struct redis_sentinel *s) {
endpoint_t best = {0}, pub;
uint64_t best_epoch = 0, pub_epoch, pub_gen;
bool found = false;
{
LOCK(&s->lock);
pub = s->master;
pub_epoch = s->epoch;
pub_gen = atomic64_get_na(&s->gen);
}
for (unsigned int i = 0; i < s->num_nodes; i++) {
endpoint_t m = {0};
uint64_t e = 0;
if (!sentinel_query(s, &s->nodes[i], &m, &e))
continue;
// at the same epoch, keep the master in use (no flapping while
// the Sentinels disagree)
if (!found || e > best_epoch || (e == best_epoch && pub_gen && endpoint_eq(&m, &pub))) {
best = m;
best_epoch = e;
found = true;
}
}
if (!found) {
if (!s->none_answered)
rlog(LOG_ERR, "Redis Sentinel: no Sentinel answered for '%s'%s%s",
s->master_name,
pub_gen ? "; keeping " : "",
pub_gen ? endpoint_print_buf(&pub) : "");
s->none_answered = true;
return false;
}
s->none_answered = false;
// Never back to an older epoch: a demoted master still says "master"
// until Sentinel reconfigures it. The master in use is kept for as long
// as it answers as a master (its connection checks the role).
if (pub_gen && best_epoch < pub_epoch) {
if (!s->epoch_back)
rlog(LOG_ERR, "Redis Sentinel: every Sentinel that answers names %s as master of '%s' "
"at config epoch %" PRIu64 ", older than %" PRIu64 " (%s, in use): not "
"followed. If the Sentinels were redeployed without their state, their "
"epochs went back, and rtpengine must be restarted",
endpoint_print_buf(&best), s->master_name, best_epoch, pub_epoch,
endpoint_print_buf(&pub));
s->epoch_back = true;
return false;
}
s->epoch_back = false;
if (pub_gen && endpoint_eq(&best, &pub)) {
LOCK(&s->lock);
s->epoch = best_epoch;
return false;
}
if (!sentinel_is_master(s, &best, best_epoch))
return true;
sentinel_publish(s, &best, best_epoch);
return false;
}
/* at startup: as for a data node, fails at once if nothing answers, and asks
* again once a second (11 times) while the candidate is not a master */
bool redis_sentinel_discover(struct redis_sentinel *s) {
for (int i = 0; i < 11; i++) {
if (i)
usleep(1000000);
bool refused = sentinel_round(s);
if (redis_sentinel_gen(s))
return true;
if (!refused)
break;
}
return false;
}
void redis_sentinel_loop(struct redis_sentinel *s) {
int64_t interval_us = rtpe_config.redis_sentinel_interval * 1000LL;
int64_t next = now_us() + interval_us, last = 0;
// a round is bounded by the timeouts; at shutdown, the waker ends the wait
s->waker.lock = &s->lock;
s->waker.cond = &s->cond;
thread_waker_add(&s->waker);
while (!rtpe_shutdown) {
{
LOCK(&s->lock);
// a wake-up comes at least 250 ms after the last round
while (!rtpe_shutdown) {
int64_t until = s->wake ? MIN(next, last + 250000) : next;
if (now_us() >= until)
break;
cond_timedwait(&s->cond, &s->lock, until);
}
s->wake = false;
}
if (rtpe_shutdown)
break;
bool refused = sentinel_round(s);
last = now_us();
next = last + (refused ? MIN(250000, interval_us) : interval_us);
}
thread_waker_del(&s->waker);
}