MT#55283 redis: do not sleep under the lock when reconnecting

Ref #2211

On a role mismatch, the reconnect in redis_check_conn() sleeps one
second, asks INFO again and sleeps once more: two seconds under
r->lock, which every Redis update holds before the ng reply (#1865).
After a failover, with --redis on the old master's address, every
update did this, and ng requests timed out.

Make one attempt, with no sleep. After a role failure, or a failure
that took --redis-reconnect-slow (100 ms) or more, skip Redis
operations on that connection for --redis-reconnect-pause (1000 ms),
through restore_tick_us as --redis-allowed-errors does. A quick failure
(connection refused) is retried at once, as before. As with a failed
attempt, a skipped operation is not written; skips are logged once per
pause. The startup connection keeps its eleven attempts.

Tested by auto-daemon-tests-redis-errors.pl.

Change-Id: Ia84288f1c6fe97c4317689fef75d0ff5fc9061f8
pull/2221/head
Kevin Binetruy 1 month ago committed by Richard Fuchs
parent 4a08a1ce26
commit 64cf271411

@ -105,6 +105,8 @@ struct rtpengine_config rtpe_config = {
.redis_write_db = -1,
.redis_allowed_errors = -1,
.redis_connect_timeout = 1000,
.redis_reconnect_pause = 1000,
.redis_reconnect_slow = 100,
.media_num_threads = -1,
.dtls_rsa_key_size = 2048,
.dtls_mtu = 1200, // chrome default mtu
@ -817,6 +819,8 @@ static void options(int *argc, char ***argv, charp_ht templates) {
{ "redis-disable-time", 0, 0, G_OPTION_ARG_INT, &redis_disable_time, "Number of seconds redis communication is disabled because of errors", "INT" },
{ "redis-cmd-timeout", 0, 0, G_OPTION_ARG_INT, &rtpe_config.redis_cmd_timeout, "Sets a timeout in milliseconds for redis commands", "INT" },
{ "redis-connect-timeout", 0, 0, G_OPTION_ARG_INT, &rtpe_config.redis_connect_timeout, "Sets a timeout in milliseconds for redis connections", "INT" },
{ "redis-reconnect-pause", 0, 0, G_OPTION_ARG_INT, &rtpe_config.redis_reconnect_pause, "Milliseconds without Redis reconnect attempts after a slow or wrong-role failure", "INT" },
{ "redis-reconnect-slow", 0, 0, G_OPTION_ARG_INT, &rtpe_config.redis_reconnect_slow, "A failed Redis reconnect attempt taking this many milliseconds or more is followed by the pause", "INT" },
{ "redis-format", 0, 0, G_OPTION_ARG_STRING, &redis_format, "Format for persistent storage in Redis/KeyDB", "native|bencode|JSON" },
{ "subscribe-keyspace", 'k', 0, G_OPTION_ARG_STRING_ARRAY,&ks_a, "Subscription keyspace list", "INT INT ..."},
{ "redis-tcp-keepalive-time",0,0,G_OPTION_ARG_INT,&rtpe_config.redis_tcp_keepalive_time,"Positive value sets tcp_keepalive_time for redis connections", "INT" },
@ -1263,6 +1267,10 @@ static void options(int *argc, char ***argv, charp_ht templates) {
rtpe_config.redis_disable_time_us = redis_disable_time * 1000000LL;
if (rtpe_config.redis_disable_time_us < 0)
die("Invalid negative redis-disable-time");
if (rtpe_config.redis_reconnect_pause < 0)
die("Invalid negative redis-reconnect-pause");
if (rtpe_config.redis_reconnect_slow < 0)
die("Invalid negative redis-reconnect-slow");
rtpe_config.mqtt_publish_interval_us = mqtt_publish_interval * 1000LL;
if (rtpe_config.mqtt_publish_interval_us < 0)

@ -35,6 +35,8 @@
#include "sdp.h"
#include "statistics.h"
#define REDIS_CONNECT_ROLE -2 /* redis_connect(): the server has the wrong role */
typedef union {
GQueue *q;
stream_fd_q *sfds_q;
@ -284,7 +286,7 @@ int redis_reconnect(struct redis* r) {
int rval;
LOCK(&r->lock);
rval = redis_connect(r, 1, r->update_resolve);
rval = redis_connect(r, 0, r->update_resolve);
if (rval)
r->state = REDIS_STATE_DISCONNECTED;
return rval;
@ -315,13 +317,16 @@ void redis_set_keepalive(int fd) {
}
/* called with r->lock held if necessary */
/* called with r->lock held if necessary
* Returns 0 when connected, REDIS_CONNECT_ROLE if the server never had the
* wanted role, or -1 on any other failure. */
static int redis_connect(struct redis *r, int wait, bool resolve) {
struct timeval tv;
redisReply *rp;
char *s;
int64_t cmd_timeout, connect_timeout;
sockaddr_t a;
int ret = -1;
if (r->ctx)
redisFree(r->ctx);
@ -386,8 +391,8 @@ static int redis_connect(struct redis *r, int wait, bool resolve) {
endpoint_print_buf(&r->endpoint));
goto done;
} else if (r->role == SLAVE_REDIS_ROLE) {
ilog(LOG_INFO, "Connected to Redis %s in master mode, but wanted mode is slave; retrying...",
endpoint_print_buf(&r->endpoint));
ilog(LOG_INFO, "Connected to Redis %s in master mode, but wanted mode is slave%s",
endpoint_print_buf(&r->endpoint), wait >= 0 ? "; retrying..." : "");
goto next;
}
} else if (!strncmp(s, "role:slave", 10)) {
@ -396,8 +401,8 @@ static int redis_connect(struct redis *r, int wait, bool resolve) {
endpoint_print_buf(&r->endpoint));
goto done;
} else if (r->role == MASTER_REDIS_ROLE) {
ilog(LOG_INFO, "Connected to Redis %s in slave mode, but wanted mode is master; retrying...",
endpoint_print_buf(&r->endpoint));
ilog(LOG_INFO, "Connected to Redis %s in slave mode, but wanted mode is master%s",
endpoint_print_buf(&r->endpoint), wait >= 0 ? "; retrying..." : "");
goto next;
}
} else {
@ -406,9 +411,11 @@ static int redis_connect(struct redis *r, int wait, bool resolve) {
next:
freeReplyObject(rp);
usleep(1000000);
if (wait >= 0)
usleep(1000000);
}
ret = REDIS_CONNECT_ROLE;
goto err2;
done:
@ -429,7 +436,7 @@ err2:
err:
rlog(LOG_ERR, "Failed to connect to Redis %s",
endpoint_print_buf(&r->endpoint));
return -1;
return ret;
}
@ -1031,13 +1038,21 @@ static int redis_check_conn(struct redis *r) {
rtpe_now = now_us(); // XXX this needed here?
if ((r->state == REDIS_STATE_DISCONNECTED) && r->restore_tick_us > rtpe_now) {
ilog(LOG_WARNING, "Redis server '%s' is disabled. Don't try RE-Establishing for %" PRId64 " more seconds",
r->hostname, (r->restore_tick_us - rtpe_now) / 1000000);
// every Redis operation ends up here: say it once
r->skipped++;
if (!r->skipped_logged)
ilog(LOG_WARNING, "Redis server %s is disabled. Don't try RE-Establishing for %" PRId64 " more ms",
endpoint_print_buf(&r->endpoint), (r->restore_tick_us - rtpe_now) / 1000);
r->skipped_logged = true;
return REDIS_STATE_DISCONNECTED;
}
if (r->state == REDIS_STATE_DISCONNECTED)
ilog(LOG_INFO, "RE-Establishing connection for Redis server '%s'", r->hostname);
if (r->state == REDIS_STATE_DISCONNECTED) {
ilog(LOG_INFO, "RE-Establishing connection for Redis server %s (%u operations skipped)",
endpoint_print_buf(&r->endpoint), r->skipped);
r->skipped = 0;
r->skipped_logged = false;
}
// try redis connection
if (r->ctx && redisCommandNR(r->ctx, "PING") == 0) {
@ -1058,9 +1073,23 @@ static int redis_check_conn(struct redis *r) {
}
// try redis reconnect => will free current r->ctx
if (redis_connect(r, 1, r->update_resolve)) {
// (one attempt: any wait here would hold every thread that uses Redis)
int64_t start = now_us();
int ret = redis_connect(r, 0, r->update_resolve);
if (ret) {
// redis is disconnected
redis_count_err_and_disable(r);
// After the wrong role, or a slow failure (a timeout), pause (one
// second by default): a failover takes longer than that, so the
// next attempt would fail the same way, under the lock. A quick
// failure (connection refused) is retried at once, as before.
int64_t now = now_us();
int64_t pause_us = rtpe_config.redis_reconnect_pause * 1000LL;
if (ret == REDIS_CONNECT_ROLE || now - start >= rtpe_config.redis_reconnect_slow * 1000LL) {
if (r->restore_tick_us < now + pause_us)
r->restore_tick_us = now + pause_us;
r->skipped_logged = false;
}
return REDIS_STATE_DISCONNECTED;
}

@ -643,6 +643,12 @@ call to inject-DTMF won't be sent to __\-\-dtmf-log-dest=__ or __\-\-listen-tcp-
Default value is -1, meaning that this feature is disabled.
This parameter can also be set or listed via __rtpengine-ctl__.
Independently of this parameter, a lost Redis connection is
re-established with a single attempt, and a failed attempt can be
followed by a pause (see __\-\-redis-reconnect-pause__). The errors
counted here are failed attempts, so a role or slow failure counts at
most once per pause.
- __\-\-redis-disable-time__
This parameter configures the number of seconds redis communication is
@ -672,10 +678,31 @@ call to inject-DTMF won't be sent to __\-\-dtmf-log-dest=__ or __\-\-listen-tcp-
is issued before attempting to connect so the __\-\-redis-cmd-timeout__ value
will also be added to the total waiting time.
This is useful if using __\-\-redis-allowed-errors__, when attempting to
estimate the total lost time in case of redis failures.
estimate the total lost time in case of redis failures (but note that a
slow failed attempt is followed by a pause, see
__\-\-redis-reconnect-pause__).
The default value for the connection timeout is 1000ms.
This parameter can also be set or listed via __rtpengine-ctl__.
- __\-\-redis-reconnect-pause=__*INT*
Time in milliseconds during which Redis operations on a connection are
skipped (not written), and no new attempt is made to re-establish it,
after an attempt that failed because the server does not have the
required role, or that took __\-\-redis-reconnect-slow__ or more. An
attempt holds every thread that uses the connection, for up to
__\-\-redis-connect-timeout__ plus __\-\-redis-cmd-timeout__ for each of
__AUTH__, __SELECT__ and __INFO__. 0 disables the pause. The default
value is 1000.
- __\-\-redis-reconnect-slow=__*INT*
A failed attempt to re-establish a Redis connection that took this many
milliseconds or more (typically a timeout) is followed by the pause of
__\-\-redis-reconnect-pause__; a quicker failure, such as a refused
connection, is retried by the next operation. 0 pauses after every
failed attempt. The default value is 100.
- __\-\-redis-resolve-on-reconnect__
Enable 'redis resolve on reconnect' functionality: when re-connecting to the

@ -81,6 +81,8 @@ recording-method = proc
# redis-disable-time = 10
# redis-cmd-timeout = 0
# redis-connect-timeout = 1000
# redis-reconnect-pause = 1000
# redis-reconnect-slow = 100
# redis-resolve-on-reconnect = false
# redis-tcp-keepalive-time = 1
# redis-tcp-keepalive-intvl = 1

@ -55,6 +55,8 @@ enum endpoint_learning {
X(redis_allowed_errors) \
X(redis_cmd_timeout) \
X(redis_connect_timeout) \
X(redis_reconnect_pause) \
X(redis_reconnect_slow) \
X(redis_delete_async) \
X(redis_delete_async_interval) \
X(redis_tcp_keepalive_time) \

@ -59,6 +59,8 @@ struct redis {
int no_redis_required;
int consecutive_errors;
int64_t restore_tick_us;
unsigned int skipped; /* operations skipped until restore_tick_us */
bool skipped_logged;
unsigned int write_errors; /* refused writes not reported in the log yet */
bool writes_refused; /* the last write was refused */
int64_t write_errors_log_us; /* time of the last report */

@ -11,6 +11,7 @@ use NGCP::Rtpengine::Test;
use NGCP::Rtpengine::AutoTest;
use NGCP::Rtpengine::FakeRedis;
use Test::More;
use Time::HiRes qw(time sleep);
my $listener = redis_listen('203.0.113.42');
@ -117,6 +118,142 @@ $NGCP::Rtpengine::req_cb = sub {
offer('write after no reply', { }, $sdp);
# The server closes the connection (as Sentinel does when it demotes a
# master) and the reconnect finds a replica, while the master role is
# required (--redis without --redis-write): one INFO, and no sleep or second
# INFO under the lock. The INFO reply has nothing after the role line, which
# must not be read past its end (an ASan build aborts without strncmp()).
redis_close($fd);
new_call;
$NGCP::Rtpengine::req_cb = sub {
$fd = redis_conn($listener, 'replica', role => 'slave', short => 1, refused => 1);
redis_closed($fd, 'replica: connection closed after one INFO');
# no ng reply if rtpengine died: fail instead of waiting forever
alarm(5);
};
offer('write while the node is a replica', { }, $sdp);
alarm(0);
# within the next second, no new connection is attempted; after it, the next
# write reconnects, and the node is master again
new_call;
$NGCP::Rtpengine::req_cb = sub {
ok(!readable($listener, 0.3), 'no reconnect within the pacing interval')
or die "reconnect attempted within the pacing interval\n";
};
offer('write within the pacing interval', { }, $sdp);
sleep(1);
new_call;
$NGCP::Rtpengine::req_cb = sub {
$fd = redis_conn($listener, 'master again');
redis_expect_set($fd, cid(), "+OK\r\n", 'master again', no_ping => 1);
};
offer('write after the pacing interval', { }, $sdp);
# The server goes away (the connection is refused): that failure is quick,
# so it is not paced, and the first write after the server is back goes
# through: the reconnect must come before the ng reply.
redis_close($fd);
close($listener);
new_call;
undef($NGCP::Rtpengine::req_cb);
my $log_pos = length(rtpe_log());
offer('write while the server is down', { }, $sdp);
# (under the test preload library, a refused connection shows as "No such
# file or directory")
like(substr(rtpe_log(), $log_pos), qr/Lost connection to Redis '203\.0\.113\.42'\n(?:.*\n)*?.*Failed to connect to Redis 203\.0\.113\.42:6379, error: /,
'connection refused');
$listener = redis_listen('203.0.113.42');
new_call;
$NGCP::Rtpengine::req_cb = sub {
ok(readable($listener, 2), 'reconnect on the first write after a refused connection')
or die "no reconnect after a refused connection\n";
$fd = redis_conn($listener, 'server back');
redis_expect_set($fd, cid(), "+OK\r\n", 'server back', no_ping => 1);
};
offer('write when the server is back', { }, $sdp);
# Non-default pacing: with --redis-reconnect-slow=0 even a refused
# connection is followed by the pause, and --redis-reconnect-pause=2500
# makes it last longer than the default second.
undef($NGCP::Rtpengine::req_cb);
NGCP::Rtpengine::AutoTest::shut_rtpe();
redis_close($fd);
autotest_start(qw(--config-file=none -t -1 -i 203.0.113.1
-n 2223 -f -L 7 -E --redis=auth@203.0.113.42:6379/2
--redis-cmd-timeout=1000
--redis-reconnect-pause=2500 --redis-reconnect-slow=0))
or die;
redis_close($fd);
close($listener);
new_call;
offer('write while the server is down, paced', { }, $sdp);
my $t0 = time(); # the pause began during that request
$listener = redis_listen('203.0.113.42');
new_call;
$NGCP::Rtpengine::req_cb = sub {
ok(!readable($listener, 0.3), 'refused connection followed by the pause');
};
offer('write after a refused connection', { }, $sdp);
sleep(1.2 - (time() - $t0));
new_call;
$NGCP::Rtpengine::req_cb = sub {
ok(!readable($listener, 0.2), 'still paused after one second');
};
offer('write after one second', { }, $sdp);
sleep(2.7 - (time() - $t0));
new_call;
$NGCP::Rtpengine::req_cb = sub {
$fd = redis_conn($listener, 'after the pause');
redis_expect_set($fd, cid(), "+OK\r\n", 'after the pause', no_ping => 1);
};
offer('write after the pause', { }, $sdp);
# --redis-reconnect-pause=0: no pause, even after a failure counted as slow
undef($NGCP::Rtpengine::req_cb);
NGCP::Rtpengine::AutoTest::shut_rtpe();
redis_close($fd);
autotest_start(qw(--config-file=none -t -1 -i 203.0.113.1
-n 2223 -f -L 7 -E --redis=auth@203.0.113.42:6379/2
--redis-cmd-timeout=1000
--redis-reconnect-pause=0 --redis-reconnect-slow=0))
or die;
redis_close($fd);
close($listener);
new_call;
offer('write while the server is down, no pause', { }, $sdp);
$listener = redis_listen('203.0.113.42');
new_call;
$NGCP::Rtpengine::req_cb = sub {
ok(readable($listener, 2), 'no pause: the next write reconnects')
or die "no reconnect with --redis-reconnect-pause=0\n";
$fd = redis_conn($listener, 'no pause');
redis_expect_set($fd, cid(), "+OK\r\n", 'no pause', no_ping => 1);
};
offer('write right after a failure', { }, $sdp);
undef($NGCP::Rtpengine::req_cb);
done_testing();

Loading…
Cancel
Save