diff --git a/daemon/main.c b/daemon/main.c index 1bdb1a70f..cb1a0a220 100644 --- a/daemon/main.c +++ b/daemon/main.c @@ -105,6 +105,8 @@ struct rtpengine_config rtpe_config = { .redis_write_db = -1, .redis_allowed_errors = -1, .redis_connect_timeout = 1000, + .redis_reconnect_pause = 1000, + .redis_reconnect_slow = 100, .media_num_threads = -1, .dtls_rsa_key_size = 2048, .dtls_mtu = 1200, // chrome default mtu @@ -817,6 +819,8 @@ static void options(int *argc, char ***argv, charp_ht templates) { { "redis-disable-time", 0, 0, G_OPTION_ARG_INT, &redis_disable_time, "Number of seconds redis communication is disabled because of errors", "INT" }, { "redis-cmd-timeout", 0, 0, G_OPTION_ARG_INT, &rtpe_config.redis_cmd_timeout, "Sets a timeout in milliseconds for redis commands", "INT" }, { "redis-connect-timeout", 0, 0, G_OPTION_ARG_INT, &rtpe_config.redis_connect_timeout, "Sets a timeout in milliseconds for redis connections", "INT" }, + { "redis-reconnect-pause", 0, 0, G_OPTION_ARG_INT, &rtpe_config.redis_reconnect_pause, "Milliseconds without Redis reconnect attempts after a slow or wrong-role failure", "INT" }, + { "redis-reconnect-slow", 0, 0, G_OPTION_ARG_INT, &rtpe_config.redis_reconnect_slow, "A failed Redis reconnect attempt taking this many milliseconds or more is followed by the pause", "INT" }, { "redis-format", 0, 0, G_OPTION_ARG_STRING, &redis_format, "Format for persistent storage in Redis/KeyDB", "native|bencode|JSON" }, { "subscribe-keyspace", 'k', 0, G_OPTION_ARG_STRING_ARRAY,&ks_a, "Subscription keyspace list", "INT INT ..."}, { "redis-tcp-keepalive-time",0,0,G_OPTION_ARG_INT,&rtpe_config.redis_tcp_keepalive_time,"Positive value sets tcp_keepalive_time for redis connections", "INT" }, @@ -1263,6 +1267,10 @@ static void options(int *argc, char ***argv, charp_ht templates) { rtpe_config.redis_disable_time_us = redis_disable_time * 1000000LL; if (rtpe_config.redis_disable_time_us < 0) die("Invalid negative redis-disable-time"); + if (rtpe_config.redis_reconnect_pause < 0) + die("Invalid negative redis-reconnect-pause"); + if (rtpe_config.redis_reconnect_slow < 0) + die("Invalid negative redis-reconnect-slow"); rtpe_config.mqtt_publish_interval_us = mqtt_publish_interval * 1000LL; if (rtpe_config.mqtt_publish_interval_us < 0) diff --git a/daemon/redis.c b/daemon/redis.c index 8bf1efc99..ab658f572 100644 --- a/daemon/redis.c +++ b/daemon/redis.c @@ -35,6 +35,8 @@ #include "sdp.h" #include "statistics.h" +#define REDIS_CONNECT_ROLE -2 /* redis_connect(): the server has the wrong role */ + typedef union { GQueue *q; stream_fd_q *sfds_q; @@ -284,7 +286,7 @@ int redis_reconnect(struct redis* r) { int rval; LOCK(&r->lock); - rval = redis_connect(r, 1, r->update_resolve); + rval = redis_connect(r, 0, r->update_resolve); if (rval) r->state = REDIS_STATE_DISCONNECTED; return rval; @@ -315,13 +317,16 @@ void redis_set_keepalive(int fd) { } -/* called with r->lock held if necessary */ +/* called with r->lock held if necessary + * Returns 0 when connected, REDIS_CONNECT_ROLE if the server never had the + * wanted role, or -1 on any other failure. */ static int redis_connect(struct redis *r, int wait, bool resolve) { struct timeval tv; redisReply *rp; char *s; int64_t cmd_timeout, connect_timeout; sockaddr_t a; + int ret = -1; if (r->ctx) redisFree(r->ctx); @@ -386,8 +391,8 @@ static int redis_connect(struct redis *r, int wait, bool resolve) { endpoint_print_buf(&r->endpoint)); goto done; } else if (r->role == SLAVE_REDIS_ROLE) { - ilog(LOG_INFO, "Connected to Redis %s in master mode, but wanted mode is slave; retrying...", - endpoint_print_buf(&r->endpoint)); + ilog(LOG_INFO, "Connected to Redis %s in master mode, but wanted mode is slave%s", + endpoint_print_buf(&r->endpoint), wait >= 0 ? "; retrying..." : ""); goto next; } } else if (!strncmp(s, "role:slave", 10)) { @@ -396,8 +401,8 @@ static int redis_connect(struct redis *r, int wait, bool resolve) { endpoint_print_buf(&r->endpoint)); goto done; } else if (r->role == MASTER_REDIS_ROLE) { - ilog(LOG_INFO, "Connected to Redis %s in slave mode, but wanted mode is master; retrying...", - endpoint_print_buf(&r->endpoint)); + ilog(LOG_INFO, "Connected to Redis %s in slave mode, but wanted mode is master%s", + endpoint_print_buf(&r->endpoint), wait >= 0 ? "; retrying..." : ""); goto next; } } else { @@ -406,9 +411,11 @@ static int redis_connect(struct redis *r, int wait, bool resolve) { next: freeReplyObject(rp); - usleep(1000000); + if (wait >= 0) + usleep(1000000); } + ret = REDIS_CONNECT_ROLE; goto err2; done: @@ -429,7 +436,7 @@ err2: err: rlog(LOG_ERR, "Failed to connect to Redis %s", endpoint_print_buf(&r->endpoint)); - return -1; + return ret; } @@ -1031,13 +1038,21 @@ static int redis_check_conn(struct redis *r) { rtpe_now = now_us(); // XXX this needed here? if ((r->state == REDIS_STATE_DISCONNECTED) && r->restore_tick_us > rtpe_now) { - ilog(LOG_WARNING, "Redis server '%s' is disabled. Don't try RE-Establishing for %" PRId64 " more seconds", - r->hostname, (r->restore_tick_us - rtpe_now) / 1000000); + // every Redis operation ends up here: say it once + r->skipped++; + if (!r->skipped_logged) + ilog(LOG_WARNING, "Redis server %s is disabled. Don't try RE-Establishing for %" PRId64 " more ms", + endpoint_print_buf(&r->endpoint), (r->restore_tick_us - rtpe_now) / 1000); + r->skipped_logged = true; return REDIS_STATE_DISCONNECTED; } - if (r->state == REDIS_STATE_DISCONNECTED) - ilog(LOG_INFO, "RE-Establishing connection for Redis server '%s'", r->hostname); + if (r->state == REDIS_STATE_DISCONNECTED) { + ilog(LOG_INFO, "RE-Establishing connection for Redis server %s (%u operations skipped)", + endpoint_print_buf(&r->endpoint), r->skipped); + r->skipped = 0; + r->skipped_logged = false; + } // try redis connection if (r->ctx && redisCommandNR(r->ctx, "PING") == 0) { @@ -1058,9 +1073,23 @@ static int redis_check_conn(struct redis *r) { } // try redis reconnect => will free current r->ctx - if (redis_connect(r, 1, r->update_resolve)) { + // (one attempt: any wait here would hold every thread that uses Redis) + int64_t start = now_us(); + int ret = redis_connect(r, 0, r->update_resolve); + if (ret) { // redis is disconnected redis_count_err_and_disable(r); + // After the wrong role, or a slow failure (a timeout), pause (one + // second by default): a failover takes longer than that, so the + // next attempt would fail the same way, under the lock. A quick + // failure (connection refused) is retried at once, as before. + int64_t now = now_us(); + int64_t pause_us = rtpe_config.redis_reconnect_pause * 1000LL; + if (ret == REDIS_CONNECT_ROLE || now - start >= rtpe_config.redis_reconnect_slow * 1000LL) { + if (r->restore_tick_us < now + pause_us) + r->restore_tick_us = now + pause_us; + r->skipped_logged = false; + } return REDIS_STATE_DISCONNECTED; } diff --git a/docs/rtpengine.md b/docs/rtpengine.md index 94ab77afd..bfc14139f 100644 --- a/docs/rtpengine.md +++ b/docs/rtpengine.md @@ -643,6 +643,12 @@ call to inject-DTMF won't be sent to __\-\-dtmf-log-dest=__ or __\-\-listen-tcp- Default value is -1, meaning that this feature is disabled. This parameter can also be set or listed via __rtpengine-ctl__. + Independently of this parameter, a lost Redis connection is + re-established with a single attempt, and a failed attempt can be + followed by a pause (see __\-\-redis-reconnect-pause__). The errors + counted here are failed attempts, so a role or slow failure counts at + most once per pause. + - __\-\-redis-disable-time__ This parameter configures the number of seconds redis communication is @@ -672,10 +678,31 @@ call to inject-DTMF won't be sent to __\-\-dtmf-log-dest=__ or __\-\-listen-tcp- is issued before attempting to connect so the __\-\-redis-cmd-timeout__ value will also be added to the total waiting time. This is useful if using __\-\-redis-allowed-errors__, when attempting to - estimate the total lost time in case of redis failures. + estimate the total lost time in case of redis failures (but note that a + slow failed attempt is followed by a pause, see + __\-\-redis-reconnect-pause__). The default value for the connection timeout is 1000ms. This parameter can also be set or listed via __rtpengine-ctl__. +- __\-\-redis-reconnect-pause=__*INT* + + Time in milliseconds during which Redis operations on a connection are + skipped (not written), and no new attempt is made to re-establish it, + after an attempt that failed because the server does not have the + required role, or that took __\-\-redis-reconnect-slow__ or more. An + attempt holds every thread that uses the connection, for up to + __\-\-redis-connect-timeout__ plus __\-\-redis-cmd-timeout__ for each of + __AUTH__, __SELECT__ and __INFO__. 0 disables the pause. The default + value is 1000. + +- __\-\-redis-reconnect-slow=__*INT* + + A failed attempt to re-establish a Redis connection that took this many + milliseconds or more (typically a timeout) is followed by the pause of + __\-\-redis-reconnect-pause__; a quicker failure, such as a refused + connection, is retried by the next operation. 0 pauses after every + failed attempt. The default value is 100. + - __\-\-redis-resolve-on-reconnect__ Enable 'redis resolve on reconnect' functionality: when re-connecting to the diff --git a/etc/rtpengine.conf b/etc/rtpengine.conf index 6dd1b6b6a..d5dc2e429 100644 --- a/etc/rtpengine.conf +++ b/etc/rtpengine.conf @@ -81,6 +81,8 @@ recording-method = proc # redis-disable-time = 10 # redis-cmd-timeout = 0 # redis-connect-timeout = 1000 +# redis-reconnect-pause = 1000 +# redis-reconnect-slow = 100 # redis-resolve-on-reconnect = false # redis-tcp-keepalive-time = 1 # redis-tcp-keepalive-intvl = 1 diff --git a/include/main.h b/include/main.h index 35bd34838..861e09fa9 100644 --- a/include/main.h +++ b/include/main.h @@ -55,6 +55,8 @@ enum endpoint_learning { X(redis_allowed_errors) \ X(redis_cmd_timeout) \ X(redis_connect_timeout) \ + X(redis_reconnect_pause) \ + X(redis_reconnect_slow) \ X(redis_delete_async) \ X(redis_delete_async_interval) \ X(redis_tcp_keepalive_time) \ diff --git a/include/redis.h b/include/redis.h index 91ec5c023..7d93e41b5 100644 --- a/include/redis.h +++ b/include/redis.h @@ -59,6 +59,8 @@ struct redis { int no_redis_required; int consecutive_errors; int64_t restore_tick_us; + unsigned int skipped; /* operations skipped until restore_tick_us */ + bool skipped_logged; unsigned int write_errors; /* refused writes not reported in the log yet */ bool writes_refused; /* the last write was refused */ int64_t write_errors_log_us; /* time of the last report */ diff --git a/t/auto-daemon-tests-redis-errors.pl b/t/auto-daemon-tests-redis-errors.pl index 4e13bba59..dd720b207 100755 --- a/t/auto-daemon-tests-redis-errors.pl +++ b/t/auto-daemon-tests-redis-errors.pl @@ -11,6 +11,7 @@ use NGCP::Rtpengine::Test; use NGCP::Rtpengine::AutoTest; use NGCP::Rtpengine::FakeRedis; use Test::More; +use Time::HiRes qw(time sleep); my $listener = redis_listen('203.0.113.42'); @@ -117,6 +118,142 @@ $NGCP::Rtpengine::req_cb = sub { offer('write after no reply', { }, $sdp); +# The server closes the connection (as Sentinel does when it demotes a +# master) and the reconnect finds a replica, while the master role is +# required (--redis without --redis-write): one INFO, and no sleep or second +# INFO under the lock. The INFO reply has nothing after the role line, which +# must not be read past its end (an ASan build aborts without strncmp()). + +redis_close($fd); + +new_call; +$NGCP::Rtpengine::req_cb = sub { + $fd = redis_conn($listener, 'replica', role => 'slave', short => 1, refused => 1); + redis_closed($fd, 'replica: connection closed after one INFO'); + # no ng reply if rtpengine died: fail instead of waiting forever + alarm(5); +}; +offer('write while the node is a replica', { }, $sdp); +alarm(0); + +# within the next second, no new connection is attempted; after it, the next +# write reconnects, and the node is master again + +new_call; +$NGCP::Rtpengine::req_cb = sub { + ok(!readable($listener, 0.3), 'no reconnect within the pacing interval') + or die "reconnect attempted within the pacing interval\n"; +}; +offer('write within the pacing interval', { }, $sdp); + +sleep(1); + +new_call; +$NGCP::Rtpengine::req_cb = sub { + $fd = redis_conn($listener, 'master again'); + redis_expect_set($fd, cid(), "+OK\r\n", 'master again', no_ping => 1); +}; +offer('write after the pacing interval', { }, $sdp); + + +# The server goes away (the connection is refused): that failure is quick, +# so it is not paced, and the first write after the server is back goes +# through: the reconnect must come before the ng reply. + +redis_close($fd); +close($listener); + +new_call; +undef($NGCP::Rtpengine::req_cb); +my $log_pos = length(rtpe_log()); +offer('write while the server is down', { }, $sdp); +# (under the test preload library, a refused connection shows as "No such +# file or directory") +like(substr(rtpe_log(), $log_pos), qr/Lost connection to Redis '203\.0\.113\.42'\n(?:.*\n)*?.*Failed to connect to Redis 203\.0\.113\.42:6379, error: /, + 'connection refused'); + +$listener = redis_listen('203.0.113.42'); + +new_call; +$NGCP::Rtpengine::req_cb = sub { + ok(readable($listener, 2), 'reconnect on the first write after a refused connection') + or die "no reconnect after a refused connection\n"; + $fd = redis_conn($listener, 'server back'); + redis_expect_set($fd, cid(), "+OK\r\n", 'server back', no_ping => 1); +}; +offer('write when the server is back', { }, $sdp); + + +# Non-default pacing: with --redis-reconnect-slow=0 even a refused +# connection is followed by the pause, and --redis-reconnect-pause=2500 +# makes it last longer than the default second. + +undef($NGCP::Rtpengine::req_cb); +NGCP::Rtpengine::AutoTest::shut_rtpe(); +redis_close($fd); +autotest_start(qw(--config-file=none -t -1 -i 203.0.113.1 + -n 2223 -f -L 7 -E --redis=auth@203.0.113.42:6379/2 + --redis-cmd-timeout=1000 + --redis-reconnect-pause=2500 --redis-reconnect-slow=0)) + or die; + +redis_close($fd); +close($listener); + +new_call; +offer('write while the server is down, paced', { }, $sdp); +my $t0 = time(); # the pause began during that request +$listener = redis_listen('203.0.113.42'); + +new_call; +$NGCP::Rtpengine::req_cb = sub { + ok(!readable($listener, 0.3), 'refused connection followed by the pause'); +}; +offer('write after a refused connection', { }, $sdp); + +sleep(1.2 - (time() - $t0)); +new_call; +$NGCP::Rtpengine::req_cb = sub { + ok(!readable($listener, 0.2), 'still paused after one second'); +}; +offer('write after one second', { }, $sdp); + +sleep(2.7 - (time() - $t0)); +new_call; +$NGCP::Rtpengine::req_cb = sub { + $fd = redis_conn($listener, 'after the pause'); + redis_expect_set($fd, cid(), "+OK\r\n", 'after the pause', no_ping => 1); +}; +offer('write after the pause', { }, $sdp); + + +# --redis-reconnect-pause=0: no pause, even after a failure counted as slow + +undef($NGCP::Rtpengine::req_cb); +NGCP::Rtpengine::AutoTest::shut_rtpe(); +redis_close($fd); +autotest_start(qw(--config-file=none -t -1 -i 203.0.113.1 + -n 2223 -f -L 7 -E --redis=auth@203.0.113.42:6379/2 + --redis-cmd-timeout=1000 + --redis-reconnect-pause=0 --redis-reconnect-slow=0)) + or die; + +redis_close($fd); +close($listener); +new_call; +offer('write while the server is down, no pause', { }, $sdp); +$listener = redis_listen('203.0.113.42'); + +new_call; +$NGCP::Rtpengine::req_cb = sub { + ok(readable($listener, 2), 'no pause: the next write reconnects') + or die "no reconnect with --redis-reconnect-pause=0\n"; + $fd = redis_conn($listener, 'no pause'); + redis_expect_set($fd, cid(), "+OK\r\n", 'no pause', no_ping => 1); +}; +offer('write right after a failure', { }, $sdp); + + undef($NGCP::Rtpengine::req_cb); done_testing();