MT#55283 redis: TLS for the Redis Sentinel connections

Closes #2213

With the TLS options, the Sentinel thread still spoke plain TCP to the
Sentinels and for the role check of the master they name, so a Sentinel
set with tls-port and port 0 could not be used.

sentinel_connect() now starts TLS with the same context and options. A
Sentinel's certificate must match its host as given in
--redis-sentinel; a master's must match the address the Sentinels gave,
or --redis-tls-server-name. The handshake happens in the thread, never
under r->lock.

Tested by auto-daemon-tests-redis-tls.pl.

Change-Id: I84daf340c5f464765f234b35255cc76238bc8d13
pull/2241/head
Kevin Binetruy 1 month ago committed by Richard Fuchs
parent f74abf8e1e
commit fcd7746deb

@ -144,7 +144,8 @@ void redis_sentinel_wake(struct redis_sentinel *s, bool limited) {
}
static redisContext *sentinel_connect(const endpoint_t *ep, const char *auth) {
/* "name": what the TLS certificate must match, if not the address */
static redisContext *sentinel_connect(const endpoint_t *ep, const char *name, const char *auth) {
char host[64];
sockaddr_print(&ep->address, host, sizeof(host));
int64_t connect_timeout = atomic_get_na(&rtpe_config.redis_connect_timeout) * 1000LL;
@ -157,7 +158,8 @@ static redisContext *sentinel_connect(const endpoint_t *ep, const char *auth) {
return NULL;
const char *err = NULL;
redisReply *rp = NULL;
if (ctx->err || redisSetTimeout(ctx, timeval_from_us(cmd_timeout)))
if (ctx->err || redisSetTimeout(ctx, timeval_from_us(cmd_timeout))
|| redis_tls_start(ctx, name ? : host, 0, NULL))
err = ctx->errstr;
else if (auth && !(rp = redisCommand(ctx, "AUTH %s", auth)))
err = ctx->errstr;
@ -220,7 +222,7 @@ static bool sentinel_query(struct redis_sentinel *s, struct sentinel_node *n,
n->ep.address = a;
}
redisContext *ctx = sentinel_connect(&n->ep, s->auth);
redisContext *ctx = sentinel_connect(&n->ep, n->host, s->auth);
if (!ctx) {
n->failing = true;
return false;
@ -251,7 +253,8 @@ static bool sentinel_query(struct redis_sentinel *s, struct sentinel_node *n,
}
static bool sentinel_is_master(struct redis_sentinel *s, const endpoint_t *ep, uint64_t epoch) {
redisContext *ctx = sentinel_connect(ep, s->probe_auth);
// a data node: --redis-tls-server-name applies, as for the data connections
redisContext *ctx = sentinel_connect(ep, rtpe_config.redis_tls_server_name, s->probe_auth);
if (!ctx)
return false;

@ -821,6 +821,12 @@ call to inject-DTMF won't be sent to __\-\-dtmf-log-dest=__ or __\-\-listen-tcp-
__\-\-redis-connect-timeout__ bounds the handshake. These options are
only available in a build with __hiredis\_ssl__ (hiredis 1.0 or newer).
With __\-\-redis-sentinel__, the Sentinels and the role checks of the
masters they name use TLS too, with the same files: a Sentinel's
certificate must match its host as given in __\-\-redis-sentinel__, a
master's the address that the Sentinels give, or
__\-\-redis-tls-server-name__.
- __\-\-redis-tls-server-name=__*NAME*
The name to send (as SNI) and to expect in the certificate of every

@ -451,6 +451,8 @@ sub serve_probes {
# (GET-MASTER-ADDR-BY-NAME) at this config epoch (SENTINEL MASTER), whose
# "ip" is MONITORED_IP if given: a Sentinel running a failover reports the
# promoted replica in the former and still the old master in the latter.
# The key "tls" gives "CERT_FILE KEY_FILE": the Sentinels speak TLS with
# them, and log "IP:PORT TLS refused" for a handshake that fails.
my ($sentinel_pid, $sentinel_state, $sentinel_log, $sentinel_password);
my $serial = 0;
@ -474,7 +476,8 @@ sub sentinels_set {
$serial++;
open(my $fh, '>', "$sentinel_state.new") or die;
print $fh "serial $serial\npassword $sentinel_password\n";
print $fh "$_:26379 $st{$_}\n" for sort(keys(%st));
print $fh "tls $st{tls}\n" if $st{tls};
print $fh "$_:26379 $st{$_}\n" for sort(grep { $_ ne 'tls' } keys(%st));
close($fh);
rename("$sentinel_state.new", "$sentinel_state") or die;
return unless $sentinel_pid;
@ -514,16 +517,23 @@ sub sentinel_main {
my $done = 0;
$SIG{TERM} = sub { $done = 1 };
$SIG{PIPE} = 'IGNORE';
my (%sentinels, %clients, $password);
my (%sentinels, %clients, $password, $tls_files);
my $applied = -1;
my $read_state = sub {
open(my $fh, '<', $statefile) or return;
my ($ser, %want);
$tls_files = undef;
while (my $l = <$fh>) {
my @f = split(' ', $l);
if ($f[0] eq 'serial') { $ser = $f[1] }
elsif ($f[0] eq 'password') { $password = $f[1] }
elsif ($f[0] eq 'tls') {
# loaded now, not at the first connection: loading takes long
# enough on a busy host for the client's handshake to time out
require IO::Socket::SSL;
$tls_files = [ @f[1, 2] ];
}
else { $want{$f[0]} = [ @f[1 .. $#f] ] }
}
close($fh);
@ -579,6 +589,16 @@ sub sentinel_main {
for my $fn (grep { vec($rout, $_, 1) } keys(%l)) {
my ($s) = grep { $sentinels{$_} == $l{$fn} } keys(%sentinels);
accept(my $fh, $l{$fn}{listener}) or next;
if ($tls_files) {
if (!IO::Socket::SSL->start_SSL($fh, SSL_server => 1, Timeout => 3,
SSL_cert_file => $tls_files->[0],
SSL_key_file => $tls_files->[1]))
{
print $log "$s TLS refused\n";
close($fh);
next;
}
}
$clients{fileno($fh)} = { fh => $fh, buf => '', sentinel => $s };
}
for my $fn (grep { vec($rout, $_, 1) } keys(%clients)) {

@ -2,8 +2,9 @@
# TLS to Redis: the certificate must come from the CA and match the address
# (or --redis-tls-server-name), a client certificate is sent, a handshake
# that gets no answer is given up, and the keyspace subscription speaks TLS
# too. Certificates are made here with the openssl tool.
# that gets no answer is given up, and the keyspace subscription and the
# Redis Sentinel connections speak TLS too. Certificates are made here with
# the openssl tool.
use strict;
use warnings;
@ -49,6 +50,7 @@ tls_cert($d, 'ca', 'name-other', 'DNS:other.sub.test');
tls_cert($d, 'ca', 'name-wild', 'DNS:*.sub.test');
tls_cert($d, 'ca', 'name-partial', 'DNS:re*.sub.test');
tls_cert($d, 'ca', 'redis.sub.test'); # the name in the CN only
tls_cert($d, 'ca', 'sentinel', 'IP:203.0.113.43');
sub cert {
my ($name, %opt) = @_;
@ -270,4 +272,52 @@ redis_expect($notify_fd, ['PING'], "+PONG\r\n", 'name: subscription refused: con
$sub_fd = subscription('name: subscription again');
NGCP::Rtpengine::AutoTest::shut_rtpe();
undef($NGCP::Rtpengine::req_cb);
# Redis Sentinel: the queries and the role probes speak TLS. The Sentinels
# are checked against their addresses, the data nodes (the role probe
# included) against --redis-tls-server-name.
my $l44 = redis_listen('203.0.113.44');
cert('name');
my $sentinel_tls = "$d/sentinel.pem $d/sentinel.key";
sentinels_start('spw', tls => $sentinel_tls, '203.0.113.43' => '203.0.113.42 6379 1');
$NGCP::Rtpengine::AutoTest::launch_cb = sub {
$fd = redis_conn($l, 'sentinel: startup');
redis_expect($fd, ['PING'], "+PONG\r\n", 'sentinel: startup PING');
redis_expect($fd, ['KEYS', '*'], "*0\r\n", 'sentinel: startup KEYS');
};
autotest_start(qw(--config-file=none -t -1 -i 203.0.113.1
-n 2223 -f -L 7 -E
--redis=auth@sentinel/2
--redis-sentinel-master=mymaster --redis-sentinel-interval=200
--redis-cmd-timeout=1000 --redis-tls-server-name=redis.sub.test),
'--redis-sentinel=spw@203.0.113.43:26379', "--redis-cafile=$d/ca.pem")
or die;
like(sentinel_log(), qr/^203\.0\.113\.43:26379 AUTH spw$/m, 'sentinel: queried over TLS');
is_deeply(\@probes, [['203.0.113.42', 'auth', 'master']], 'sentinel: role probe over TLS');
# a Sentinel whose certificate is for another address is not asked
sentinels_set(tls => "$d/srv-ip.pem $d/srv-ip.key", '203.0.113.43' => '203.0.113.42 6379 1');
wait_log(qr/Redis Sentinel: cannot connect to 203\.0\.113\.43:26379: TLS: certificate of 203\.0\.113\.43 not accepted: IP address mismatch/,
'sentinel: certificate for another address refused');
like(sentinel_log(), qr/^203\.0\.113\.43:26379 TLS refused$/m, 'sentinel: handshake not completed');
# a master whose certificate is for another name is not followed
cert('name-other');
sentinels_set(tls => $sentinel_tls, '203.0.113.43' => '203.0.113.44 6379 2');
redis_tls_refused($l44, 'sentinel: role probe of a master with the wrong certificate');
wait_log(qr/Redis Sentinel: cannot connect to 203\.0\.113\.44:6379: TLS: certificate of redis\.sub\.test not accepted: hostname mismatch/,
'sentinel: role probe refused the certificate');
new_call;
$NGCP::Rtpengine::req_cb = sub { redis_expect_set($fd, cid(), "+OK\r\n", 'sentinel: write') };
offer('sentinel: still on the master in use', { }, $sdp);
unlike(rtpe_log(), qr/master of 'mymaster' is now 203\.0\.113\.44/, 'sentinel: not followed');
done_testing();

Loading…
Cancel
Save