MT#64891 enhance Redis connection handling

* Catalyst::Plugin::NGCP::Redis changes
  - new function _connect_and_cache() that
    now contains the migrated connect & cache code
    returns the connection reference
  - new function _get_replication_role() that
    sends "INFO replication" command to the server
    and returns the role string back
  - new function _wait_for_master_role() that
    tries to wait 5 seconds for the server to
    return the 'master' role, with 1 second attempt
    intervals
  - rework redis_get_connection to obtain the connection
    inside try/catch and use _wait_for_master_role() after
    reconnect, as well as using it for the 'heartbeat' part
    to test dead connections instead of PING (so 1 command
    is sent in the end instead of 2).
* With the redis_get_connection rework not only does it
  perform an automatic reconnect attempt if it detects a dead
  connection but it also waits until the server is 'master',
  which might happen during the db node switchovers, so API requests
  that happen during the switchover do not end up with errors
  "READ ONLY ROLE", and unless there is something persistently
  wrong on the server side, the reconnections/swithovers should
  be transparent now for the clients, with only probabaly 1-2
  second delays on select API requests while the server is becoming
  the new 'master'.

Change-Id: I41bf1f55a07b56e4fceef243f09f316694d62a39
(cherry picked from commit e5d7232b63)
(cherry picked from commit 3a07710ad5)

lib/Catalyst/Plugin/NGCP/Redis.pm

@@ -3,29 +3,67 @@ use strict;
 use warnings;
 use MRO::Compat;
 use Redis;
+use Try::Tiny;
 
 my $conn = {};
 
 sub redis_get_connection {
     my ($c, $params) = @_;
+    my $db = $params->{database} // return;
+    my $conn_ref;
+
+    try {
+        $conn_ref = $conn->{$db} // _connect_and_cache($c, $params);
+        _wait_for_master_role($conn_ref);
+    } catch {
+        $conn_ref = _connect_and_cache($c, $params) // return;
+        _wait_for_master_role($conn_ref);
+    };
+
+    return $conn_ref;
+}
 
+sub _connect_and_cache {
+    my ($c, $params) = @_;
     my $db = $params->{database} // return;
-    my $redis;
-    $redis = $conn->{$db} // do {
-        $redis = Redis->new(
-            server => $c->config->{redis}->{central_url},
-            reconnect => 10, every => 500000, # 500ms
-            cnx_timeout => 3,
-        );
-        unless ($redis) {
-            $c->log->error("Failed to connect to central redis url " . $c->config->{redis}->{central_url});
-            return;
-        }
-        $redis->select($params->{database});
-        $conn->{$db} = $redis;
+
+    my $conn_ref = Redis->new(
+        server => $c->config->{redis}->{central_url},
+        reconnect => 10, every => 500000, # 500ms
+        cnx_timeout => 3,
+    ) or do {
+        $c->log->error("Failed to connect to redis url " . $c->config->{redis}->{central_url});
+        return;
     };
 
-    return $redis;
+    $conn_ref->select($params->{database});
+    $conn->{$db} = $conn_ref;
+
+    return $conn_ref;
+}
+
+sub _wait_for_master_role {
+    my $conn_ref = shift;
+
+    my $cnt = 0;
+    my $interval = 1;
+    my $wait = 5;
+
+    while ($cnt < $wait) {
+        return 1 if _get_replication_role($conn_ref) eq 'master';
+        sleep $interval;
+        $cnt += $interval;
+    }
+
+    return;
+}
+
+sub _get_replication_role {
+    my $conn_ref = shift;
+
+    my $repl_info = $conn_ref->info('replication');
+
+    return ref $repl_info eq 'HASH' ? $repl_info->{role} : '';
 }
 
 1;