From fb83ee27343a657ea3ffdb7b4c7fe4878d4f389b Mon Sep 17 00:00:00 2001
From: Kirill Solomko <ksolomko@sipwise.com>
Date: Tue, 14 Apr 2026 17:54:42 +0200
Subject: [PATCH] MT#64891 enhance Redis connection handling

* Catalyst::Plugin::NGCP::Redis changes
  - new function _connect_and_cache() that
    now contains the migrated connect & cache code
    returns the connection reference
  - new function _get_replication_role() that
    sends "INFO replication" command to the server
    and returns the role string back
  - new function _wait_for_master_role() that
    tries to wait 5 seconds for the server to
    return the 'master' role, with 1 second attempt
    intervals
  - rework redis_get_connection to obtain the connection
    inside try/catch and use _wait_for_master_role() after
    reconnect, as well as using it for the 'heartbeat' part
    to test dead connections instead of PING (so 1 command
    is sent in the end instead of 2).
* With the redis_get_connection rework not only does it
  perform an automatic reconnect attempt if it detects a dead
  connection but it also waits until the server is 'master',
  which might happen during the db node switchovers, so API requests
  that happen during the switchover do not end up with errors
  "READ ONLY ROLE", and unless there is something persistently
  wrong on the server side, the reconnections/swithovers should
  be transparent now for the clients, with only probabaly 1-2
  second delays on select API requests while the server is becoming
  the new 'master'.

Change-Id: I41bf1f55a07b56e4fceef243f09f316694d62a39
(cherry picked from commit e5d7232b631aeae88c975151384e9e136a9fc561)
(cherry picked from commit 3a07710ad53dfa1602c95f611e88b4a9993a0d76)
---
 lib/Catalyst/Plugin/NGCP/Redis.pm | 66 ++++++++++++++++++++++++-------
 1 file changed, 52 insertions(+), 14 deletions(-)

diff --git a/lib/Catalyst/Plugin/NGCP/Redis.pm b/lib/Catalyst/Plugin/NGCP/Redis.pm
index 097eb44b13..2872f0f276 100644
--- a/lib/Catalyst/Plugin/NGCP/Redis.pm
+++ b/lib/Catalyst/Plugin/NGCP/Redis.pm
@@ -3,29 +3,67 @@ use strict;
 use warnings;
 use MRO::Compat;
 use Redis;
+use Try::Tiny;
 
 my $conn = {};
 
 sub redis_get_connection {
     my ($c, $params) = @_;
+    my $db = $params->{database} // return;
+    my $conn_ref;
+
+    try {
+        $conn_ref = $conn->{$db} // _connect_and_cache($c, $params);
+        _wait_for_master_role($conn_ref);
+    } catch {
+        $conn_ref = _connect_and_cache($c, $params) // return;
+        _wait_for_master_role($conn_ref);
+    };
+
+    return $conn_ref;
+}
 
+sub _connect_and_cache {
+    my ($c, $params) = @_;
     my $db = $params->{database} // return;
-    my $redis;
-    $redis = $conn->{$db} // do {
-        $redis = Redis->new(
-            server => $c->config->{redis}->{central_url},
-            reconnect => 10, every => 500000, # 500ms
-            cnx_timeout => 3,
-        );
-        unless ($redis) {
-            $c->log->error("Failed to connect to central redis url " . $c->config->{redis}->{central_url});
-            return;
-        }
-        $redis->select($params->{database});
-        $conn->{$db} = $redis;
+
+    my $conn_ref = Redis->new(
+        server => $c->config->{redis}->{central_url},
+        reconnect => 10, every => 500000, # 500ms
+        cnx_timeout => 3,
+    ) or do {
+        $c->log->error("Failed to connect to redis url " . $c->config->{redis}->{central_url});
+        return;
     };
 
-    return $redis;
+    $conn_ref->select($params->{database});
+    $conn->{$db} = $conn_ref;
+
+    return $conn_ref;
+}
+
+sub _wait_for_master_role {
+    my $conn_ref = shift;
+
+    my $cnt = 0;
+    my $interval = 1;
+    my $wait = 5;
+
+    while ($cnt < $wait) {
+        return 1 if _get_replication_role($conn_ref) eq 'master';
+        sleep $interval;
+        $cnt += $interval;
+    }
+
+    return;
+}
+
+sub _get_replication_role {
+    my $conn_ref = shift;
+
+    my $repl_info = $conn_ref->info('replication');
+
+    return ref $repl_info eq 'HASH' ? $repl_info->{role} : '';
 }
 
 1;