MT#63182 /api/subscribers add lockwait and deadlock detection

* API::check_deadlock() add lockwait detection.
* lockwait and deadlock detection is now in place for ALL
  /api/subscribers endpoint methods. because of logic complexity
  it uses own deadlock detection instead of relying on Entities.
  It however uses $self->check_deadlock() and similar logic.

Change-Id: Idc1876910711563fe14c878541e350ad373d6d31
(cherry picked from commit 3bb3d9e4e4)
(cherry picked from commit 09d011af45)

lib/NGCP/Panel/Controller/API/Subscribers.pm

@@ -315,6 +315,9 @@ sub GET :Allow {
     my $rows = $c->request->params->{rows} // 10;
     my $schema = $c->model('DB');
     $schema->set_transaction_isolation('READ COMMITTED');
+    TX_START:
+    $c->clear_errors;
+    try {
         my $guard = $schema->txn_scope_guard;
         {
             my $subscribers_rs = $self->item_rs($c);
@@ -367,15 +370,26 @@ sub GET :Allow {
             $c->response->body($response->content);
             return;
         }
+    } catch($e) {
+        if ($self->check_deadlock($c, $e)) {
+            goto TX_START;
+        }
+        unless ($c->has_errors) {
+            $self->error($c, HTTP_INTERNAL_SERVER_ERROR, 'Internal Server Error', $e);
+            last;
+        }
+    }
     return;
 }
 
 sub POST :Allow {
     my ($self, $c) = @_;
 
-
     my $schema = $c->model('DB');
     $schema->set_transaction_isolation('READ COMMITTED');
+    TX_START:
+    $c->clear_errors;
+    try {
         my $guard = $schema->txn_scope_guard;
         {
             my $resource = $self->get_valid_post_data(
@@ -461,6 +475,9 @@ sub POST :Allow {
                 $self->error($c, HTTP_UNPROCESSABLE_ENTITY, $http_errors[0], $http_errors[1], $log_error);
                 last;
             } catch($e) {
+                if ($self->check_deadlock($c, $e)) {
+                    goto TX_START;
+                }
                 if (ref $error_info->{extended} eq 'HASH' && $error_info->{extended}->{response_code}) {
                     $self->error($c,
                                  $error_info->{extended}->{response_code},
@@ -487,6 +504,15 @@ sub POST :Allow {
             $c->response->header(Location => sprintf('%s%d', $self->dispatch_path, $subscriber->id));
             $c->response->body(q());
         }
+    } catch($e) {
+        if ($self->check_deadlock($c, $e)) {
+            goto TX_START;
+        }
+        unless ($c->has_errors) {
+            $self->error($c, HTTP_INTERNAL_SERVER_ERROR, 'Internal Server Error', $e);
+            last;
+        }
+    }
     return;
 }
 1;

lib/NGCP/Panel/Controller/API/SubscribersItem.pm

@@ -39,6 +39,9 @@ sub journal_query_params {
 sub GET :Allow {
     my ($self, $c, $id) = @_;
     $c->model('DB')->set_transaction_isolation('READ COMMITTED');
+    TX_START:
+    $c->clear_errors;
+    try {
         my $guard = $c->model('DB')->txn_scope_guard;
         {
             last unless $self->valid_id($c, $id);
@@ -65,6 +68,15 @@ sub GET :Allow {
             $c->response->body($response->content);
             return;
         }
+    } catch($e) {
+        if ($self->check_deadlock($c, $e)) {
+            goto TX_START;
+        }
+        unless ($c->has_errors) {
+            $self->error($c, HTTP_INTERNAL_SERVER_ERROR, 'Internal Server Error', $e);
+            last;
+        }
+    }
     return;
 }
 
@@ -75,6 +87,9 @@ sub PUT :Allow {
 
     my $schema = $c->model('DB');
     $schema->set_transaction_isolation('READ COMMITTED');
+    TX_START:
+    $c->clear_errors;
+    try {
         my $guard = $schema->txn_scope_guard;
         {
             my $preference = $self->require_preference($c);
@@ -108,6 +123,15 @@ sub PUT :Allow {
             $guard->commit;
             $self->return_representation($c, 'hal' => $hal, 'preference' => $preference );
         }
+    } catch($e) {
+        if ($self->check_deadlock($c, $e)) {
+            goto TX_START;
+        }
+        unless ($c->has_errors) {
+            $self->error($c, HTTP_INTERNAL_SERVER_ERROR, 'Internal Server Error', $e);
+            last;
+        }
+    }
     return;
 }
 
@@ -118,6 +142,9 @@ sub PATCH :Allow {
 
     my $schema = $c->model('DB');
     $schema->set_transaction_isolation('READ COMMITTED');
+    TX_START:
+    $c->clear_errors;
+    try {
         my $guard = $schema->txn_scope_guard;
         {
             my $preference = $self->require_preference($c);
@@ -159,6 +186,15 @@ sub PATCH :Allow {
             $guard->commit;
             $self->return_representation($c, 'hal' => $hal, 'preference' => $preference );
         }
+    } catch($e) {
+        if ($self->check_deadlock($c, $e)) {
+            goto TX_START;
+        }
+        unless ($c->has_errors) {
+            $self->error($c, HTTP_INTERNAL_SERVER_ERROR, 'Internal Server Error', $e);
+            last;
+        }
+    }
     return;
 }
 

lib/NGCP/Panel/Role/API.pm

@@ -2207,17 +2207,20 @@ sub check_deadlock {
 
     return 0 unless $error;
 
-    #my $lock_retry = $error =~ /Lock wait timeout exceeded; try restarting transaction/;
+    my $lockwait_retry = $error =~ /Lock wait timeout exceeded; try restarting transaction/;
     my $deadlock_retry = $error =~ /Deadlock found when trying to get lock; try restarting transaction/;
 
-    return 0 unless $deadlock_retry;
+    return 0 unless $deadlock_retry or $lockwait_retry;
 
     my $attempt = $c->stash->{deadlock_retry_attempt} //= 1;
+    my $lockwait_err = "lock timeout detected, retry transaction attempt=$attempt/$max_attempts";
+    my $deadlock_err = "deadlock detected, retry transaction attempt=$attempt/$max_attempts";
+
     return 0 if $attempt > $max_attempts;
     NGCP::Panel::Utils::Message::info(
         c    => $c,
         type => 'api_retry',
-        log  => "deadlock detected, retry transaction attempt=$attempt/$max_attempts",
+        log  => ($lockwait_retry and $lockwait_err or $deadlock_err),
     );
     $c->stash->{deadlock_retry_attempt} = $attempt+1;
     return 1;