From 60203fabaaf60bf17b7357de99fb604991f80122 Mon Sep 17 00:00:00 2001 From: Kirill Solomko Date: Fri, 5 Dec 2025 15:43:54 +0100 Subject: [PATCH] MT#64067 address deadlock detection for preferences * improve Role/API/Preferences err_code callback to check deadlocks (via $self->check_deadlock) and set $process_extras->{retry_tx} = 1 so that the outer caller can act upon it and restart the transaction. * increase deadlock attempts from 2 to 5 as some deadlocks seem to take longer time to resolve. * wait 1 second between deadlock retry attempts as otherwise if a deadlock takes more than 1 second and all the retries are done within less than a second it will cause an error. * EntitiesItem: add support for retry_tx and call goto TX_START if $process_extras->{retry_tx} (when check_deadlock is processed externally, like in Utils::Preferences::update_item in the err_code callback). Change-Id: Ib56383710041c21d72782047a852353296c22215 (cherry picked from commit eae620f204a751890cec633bad7995a157520cf0) (cherry picked from commit b3391957d40e11d2f95fd0fe40e8bdef3fc2e228) --- .../API/SubscriberPreferencesItem.pm | 8 +++-- lib/NGCP/Panel/Role/API.pm | 3 +- lib/NGCP/Panel/Role/API/Preferences.pm | 29 +++++++++++++++---- lib/NGCP/Panel/Role/EntitiesItem.pm | 6 ++-- lib/NGCP/Panel/Utils/Preferences.pm | 14 ++++----- 5 files changed, 42 insertions(+), 18 deletions(-) diff --git a/lib/NGCP/Panel/Controller/API/SubscriberPreferencesItem.pm b/lib/NGCP/Panel/Controller/API/SubscriberPreferencesItem.pm index d47e3c8f64..2389b03f24 100644 --- a/lib/NGCP/Panel/Controller/API/SubscriberPreferencesItem.pm +++ b/lib/NGCP/Panel/Controller/API/SubscriberPreferencesItem.pm @@ -120,7 +120,9 @@ sub PATCH :Allow { # last param is "no replace" to NOT delete existing prefs # for proper PATCH behavior - $subscriber = $self->update_item($c, $subscriber, $old_resource, $resource, 0, "subscribers"); + my $process_extras; + ($subscriber, undef, $process_extras) = $self->update_item($c, $subscriber, $old_resource, $resource, undef, $process_extras); + goto TX_START if $process_extras->{retry_tx}; last unless $subscriber; my $hal = $self->hal_from_item($c, $subscriber, "subscribers"); @@ -181,7 +183,9 @@ sub PUT :Allow { # last param is "replace" to delete all existing prefs # for proper PUT behavior - $subscriber = $self->update_item($c, $subscriber, $old_resource, $resource, 1, "subscribers"); + my $process_extras; + $subscriber = $self->update_item($c, $subscriber, $old_resource, $resource, undef, $process_extras); + goto TX_START if $process_extras->{retry_tx}; last unless $subscriber; my $hal = $self->hal_from_item($c, $subscriber, "subscribers"); diff --git a/lib/NGCP/Panel/Role/API.pm b/lib/NGCP/Panel/Role/API.pm index 640230dc37..9336da62e7 100644 --- a/lib/NGCP/Panel/Role/API.pm +++ b/lib/NGCP/Panel/Role/API.pm @@ -2204,7 +2204,7 @@ sub check_wildcard_search { sub check_deadlock { my ($self, $c, $error) = @_; - my $max_attempts = 2; + my $max_attempts = 5; return 0 unless $error; @@ -2224,6 +2224,7 @@ sub check_deadlock { log => ($lockwait_retry and $lockwait_err or $deadlock_err), ); $c->stash->{deadlock_retry_attempt} = $attempt+1; + sleep 1; return 1; } diff --git a/lib/NGCP/Panel/Role/API/Preferences.pm b/lib/NGCP/Panel/Role/API/Preferences.pm index 9341fdc438..cee3b9da4f 100644 --- a/lib/NGCP/Panel/Role/API/Preferences.pm +++ b/lib/NGCP/Panel/Role/API/Preferences.pm @@ -208,10 +208,28 @@ sub _item_rs { } sub update_item { + my ($self, $c, $item, $old_resource, $resource, $form, $process_extras) = @_; - my ($self, $c, $item, $old_resource, $resource) = @_; + $process_extras //= {}; - return NGCP::Panel::Utils::Preferences::update_preferences( + if (ref $process_extras eq 'HASH') { + $process_extras->{retry_tx} = 0; + } + + my $err_cb = sub { + my ($code, $msg, $err) = @_; + if ($err && $self->check_deadlock($c, $err)) { + if (ref $process_extras eq 'HASH') { + $process_extras->{retry_tx} = 1; + } + return; + } + unless ($c->has_errors) { + $self->error($c, $code, $msg, $err); + } + }; + + $item = NGCP::Panel::Utils::Preferences::update_preferences( c => $c, schema => $c->model('DB'), item => $item, @@ -219,12 +237,11 @@ sub update_item { resource => $resource, type => $self->container_resource_type, replace => uc($c->request->method) eq 'PUT', - err_code => sub { - my ($code, $msg) = @_; - $self->error($c, $code, $msg); - }, + err_code => $err_cb, ); + return $item, $form, $process_extras; + } 1; diff --git a/lib/NGCP/Panel/Role/EntitiesItem.pm b/lib/NGCP/Panel/Role/EntitiesItem.pm index 9e4ba3a9b6..0fc3ce10a5 100644 --- a/lib/NGCP/Panel/Role/EntitiesItem.pm +++ b/lib/NGCP/Panel/Role/EntitiesItem.pm @@ -306,7 +306,8 @@ sub patch { goto TX_END; } - ($item, $form, $process_extras) = $self->update_item($c, $item, $old_resource, $resource, $form, $process_extras ); + ($item, $form, $process_extras) = $self->update_item($c, $item, $old_resource, $resource, $form, $process_extras); + goto TX_START if ref $process_extras eq 'HASH' && $process_extras->{retry_tx}; if ($c->has_errors) { goto TX_END; } @@ -403,7 +404,8 @@ sub put { my ($data_processed_result); if (!$non_json_data || !$data) { - ($item, $form, $process_extras) = $self->update_item($c, $item, $old_resource, $resource, $form, $process_extras ); + ($item, $form, $process_extras) = $self->update_item($c, $item, $old_resource, $resource, $form, $process_extras); + goto TX_START if ref $process_extras eq 'HASH' && $process_extras->{retry_tx}; if ($c->has_errors) { goto TX_END; } diff --git a/lib/NGCP/Panel/Utils/Preferences.pm b/lib/NGCP/Panel/Utils/Preferences.pm index ef4be09e2a..cf7f4b822b 100644 --- a/lib/NGCP/Panel/Utils/Preferences.pm +++ b/lib/NGCP/Panel/Utils/Preferences.pm @@ -591,7 +591,7 @@ sub update_preferences { $full_rs->delete_all; } catch($e) { $c->log->error("failed to clear preferences for '$accessor': $e"); - &$err_code(HTTP_INTERNAL_SERVER_ERROR, "Internal Server Error."); + &$err_code(HTTP_INTERNAL_SERVER_ERROR, "Internal Server Error.", $e); return; }; } else { @@ -693,7 +693,7 @@ sub update_preferences { } } catch($e) { $c->log->error("failed to clear preference for '$accessor': $e"); - &$err_code(HTTP_INTERNAL_SERVER_ERROR, "Internal Server Error."); + &$err_code(HTTP_INTERNAL_SERVER_ERROR, "Internal Server Error.", $e); return; }; } @@ -1015,7 +1015,7 @@ sub update_preferences { }; if ($@) { $c->log->error("Failed to transform pref value - $@"); - &$err_code(HTTP_INTERNAL_SERVER_ERROR, "Internal Server Error."); # TODO? + &$err_code(HTTP_INTERNAL_SERVER_ERROR, "Internal Server Error.", $@); # TODO? return; } if(JSON::is_bool($v)){ @@ -1030,7 +1030,7 @@ sub update_preferences { }; if ($@) { $c->log->error("Failed to transform pref value - $@"); - &$err_code(HTTP_INTERNAL_SERVER_ERROR, "Internal Server Error."); # TODO? + &$err_code(HTTP_INTERNAL_SERVER_ERROR, "Internal Server Error.", $@); # TODO? return; } if(JSON::is_bool($resource->{$pref})){ @@ -1044,7 +1044,7 @@ sub update_preferences { }; if ($@) { $c->log->error("Failed to transform pref value - $@"); - &$err_code(HTTP_INTERNAL_SERVER_ERROR, "Internal Server Error."); # TODO? + &$err_code(HTTP_INTERNAL_SERVER_ERROR, "Internal Server Error.", $@); # TODO? return; } if(JSON::is_bool($resource->{$pref})){ @@ -1061,7 +1061,7 @@ sub update_preferences { } } catch($e) { $c->log->error("failed to update preference for '$accessor': $e"); - &$err_code(HTTP_INTERNAL_SERVER_ERROR, "Internal Server Error."); + &$err_code(HTTP_INTERNAL_SERVER_ERROR, "Internal Server Error.", $e); return; } } @@ -1081,7 +1081,7 @@ sub update_preferences { ); } catch($e) { $c->log->error("Failed to set peer registration: $e"); - &$err_code(HTTP_INTERNAL_SERVER_ERROR, "Internal Server Error."); # TODO? + &$err_code(HTTP_INTERNAL_SERVER_ERROR, "Internal Server Error.", $e); # TODO? return; } }