TT#77452 properly synchronize subscriber create/update

acquire the billing.contract row lock *before* any
unordered billing.voip_numbers rowlocks by
sub manage_subscriber_numbers().

- "deadlock" waittimeout errors will cease when
  creating subscribers concurrently via api
- max_subscribers, is_pilot and other per-contract
  constraints will be respected accurately

Change-Id: I73bb7525b327bbb09217b790be9c14cc65ddebcc
changes/48/38748/7
Rene Krenn 6 years ago
parent c753a285e6
commit a11f2bed5e

@ -9,6 +9,7 @@ use Data::HAL::Link qw();
use HTTP::Headers qw();
use HTTP::Status qw(:constants);
use NGCP::Panel::Utils::Contract qw();
use NGCP::Panel::Utils::ProfilePackages qw();
use NGCP::Panel::Utils::DateTime;
@ -98,7 +99,7 @@ sub GET :Allow {
my $now = NGCP::Panel::Utils::DateTime::current_local;
my $contracts_rs = $self->item_rs($c,0,$now);
(my $total_count, $contracts_rs, my $contracts_rows) = $self->paginate_order_collection($c, $contracts_rs);
my $contracts = NGCP::Panel::Utils::ProfilePackages::lock_contracts(c => $c,
my $contracts = NGCP::Panel::Utils::Contract::rowlock_contracts(c => $c,
rs => $contracts_rs,
contract_id_field => 'id');
my (@embedded, @links);
@ -134,7 +135,7 @@ sub GET :Allow {
$hal->resource({
total_count => $total_count,
});
my $response = HTTP::Response->new(HTTP_OK, undef,
my $response = HTTP::Response->new(HTTP_OK, undef,
HTTP::Headers->new($hal->http_headers(skip_links => 1)), $hal->as_json);
$c->response->headers($response->headers);
$c->response->body($response->content);

@ -88,7 +88,7 @@ sub GET :Allow {
my $now = NGCP::Panel::Utils::DateTime::current_local;
my $contracts_rs = $self->item_rs($c,0,$now);
(my $total_count, $contracts_rs, my $contracts_rows) = $self->paginate_order_collection($c, $contracts_rs);
my $contracts = NGCP::Panel::Utils::ProfilePackages::lock_contracts(c => $c,
my $contracts = NGCP::Panel::Utils::Contract::rowlock_contracts(c => $c,
rs => $contracts_rs,
contract_id_field => 'id');
my (@embedded, @links);

@ -10,7 +10,7 @@ use HTTP::Headers qw();
use HTTP::Status qw(:constants);
use NGCP::Panel::Utils::DateTime;
use NGCP::Panel::Utils::ProfilePackages qw();
use NGCP::Panel::Utils::Contract qw();
sub allowed_methods{
return [qw/GET OPTIONS HEAD/];
@ -130,7 +130,7 @@ sub GET :Allow {
my $now = NGCP::Panel::Utils::DateTime::current_local;
my $items_rs = $self->item_rs($c,0,$now);
(my $total_count, $items_rs, my $items_rows) = $self->paginate_order_collection($c, $items_rs);
my $items = NGCP::Panel::Utils::ProfilePackages::lock_contracts(c => $c,
my $items = NGCP::Panel::Utils::Contract::rowlock_contracts(c => $c,
rs => $items_rs,
contract_id_field => 'id');
my (@embedded, @links);

@ -160,7 +160,7 @@ sub GET :Allow {
my $now = NGCP::Panel::Utils::DateTime::current_local;
my $customers_rs = $self->item_rs($c,$now);
(my $total_count, $customers_rs, my $customers_rows) = $self->paginate_order_collection($c, $customers_rs);
my $customers = NGCP::Panel::Utils::ProfilePackages::lock_contracts(c => $c,
my $customers = NGCP::Panel::Utils::Contract::rowlock_contracts(c => $c,
rs => $customers_rs,
contract_id_field => 'id');
my (@embedded, @links);

@ -13,6 +13,7 @@ use HTTP::Status qw(:constants);
use JSON qw();
use NGCP::Panel::Utils::DateTime;
use NGCP::Panel::Utils::Contract qw();
use NGCP::Panel::Utils::ProfilePackages qw();
__PACKAGE__->set_config({
@ -91,9 +92,9 @@ sub GET :Allow {
{
my $subscribers_rs = $self->item_rs($c, "subscribers");
(my $total_count, $subscribers_rs, my $subscribers_rows) = $self->paginate_order_collection($c, $subscribers_rs);
my $subscribers = NGCP::Panel::Utils::ProfilePackages::lock_contracts(c => $c,
my $subscribers = NGCP::Panel::Utils::Contract::rowlock_contracts(c => $c,
rs => $subscribers_rs,
contract_id_field => 'contract_id');
contract_id_field => 'contract_id');
my $now = NGCP::Panel::Utils::DateTime::current_local;
my (@embedded, @links, %contract_map);
for my $subscriber (@$subscribers) {
@ -127,7 +128,7 @@ sub GET :Allow {
$hal->resource({
total_count => $total_count,
});
my $response = HTTP::Response->new(HTTP_OK, undef,
my $response = HTTP::Response->new(HTTP_OK, undef,
HTTP::Headers->new($hal->http_headers(skip_links => 1)), $hal->as_json);
$c->response->headers($response->headers);
$c->response->body($response->content);

@ -13,6 +13,7 @@ use HTTP::Status qw(:constants);
use NGCP::Panel::Utils::DateTime;
use NGCP::Panel::Utils::Subscriber;
use NGCP::Panel::Utils::Preferences;
use NGCP::Panel::Utils::Contract qw();
use NGCP::Panel::Utils::ProfilePackages qw();
use NGCP::Panel::Utils::Events qw();
use UUID;
@ -289,7 +290,7 @@ sub GET :Allow {
{
my $subscribers_rs = $self->item_rs($c);
(my $total_count, $subscribers_rs, my $subscribers_rows) = $self->paginate_order_collection($c, $subscribers_rs);
my $subscribers = NGCP::Panel::Utils::ProfilePackages::lock_contracts(c => $c,
my $subscribers = NGCP::Panel::Utils::Contract::rowlock_contracts(c => $c,
rs => $subscribers_rs,
contract_id_field => 'contract_id');
my $now = NGCP::Panel::Utils::DateTime::current_local;

@ -813,6 +813,10 @@ sub subscriber_create :Chained('base') :PathPart('subscriber/create') :Args(0) {
my $schema = $c->model('DB');
$schema->set_transaction_isolation('READ COMMITTED');
$schema->txn_do(sub {
NGCP::Panel::Utils::Contract::rowlock_contracts(
schema => $schema, contract_id => $c->stash->{contract}->id) if $c->stash->{contract};
my $preferences = {};
my $pbx_group_ids = [];
if($pbx && !$pbxadmin) {
@ -1403,6 +1407,10 @@ sub pbx_group_create :Chained('base') :PathPart('pbx/group/create') :Args(0) {
my $schema = $c->model('DB');
$schema->set_transaction_isolation('READ COMMITTED');
$schema->txn_do( sub {
NGCP::Panel::Utils::Contract::rowlock_contracts(
schema => $schema, contract_id => $c->stash->{contract}->id) if $c->stash->{contract};
my $preferences = {};
my $base_number = $pilot->primary_number;

@ -2853,6 +2853,9 @@ sub edit_master :Chained('master') :PathPart('edit') :Args(0) :Does(ACL) :ACLDet
$schema->set_transaction_isolation('READ COMMITTED');
$schema->txn_do(sub {
NGCP::Panel::Utils::Contract::rowlock_contracts(
schema => $schema, contract_id => $subscriber->contract->id);
my $email = delete $form->params->{email} || undef;
my $timezone = delete $form->values->{timezone}{name} || undef;
if ($subscriber->contact) {

@ -17,6 +17,7 @@ use NGCP::Panel::Utils::Prosody;
use NGCP::Panel::Utils::Subscriber;
use NGCP::Panel::Utils::Events;
use NGCP::Panel::Utils::DateTime;
use NGCP::Panel::Utils::Contract qw();
sub resource_name{
return 'subscribers';
@ -325,7 +326,10 @@ sub prepare_resource {
},
getcustomer_code => sub {
my ($cid) = @_;
return $self->get_customer($c, $cid);
my $contract = $self->get_customer($c, $cid);
NGCP::Panel::Utils::Contract::acquire_contract_rowlocks(
schema => $c->model('DB'), contract_id => $contract->id) if $contract;
return $contract;
},
);

@ -5,7 +5,6 @@ use warnings;
use Sipwise::Base;
use DBIx::Class::Exception;
use NGCP::Panel::Utils::DateTime;
#use DateTime::Format::Strptime qw();
use NGCP::Panel::Utils::CallList qw();
sub recursively_lock_contract {
@ -341,6 +340,58 @@ sub is_peering_reseller_product {
}
return 0;
}
sub acquire_contract_rowlocks {
my %params = @_;
my($c,$schema,$rs,$contract_id_field,$contract_ids,$contract_id) = @params{qw/c schema rs contract_id_field contract_ids contract_id/};
$schema //= $c->model('DB');
my %contract_id_map = ();
my $rs_result = undef;
if (defined $rs and defined $contract_id_field) {
$rs_result = [ $rs->all ];
foreach my $item (@$rs_result) {
$contract_id_map{$item->$contract_id_field} = 1;
}
}
if (defined $contract_ids) {
foreach my $id (@$contract_ids) {
$contract_id_map{$id} = 1;
}
}
if (defined $contract_id) {
$contract_id_map{$contract_id} = 1;
}
my @contract_ids_to_lock = keys %contract_id_map;
my ($t1,$t2) = (time,undef);
if (defined $contract_id && !defined $rs_result && !defined $contract_ids) {
$c->log->debug('contract ID to be locked: ' . $contract_id) if $c;
my $contract = $schema->resultset('contracts')->find({
id => $contract_id
},{for => 'update'});
$t2 = time;
$c->log->debug('contract ID ' . $contract_id . ' locked (' . ($t2 - $t1) . ' secs)') if $c;
return $contract;
} elsif ((scalar @contract_ids_to_lock) > 0) {
@contract_ids_to_lock = sort { $a <=> $b } @contract_ids_to_lock; #"Access your tables and rows in a fixed order."
my $contract_ids_label = join(', ',@contract_ids_to_lock);
$c->log->debug('contract IDs to be locked: ' . $contract_ids_label) if $c;
my @contracts = $schema->resultset('contracts')->search({
id => { -in => [ @contract_ids_to_lock ] }
},{for => 'update'})->all;
$t2 = time;
$c->log->debug('contract IDs ' . $contract_ids_label . ' locked (' . ($t2 - $t1) . ' secs)') if $c;
if (defined $contract_ids || defined $contract_id) {
return [ @contracts ];
} else {
return $rs_result;
}
}
$c->log->debug('no contract IDs to be locked!') if $c;
return [];
}
1;
__END__

@ -5,6 +5,7 @@ use Scalar::Util qw(looks_like_number);
#use TryCatch;
use NGCP::Panel::Utils::DateTime qw();
use NGCP::Panel::Utils::Contract qw();
use NGCP::Panel::Utils::Subscriber qw();
use NGCP::Panel::Utils::BillingMappings qw();
use Data::Dumper;
@ -76,7 +77,7 @@ sub resize_actual_contract_balance {
my($c,$contract,$old_package,$actual_balance,$is_topup,$topup_amount,$now,$schema,$profiles_added) = @params{qw/c contract old_package balance is_topup topup_amount now schema profiles_added/};
$schema //= $c->model('DB');
$contract = lock_contracts(schema => $schema, contract_id => $contract->id);
$contract = NGCP::Panel::Utils::Contract::acquire_contract_rowlocks(schema => $schema, contract_id => $contract->id);
$is_topup //= 0;
$topup_amount //= 0.0;
$profiles_added //= 0;
@ -216,7 +217,7 @@ sub catchup_contract_balances {
my($c,$contract,$old_package,$now,$suppress_underrun,$is_create_next,$last_notopup_discard_intervals,$last_carry_over_mode,$topup_amount,$profiles_added) = @params{qw/c contract old_package now suppress_underrun is_create_next last_notopup_discard_intervals last_carry_over_mode topup_amount profiles_added/};
my $schema = $c->model('DB');
$contract = lock_contracts(schema => $schema, contract_id => $contract->id);
$contract = NGCP::Panel::Utils::Contract::acquire_contract_rowlocks(schema => $schema, contract_id => $contract->id);
$now //= NGCP::Panel::Utils::DateTime::set_local_tz($contract->modify_timestamp);
$old_package = $contract->profile_package if !exists $params{old_package};
my $contract_create = NGCP::Panel::Utils::DateTime::set_local_tz($contract->create_timestamp // $contract->modify_timestamp);
@ -439,7 +440,7 @@ sub topup_contract_balance {
my($c,$contract,$package,$voucher,$amount,$now,$request_token,$schema,$log_vals,$subscriber) = @params{qw/c contract package voucher amount now request_token schema log_vals subscriber/};
$schema //= $c->model('DB');
$contract = lock_contracts(schema => $schema, contract_id => $contract->id);
$contract = NGCP::Panel::Utils::Contract::acquire_contract_rowlocks(schema => $schema, contract_id => $contract->id);
$now //= NGCP::Panel::Utils::DateTime::current_local;
my $voucher_package = ($voucher ? $voucher->profile_package : $package);
@ -584,7 +585,7 @@ sub create_initial_contract_balances {
my($c,$contract,$now) = @params{qw/c contract now/};
my $schema = $c->model('DB');
$contract = lock_contracts(schema => $schema, contract_id => $contract->id);
$contract = NGCP::Panel::Utils::Contract::acquire_contract_rowlocks(schema => $schema, contract_id => $contract->id);
$now //= NGCP::Panel::Utils::DateTime::set_local_tz($contract->create_timestamp // $contract->modify_timestamp);
my ($start_mode,$interval_unit,$interval_value,$initial_balance,$underrun_profile_threshold,$underrun_lock_threshold);
@ -1079,57 +1080,6 @@ sub add_profile_mappings {
return 0;
}
sub lock_contracts {
my %params = @_;
my($c,$schema,$rs,$contract_id_field,$contract_ids,$contract_id) = @params{qw/c schema rs contract_id_field contract_ids contract_id/};
$schema //= $c->model('DB');
my %contract_id_map = ();
my $rs_result = undef;
if (defined $rs and defined $contract_id_field) {
$rs_result = [ $rs->all ];
foreach my $item (@$rs_result) {
$contract_id_map{$item->$contract_id_field} = 1;
}
}
if (defined $contract_ids) {
foreach my $id (@$contract_ids) {
$contract_id_map{$id} = 1;
}
}
if (defined $contract_id) {
$contract_id_map{$contract_id} = 1;
}
my @contract_ids_to_lock = keys %contract_id_map;
my ($t1,$t2) = (time,undef);
if (defined $contract_id && !defined $rs_result && !defined $contract_ids) {
$c->log->debug('contract ID to be locked: ' . $contract_id) if $c;
my $contract = $schema->resultset('contracts')->find({
id => $contract_id
},{for => 'update'});
$t2 = time;
$c->log->debug('contract ID ' . $contract_id . ' locked (' . ($t2 - $t1) . ' secs)') if $c;
return $contract;
} elsif ((scalar @contract_ids_to_lock) > 0) {
@contract_ids_to_lock = sort { $a <=> $b } @contract_ids_to_lock; #"Access your tables and rows in a fixed order."
my $contract_ids_label = join(', ',@contract_ids_to_lock);
$c->log->debug('contract IDs to be locked: ' . $contract_ids_label) if $c;
my @contracts = $schema->resultset('contracts')->search({
id => { -in => [ @contract_ids_to_lock ] }
},{for => 'update'})->all;
$t2 = time;
$c->log->debug('contract IDs ' . $contract_ids_label . ' locked (' . ($t2 - $t1) . ' secs)') if $c;
if (defined $contract_ids || defined $contract_id) {
return [ @contracts ];
} else {
return $rs_result;
}
}
$c->log->debug('no contract IDs to be locked!') if $c;
return [];
}
sub _dump_contract_balance {
my $balance = shift;
my $row = { $balance->get_inflated_columns };

@ -22,6 +22,7 @@ $Data::Dumper::Sortkeys = sub {
use Text::CSV_XS qw();
use NGCP::Panel::Utils::DateTime qw();
use NGCP::Panel::Utils::BillingMappings qw();
use NGCP::Panel::Utils::Contract qw();
use NGCP::Panel::Utils::ProfilePackages qw();
use NGCP::Panel::Utils::Subscriber qw();
use NGCP::Panel::Utils::Preferences qw();
@ -784,7 +785,10 @@ sub _init_subscriber_context {
},
getcustomer_code => sub {
my ($cid) = @_;
return $schema->resultset('contracts')->find($cid);
my $contract = $schema->resultset('contracts')->find($cid);
NGCP::Panel::Utils::Contract::acquire_contract_rowlocks(
schema => $schema, contract_id => $contract->id) if $contract;
return $contract;
},
);

@ -15,6 +15,7 @@ use NGCP::Panel::Utils::Events;
use NGCP::Panel::Utils::DateTime qw();
use NGCP::Panel::Utils::License;
use NGCP::Panel::Utils::Generic;
use NGCP::Panel::Utils::Contract;
use NGCP::Panel::Utils::RedisLocationResultSet;
use UUID qw/generate unparse/;
use JSON qw/decode_json encode_json/;
@ -1634,6 +1635,9 @@ sub terminate {
my $schema = $c->model('DB');
$schema->txn_do(sub {
NGCP::Panel::Utils::Contract::acquire_contract_rowlocks(
schema => $schema, contract_id => $subscriber->contract->id);
my $prov_subscriber = $subscriber->provisioning_voip_subscriber;
my @events_to_create = ();

Loading…
Cancel
Save