From 5a07957ff330c9d9fa754081f0bf1e6679162c03 Mon Sep 17 00:00:00 2001 From: Rene Krenn Date: Fri, 20 Oct 2017 15:02:46 +0200 Subject: [PATCH] TT#23594 ppa and modifications + fixed block iterator counts + removed voip_cf_destinations.announcement_id from trunk dao + prevent "mysql gone away" error: - xa_db connection for ncos level lookups - *reverted* - set net_read_timeout to a higher value than default 30 secs - implement sql connection pinger for managed conenctions + fix processor thread wrong rowcount + write trusted sources for permanent registration's contact IP addresses + write a default ncos level for subscribers without barring + set mysql connection encoding to utf8 + create multi-subscriber contracts properly + properly skip subscribers with non-unique contracts + generate sip password if empty Change-Id: I533149a2d6328a0aa1f20ab7eab795ffff720020 --- lib/NGCP/BulkProcessor/ConnectorPool.pm | 25 +++ .../Dao/Trunk/billing/contacts.pm | 23 +++ .../Dao/Trunk/billing/contracts.pm | 18 ++ .../provisioning/voip_cf_destinations.pm | 2 +- .../provisioning/voip_trusted_sources.pm | 180 ++++++++++++++++++ .../Projects/Migration/Teletek/Check.pm | 4 + .../Teletek/Dao/import/AllowedCli.pm | 2 +- .../Teletek/Dao/import/Subscriber.pm | 2 +- .../Migration/Teletek/ProjectConnectorPool.pm | 12 ++ .../Migration/Teletek/Provisioning.pm | 119 ++++++++++-- .../Projects/Migration/Teletek/Settings.pm | 7 + .../Migration/Teletek/barring_profiles.yml | 7 +- .../Projects/Migration/Teletek/process.pl | 6 +- .../Projects/Migration/Teletek/settings.cfg | 1 + lib/NGCP/BulkProcessor/SqlConnector.pm | 9 + .../BulkProcessor/SqlConnectors/MySQLDB.pm | 16 +- 16 files changed, 412 insertions(+), 21 deletions(-) create mode 100644 lib/NGCP/BulkProcessor/Dao/Trunk/provisioning/voip_trusted_sources.pm diff --git a/lib/NGCP/BulkProcessor/ConnectorPool.pm b/lib/NGCP/BulkProcessor/ConnectorPool.pm index c8ba797..c7c9aea 100644 --- a/lib/NGCP/BulkProcessor/ConnectorPool.pm +++ b/lib/NGCP/BulkProcessor/ConnectorPool.pm @@ -89,6 +89,9 @@ our @EXPORT_OK = qw( destroy_dbs get_connectorinstancename get_cluster_db + + ping_dbs + ping ); my $connectorinstancenameseparator = '_'; @@ -253,6 +256,28 @@ sub get_connectorinstancename { return $instance_name; } +sub ping_dbs { + ping($accounting_dbs); + ping($billing_dbs); + ping($provisioning_dbs); + ping($kamailio_dbs); + ping($xa_dbs); +} + +sub ping { + my $dbs = shift; + my $this_tid = threadid(); + foreach my $instance_name (keys %$dbs) { + my ($tid,$name) = split(quotemeta($connectorinstancenameseparator),$instance_name,2); + next unless ($this_tid == $tid and defined $dbs->{$instance_name}); + my $result = 0; + eval { + $result = $dbs->{$instance_name}->ping(); + }; + undef $dbs->{$instance_name} unless $result; + } +} + sub destroy_dbs { diff --git a/lib/NGCP/BulkProcessor/Dao/Trunk/billing/contacts.pm b/lib/NGCP/BulkProcessor/Dao/Trunk/billing/contacts.pm index 37c81ab..69d519e 100644 --- a/lib/NGCP/BulkProcessor/Dao/Trunk/billing/contacts.pm +++ b/lib/NGCP/BulkProcessor/Dao/Trunk/billing/contacts.pm @@ -27,6 +27,8 @@ our @EXPORT_OK = qw( check_table insert_row update_row + + findby_reselleridfields ); my $tablename = 'contacts'; @@ -83,6 +85,27 @@ sub new { } +sub findby_reselleridfields { + + my ($reseller_id,$fields,$load_recursive) = @_; + + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + + my $stmt = 'SELECT * FROM ' . $table . ' WHERE ' . + $db->columnidentifier('reseller_id') . ' = ?'; + my @params = ($reseller_id); + foreach my $field (keys %$fields) { + $stmt .= ' AND ' . $db->columnidentifier($field) . ' = ?'; + push(@params,$fields->{$field}); + } + my $rows = $db->db_get_all_arrayref($stmt,@params); + + return buildrecords_fromrows($rows,$load_recursive); + +} + sub update_row { my ($xa_db,$data) = @_; diff --git a/lib/NGCP/BulkProcessor/Dao/Trunk/billing/contracts.pm b/lib/NGCP/BulkProcessor/Dao/Trunk/billing/contracts.pm index a6624d4..f3fc8c5 100644 --- a/lib/NGCP/BulkProcessor/Dao/Trunk/billing/contracts.pm +++ b/lib/NGCP/BulkProcessor/Dao/Trunk/billing/contracts.pm @@ -30,6 +30,7 @@ our @EXPORT_OK = qw( insert_row countby_status_resellerid + findby_contactid process_records @@ -81,6 +82,23 @@ sub new { } +sub findby_contactid { + + my ($contact_id,$load_recursive) = @_; + + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + + my $stmt = 'SELECT * FROM ' . $table . ' WHERE ' . + $db->columnidentifier('contact_id') . ' = ?'; + my @params = ($contact_id); + my $rows = $db->db_get_all_arrayref($stmt,@params); + + return buildrecords_fromrows($rows,$load_recursive); + +} + sub countby_status_resellerid { my ($status,$reseller_id) = @_; diff --git a/lib/NGCP/BulkProcessor/Dao/Trunk/provisioning/voip_cf_destinations.pm b/lib/NGCP/BulkProcessor/Dao/Trunk/provisioning/voip_cf_destinations.pm index ee634ef..325abb0 100644 --- a/lib/NGCP/BulkProcessor/Dao/Trunk/provisioning/voip_cf_destinations.pm +++ b/lib/NGCP/BulkProcessor/Dao/Trunk/provisioning/voip_cf_destinations.pm @@ -34,7 +34,7 @@ my $expected_fieldnames = [ 'destination', 'priority', 'timeout', - 'announcement_id', + #'announcement_id', ]; my $indexes = {}; diff --git a/lib/NGCP/BulkProcessor/Dao/Trunk/provisioning/voip_trusted_sources.pm b/lib/NGCP/BulkProcessor/Dao/Trunk/provisioning/voip_trusted_sources.pm new file mode 100644 index 0000000..920c1b7 --- /dev/null +++ b/lib/NGCP/BulkProcessor/Dao/Trunk/provisioning/voip_trusted_sources.pm @@ -0,0 +1,180 @@ +package NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_trusted_sources; +use strict; + +## no critic + +use NGCP::BulkProcessor::Logging qw( + getlogger + rowinserted + rowsdeleted +); + +use NGCP::BulkProcessor::ConnectorPool qw( + get_provisioning_db +); + +use NGCP::BulkProcessor::SqlProcessor qw( + checktableinfo + insert_record + copy_row +); +use NGCP::BulkProcessor::SqlRecord qw(); + +require Exporter; +our @ISA = qw(Exporter NGCP::BulkProcessor::SqlRecord); +our @EXPORT_OK = qw( + gettablename + check_table + insert_row + + countby_subscriberid + + $PROTOCOL_UDP + $PROTOCOL_TCP + $PROTOCOL_TLS + $PROTOCOL_ANY +); + +my $tablename = 'voip_trusted_sources'; +my $get_db = \&get_provisioning_db; + +my $expected_fieldnames = [ + 'id', + 'subscriber_id', + 'src_ip', + 'protocol', + 'from_pattern', + 'uuid', + +]; + +my $indexes = {}; + +my $insert_unique_fields = []; + +our $PROTOCOL_UDP = 'UDP'; +our $PROTOCOL_TCP = 'TCP'; +our $PROTOCOL_TLS = 'TLS'; +our $PROTOCOL_ANY = 'ANY'; + +sub new { + + my $class = shift; + my $self = NGCP::BulkProcessor::SqlRecord->new($class,$get_db, + $tablename,$expected_fieldnames,$indexes); + + copy_row($self,shift,$expected_fieldnames); + + return $self; + +} + +sub countby_subscriberid { + + my ($subscriber_id) = @_; + + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + + my $stmt = 'SELECT COUNT(*) FROM ' . $table; + my @params = (); + my @terms = (); + if (defined $subscriber_id) { + push(@terms,$db->columnidentifier('subscriber_id') . ' = ?'); + push(@params,$subscriber_id); + } + if ((scalar @terms) > 0) { + $stmt .= ' WHERE ' . join(' AND ',@terms); + } + + return $db->db_get_value($stmt,@params); + +} + +sub insert_row { + + my $db = &$get_db(); + my $xa_db = shift // $db; + if ('HASH' eq ref $_[0]) { + my ($data,$insert_ignore) = @_; + check_table(); + if (insert_record($db,$xa_db,__PACKAGE__,$data,$insert_ignore,$insert_unique_fields)) { + return $xa_db->db_last_insert_id(); + } + } else { + my %params = @_; + my ($subscriber_id, + $src_ip, + $protocol, + $from_pattern, + $uuid) = @params{qw/ + subscriber_id + src_ip + protocol + from_pattern + uuid + /}; + + if ($xa_db->db_do('INSERT INTO ' . $db->tableidentifier($tablename) . ' (' . + $db->columnidentifier('subscriber_id') . ', ' . + $db->columnidentifier('src_ip') . ', ' . + $db->columnidentifier('protocol') . ', ' . + $db->columnidentifier('from_pattern') . ', ' . + $db->columnidentifier('uuid') . ') VALUES (' . + '?, ' . + '?, ' . + '?, ' . + '?, ' . + '?)', + $subscriber_id, + $src_ip, + $protocol, + $from_pattern, + $uuid, + )) { + rowinserted($db,$tablename,getlogger(__PACKAGE__)); + return $xa_db->db_last_insert_id(); + } + } + return undef; + +} + +sub buildrecords_fromrows { + + my ($rows,$load_recursive) = @_; + + my @records = (); + my $record; + + if (defined $rows and ref $rows eq 'ARRAY') { + foreach my $row (@$rows) { + $record = __PACKAGE__->new($row); + + # transformations go here ... + + push @records,$record; + } + } + + return \@records; + +} + +sub gettablename { + + return $tablename; + +} + +sub check_table { + + return checktableinfo($get_db, + __PACKAGE__,$tablename, + $expected_fieldnames, + $indexes); + +} + +1; diff --git a/lib/NGCP/BulkProcessor/Projects/Migration/Teletek/Check.pm b/lib/NGCP/BulkProcessor/Projects/Migration/Teletek/Check.pm index 04f8695..431da0d 100644 --- a/lib/NGCP/BulkProcessor/Projects/Migration/Teletek/Check.pm +++ b/lib/NGCP/BulkProcessor/Projects/Migration/Teletek/Check.pm @@ -26,6 +26,7 @@ use NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_dbaliases qw(); use NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_cf_mappings qw(); use NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_cf_destination_sets qw(); use NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_cf_destinations qw(); +use NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_trusted_sources qw(); use NGCP::BulkProcessor::Dao::Trunk::kamailio::voicemail_users qw(); use NGCP::BulkProcessor::Dao::Trunk::kamailio::location qw(); @@ -169,6 +170,9 @@ sub check_provisioning_db_tables { ($check_result,$message) = _check_table($message_prefix,'NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_cf_destinations'); $result &= $check_result; push(@$messages,$message); + ($check_result,$message) = _check_table($message_prefix,'NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_trusted_sources'); + $result &= $check_result; push(@$messages,$message); + return $result; } diff --git a/lib/NGCP/BulkProcessor/Projects/Migration/Teletek/Dao/import/AllowedCli.pm b/lib/NGCP/BulkProcessor/Projects/Migration/Teletek/Dao/import/AllowedCli.pm index 4e58fc0..91e7b4f 100644 --- a/lib/NGCP/BulkProcessor/Projects/Migration/Teletek/Dao/import/AllowedCli.pm +++ b/lib/NGCP/BulkProcessor/Projects/Migration/Teletek/Dao/import/AllowedCli.pm @@ -332,7 +332,7 @@ sub buildrecords_fromrows { # multithreading => $multithreading, # tableprocessing_threads => $numofthreads, # 'select' => 'SELECT ' . join(',',@cols) . ' FROM ' . $table . ' GROUP BY ' . join(',',@cols), -# 'select_count' => 'SELECT COUNT(DISTINCT(' . join(',',@cols) . ')) FROM ' . $table, +# 'selectcount' => 'SELECT COUNT(DISTINCT(' . join(',',@cols) . ')) FROM ' . $table, # ); #} diff --git a/lib/NGCP/BulkProcessor/Projects/Migration/Teletek/Dao/import/Subscriber.pm b/lib/NGCP/BulkProcessor/Projects/Migration/Teletek/Dao/import/Subscriber.pm index 450598f..157413f 100644 --- a/lib/NGCP/BulkProcessor/Projects/Migration/Teletek/Dao/import/Subscriber.pm +++ b/lib/NGCP/BulkProcessor/Projects/Migration/Teletek/Dao/import/Subscriber.pm @@ -433,7 +433,7 @@ sub process_records { numeric => 1, dir => 1, }]), - 'select_count' => 'SELECT COUNT(DISTINCT(' . join(',',@cols) . ')) FROM ' . $table, + 'selectcount' => 'SELECT COUNT(*) FROM (SELECT ' . join(',',@cols) . ' FROM ' . $table . ' GROUP BY ' . join(',',@cols) . ') AS g', ); } diff --git a/lib/NGCP/BulkProcessor/Projects/Migration/Teletek/ProjectConnectorPool.pm b/lib/NGCP/BulkProcessor/Projects/Migration/Teletek/ProjectConnectorPool.pm index 6e9e4f5..320f9a4 100644 --- a/lib/NGCP/BulkProcessor/Projects/Migration/Teletek/ProjectConnectorPool.pm +++ b/lib/NGCP/BulkProcessor/Projects/Migration/Teletek/ProjectConnectorPool.pm @@ -13,6 +13,7 @@ use NGCP::BulkProcessor::Projects::Migration::Teletek::Settings qw( use NGCP::BulkProcessor::ConnectorPool qw( get_connectorinstancename + ping ); #use NGCP::BulkProcessor::SqlConnectors::MySQLDB; @@ -36,6 +37,9 @@ our @EXPORT_OK = qw( destroy_dbs destroy_all_dbs + + ping_dbs + ping_all_dbs ); # thread connector pools: @@ -69,6 +73,14 @@ sub import_db_tableidentifier { } +sub ping_dbs { + ping($import_dbs); +} + +sub ping_all_dbs { + ping_dbs(); + NGCP::BulkProcessor::ConnectorPool::ping_dbs(); +} sub destroy_dbs { diff --git a/lib/NGCP/BulkProcessor/Projects/Migration/Teletek/Provisioning.pm b/lib/NGCP/BulkProcessor/Projects/Migration/Teletek/Provisioning.pm index cbba029..8dcd864 100644 --- a/lib/NGCP/BulkProcessor/Projects/Migration/Teletek/Provisioning.pm +++ b/lib/NGCP/BulkProcessor/Projects/Migration/Teletek/Provisioning.pm @@ -21,6 +21,7 @@ use NGCP::BulkProcessor::Projects::Migration::Teletek::Settings qw( $provision_subscriber_numofthreads $webpassword_length $webusername_length + $sippassword_length $default_channels_map $reseller_mapping @@ -70,6 +71,7 @@ use NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_usr_preferences qw(); use NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_cf_mappings qw(); use NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_cf_destination_sets qw(); use NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_cf_destinations qw(); +use NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_trusted_sources qw(); use NGCP::BulkProcessor::Dao::Trunk::kamailio::voicemail_users qw(); use NGCP::BulkProcessor::Dao::Trunk::kamailio::location qw(); @@ -92,6 +94,7 @@ use NGCP::BulkProcessor::ConnectorPool qw( use NGCP::BulkProcessor::Projects::Migration::Teletek::ProjectConnectorPool qw( destroy_all_dbs + ping_all_dbs ); use NGCP::BulkProcessor::Utils qw(create_uuid threadid timestamp stringtobool check_ipnet trim); @@ -118,9 +121,13 @@ my $cf_types_pattern = '^' . $NGCP::BulkProcessor::Dao::Trunk::provisioning::voi my $db_lock :shared = undef; my $file_lock :shared = undef; +my $default_barring = 'default'; + +my $contact_hash_field = 'gpp9'; + sub provision_subscribers { - my $static_context = { now => timestamp(), rowcount => undef }; + my $static_context = { now => timestamp(), _rowcount => undef }; my $result = _provision_subscribers_checks($static_context); destroy_all_dbs(); @@ -130,10 +137,11 @@ sub provision_subscribers { static_context => $static_context, process_code => sub { my ($context,$records,$row_offset) = @_; - $context->{rowcount} = $row_offset; + ping_all_dbs(); + $context->{_rowcount} = $row_offset; my @report_data = (); foreach my $domain_sipusername (@$records) { - $context->{rowcount} += 1; + $context->{_rowcount} += 1; next unless _provision_susbcriber($context, NGCP::BulkProcessor::Projects::Migration::Teletek::Dao::import::Subscriber::findby_domain_sipusername(@$domain_sipusername)); push(@report_data,_get_report_obj($context)); @@ -519,7 +527,7 @@ sub _provision_subscribers_checks { # $result = 0; #even in skip-error mode.. #} my $barring = $barring_resellername->{barrings}; - next unless ($barring); + $barring = $default_barring unless ($barring); $result &= _check_ncos_level($context,$resellername,$barring); } } @@ -564,10 +572,10 @@ sub _provision_subscribers_checks { sub _check_ncos_level { my ($context,$resellername,$barring) = @_; my $result = 1; - if (not exists $barring_profiles->{$resellername}) { + if ($barring ne $default_barring and not exists $barring_profiles->{$resellername}) { rowprocessingerror(threadid(),"barring mappings for reseller $resellername missing",getlogger(__PACKAGE__)); $result = 0; #even in skip-error mode.. - } elsif (not exists $barring_profiles->{$resellername}->{$barring}) { + } elsif ($barring ne $default_barring and not exists $barring_profiles->{$resellername}->{$barring}) { rowprocessingerror(threadid(),"mappings for barring '" . $barring . "' of reseller $resellername missing",getlogger(__PACKAGE__)); $result = 0; #even in skip-error mode.. } else { @@ -584,14 +592,19 @@ sub _check_ncos_level { }; if ($@ or not defined $context->{ncos_level_map}->{$reseller_id}->{$barring}) { my $err = "cannot find ncos level '$level' of reseller $resellername"; - if (not defined $context->{rowcount}) { - rowprocessingerror(threadid(),$err,getlogger(__PACKAGE__)); + if (not defined $context->{_rowcount}) { + if ($barring ne $default_barring) { + rowprocessingerror(threadid(),$err,getlogger(__PACKAGE__)); + $result = 0; #even in skip-error mode.. + } else { + rowprocessingwarn(threadid(),$err,getlogger(__PACKAGE__)); + } } elsif ($skip_errors) { _warn($context, $err); } else { _error($context, $err); + $result = 0; #even in skip-error mode.. } - $result = 0; #even in skip-error mode.. } else { processing_info(threadid(),"ncos level '$level' of reseller $resellername found",getlogger(__PACKAGE__)); } @@ -605,17 +618,54 @@ sub _update_contact { my ($context) = @_; - if ($context->{contract}->{contact_id}) { - #NGCP::BulkProcessor::Dao::Trunk::billing::contacts::update_row($context->{db}, - # { @{ $context->{contract}->{contact} }, id => $context->{contract}->{contact_id}, } - #); - } else { + my $existing_contacts = NGCP::BulkProcessor::Dao::Trunk::billing::contacts::findby_reselleridfields( + $context->{reseller}->{id}, + { $contact_hash_field => $context->{contract}->{contact}->{$contact_hash_field} }, + ); + if ((scalar @$existing_contacts) == 0) { $context->{contract}->{contact}->{id} = NGCP::BulkProcessor::Dao::Trunk::billing::contacts::insert_row($context->{db}, $context->{contract}->{contact}, ); $context->{contract}->{contact_id} = $context->{contract}->{contact}->{id}; _info($context,"contact id $context->{contract}->{contact}->{id} created",1); + } else { + my $existing_contact = $existing_contacts->[0]; + if ((scalar @$existing_contacts) > 1) { + _warn($context,(scalar @$existing_contacts) . " existing contacts found, using first contact id $existing_contact->{id}"); + } else { + _info($context,"existing contact id $existing_contact->{id} found",1); + } + $context->{contract}->{contact}->{id} = $existing_contact->{id}; + $context->{contract}->{contact_id} = $context->{contract}->{contact}->{id}; + + my $existing_contracts = NGCP::BulkProcessor::Dao::Trunk::billing::contracts::findby_contactid($existing_contact->{id}); + if ((scalar @$existing_contracts) > 0) { + my $existing_contract = $existing_contracts->[0]; + if ((scalar @$existing_contracts) > 1) { + _warn($context,(scalar @$existing_contracts) . " existing contracts found, using first contact id $existing_contract->{id}"); + } else { + _info($context,"existing contract id $existing_contact->{id} found",1); + } + $context->{contract}->{id} = $existing_contract->{id}; + $context->{bill_subscriber}->{contract_id} = $context->{contract}->{id}; + $context->{prov_subscriber}->{account_id} = $context->{contract}->{id}; + } else { + _warn($context,"no existing contract of contact id $existing_contact->{id} found, will be created"); + } } + $context->{contract}->{contact_id} = $context->{contract}->{contact}->{id}; + + #if ($context->{contract}->{contact_id}) { + # #NGCP::BulkProcessor::Dao::Trunk::billing::contacts::update_row($context->{db}, + # # { @{ $context->{contract}->{contact} }, id => $context->{contract}->{contact_id}, } + # #); + #} else { + # $context->{contract}->{contact}->{id} = NGCP::BulkProcessor::Dao::Trunk::billing::contacts::insert_row($context->{db}, + # $context->{contract}->{contact}, + # ); + # $context->{contract}->{contact_id} = $context->{contract}->{contact}->{id}; + # _info($context,"contact id $context->{contract}->{contact}->{id} created",1); + #} return 1; @@ -919,6 +969,15 @@ sub _set_registrations { %$registration); _info($context,"permanent registration $registration->{contact} added",1); } + foreach my $trusted_source (@{$context->{trusted_sources}}) { + #print "blah"; + $trusted_source->{id} = NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_trusted_sources::insert_row($context->{db},{ + %$trusted_source, + subscriber_id => $context->{prov_subscriber}->{id}, + uuid => $context->{prov_subscriber}->{uuid}, + }); + _info($context,"trusted source $trusted_source->{protocol} $trusted_source->{src_ip} from $trusted_source->{from_pattern} added",1); + } return $result; } @@ -1055,12 +1114,14 @@ sub _provision_susbcriber_init_context { phonenumber => $subscriber->{phone_number}, email => $subscriber->{email}, vatnum => $subscriber->{vat_number}, + $contact_hash_field => $subscriber->{contact_hash}, }, }; $contact_dupes{$subscriber->{contact_hash}} = 1; } else { _warn($context,'non-unique contact data, skipped'); $context->{nonunique_contacts}->{$context->{prov_subscriber}->{username}} += 1; + $result = 0; } } @@ -1119,6 +1180,12 @@ sub _provision_susbcriber_init_context { $context->{channels} = $default_channels; } + unless (defined $context->{prov_subscriber}->{password} and length($context->{prov_subscriber}->{password}) > 0) { + my $generated = _generate_sippassword(); + $context->{prov_subscriber}->{password} = $generated; + _info($context,"empty sip_password, using generated '$generated'",1); + } + unless (defined $context->{prov_subscriber}->{webusername} and length($context->{prov_subscriber}->{webusername}) > 0) { $context->{prov_subscriber}->{webusername} = $webusername; $context->{prov_subscriber}->{webpassword} = undef; @@ -1160,6 +1227,11 @@ sub _provision_susbcriber_init_context { } elsif ((scalar keys %barrings) == 1) { my ($barring) = keys %barrings; $context->{ncos_level} = $context->{ncos_level_map}->{$context->{reseller}->{id}}->{$barring}; + } else { + if (exists $context->{ncos_level_map}->{$context->{reseller}->{id}}->{$default_barring}) { + $context->{ncos_level} = $context->{ncos_level_map}->{$context->{reseller}->{id}}->{$default_barring}; + _info($context,"no ncos level, using default '$context->{ncos_level}->{level}'",1); + } } foreach my $allowed_cli (@{NGCP::BulkProcessor::Projects::Migration::Teletek::Dao::import::AllowedCli::findby_sipusername($first->{sip_username})}) { @@ -1276,6 +1348,7 @@ sub _provision_susbcriber_init_context { $context->{callforwards} = \%cfsimple; my @registrations = (); + my @trusted_sources = (); if (my $registration = NGCP::BulkProcessor::Projects::Migration::Teletek::Dao::import::Registration::findby_sipusername($first->{sip_username})) { #todo: check/transform, multiple contacts push(@registrations,{ @@ -1284,8 +1357,22 @@ sub _provision_susbcriber_init_context { contact => $registration->{sip_contact}, ruid => NGCP::BulkProcessor::Dao::Trunk::kamailio::location::next_ruid(), }); + if ($registration->{sip_contact} =~ /(\d{0,3}\.\d{0,3}\.\d{0,3}\.\d{0,3})/) { + if (check_ipnet($1)) { + push(@trusted_sources,{ + src_ip => $1, + protocol => $NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_trusted_sources::PROTOCOL_UDP, + from_pattern => 'sip:.+' . quotemeta($context->{domain}->{domain}), + }); + } else { + _warn($context,"invalid IP address in sip contact '$registration->{sip_contact}', skipping trusted source"); + } + } else { + _warn($context,"cannot extract IP address from sip contact '$registration->{sip_contact}', skipping trusted source"); + } } $context->{registrations} = \@registrations; + $context->{trusted_sources} = \@trusted_sources; #$context->{counts} = {} unless defined $context->{counts}; @@ -1302,6 +1389,10 @@ sub _generate_webpassword { ); } +sub _generate_sippassword { + return createtmpstring($sippassword_length); +} + sub _generate_webusername { return createtmpstring($webusername_length); } diff --git a/lib/NGCP/BulkProcessor/Projects/Migration/Teletek/Settings.pm b/lib/NGCP/BulkProcessor/Projects/Migration/Teletek/Settings.pm index 5ab3130..305bbe0 100644 --- a/lib/NGCP/BulkProcessor/Projects/Migration/Teletek/Settings.pm +++ b/lib/NGCP/BulkProcessor/Projects/Migration/Teletek/Settings.pm @@ -87,6 +87,7 @@ our @EXPORT_OK = qw( $provision_subscriber_numofthreads $webpassword_length $webusername_length + $sippassword_length $default_channels_map $cf_default_priority @@ -144,6 +145,7 @@ our $provision_subscriber_multithreading = $enablemultithreading; our $provision_subscriber_numofthreads = $cpucount; our $webpassword_length = 8; our $webusername_length = 8; +our $sippassword_length = 16; our $default_channels_map = { 0 => 1, 4 => 10, @@ -225,6 +227,11 @@ sub update_settings { configurationerror($configfile,'webusername_length greater than 7 required',getlogger(__PACKAGE__)); $result = 0; } + $sippassword_length = $data->{sippassword_length} if exists $data->{sippassword_length}; + if (not defined $sippassword_length or $sippassword_length <= 7) { + configurationerror($configfile,'sippassword_length greater than 7 required',getlogger(__PACKAGE__)); + $result = 0; + } #$default_channels = $data->{default_channels} if exists $data->{default_channels}; $cf_default_priority = $data->{cf_default_priority} if exists $data->{cf_default_priority}; diff --git a/lib/NGCP/BulkProcessor/Projects/Migration/Teletek/barring_profiles.yml b/lib/NGCP/BulkProcessor/Projects/Migration/Teletek/barring_profiles.yml index c7e9aef..79ed2e0 100644 --- a/lib/NGCP/BulkProcessor/Projects/Migration/Teletek/barring_profiles.yml +++ b/lib/NGCP/BulkProcessor/Projects/Migration/Teletek/barring_profiles.yml @@ -1,19 +1,22 @@ mapping: Junet: + default: 'Junet 38' 21: 'Junet 21' 25: 'Junet 25' 29: 'Junet 29' 33: 'Junet 33' Teleman: + default: 'Teleman 38' 21: 'Teleman 21' 25: 'Teleman 25' 29: 'Teleman 29' 33: 'Teleman 33' Teletek: + default: 'Teletek 38' 21: 'Teletek 21' 25: 'Teletek 25' 27: 'Teletek 27' 29: 'Teletek 29' 33: 'Teletek 33' - 25_29: 'Teletek 25 29' - 25_33: 'Teletek 25 33' + 25_29: 'Teletek 25' + 25_33: 'Teletek 25' diff --git a/lib/NGCP/BulkProcessor/Projects/Migration/Teletek/process.pl b/lib/NGCP/BulkProcessor/Projects/Migration/Teletek/process.pl index de2ecc3..19c63df 100644 --- a/lib/NGCP/BulkProcessor/Projects/Migration/Teletek/process.pl +++ b/lib/NGCP/BulkProcessor/Projects/Migration/Teletek/process.pl @@ -79,6 +79,7 @@ use NGCP::BulkProcessor::Dao::Trunk::billing::contracts qw(); use NGCP::BulkProcessor::Dao::Trunk::billing::voip_subscribers qw(); use NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_dbaliases qw(); use NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_cf_mappings qw(); +use NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_trusted_sources qw(); use NGCP::BulkProcessor::Dao::Trunk::kamailio::location qw(); @@ -635,6 +636,9 @@ sub create_subscriber_task { $stats .= "\n registrations: " . NGCP::BulkProcessor::Dao::Trunk::kamailio::location::countby_usernamedomain(undef,undef) . ' rows'; + $stats .= "\n trusted sources: " . + NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_trusted_sources::countby_subscriberid(undef) . ' rows'; + $stats .= "\n non-unique contacts skipped:\n " . join("\n ",keys %$nonunique_contacts) if (scalar keys %$nonunique_contacts) > 0; }; @@ -646,7 +650,7 @@ sub create_subscriber_task { push(@$messages,"YOU MIGHT WANT TO RESTART KAMAILIO FOR PERMANENT REGISTRATIONS TO COME INTO EFFECT"); } } - destroy_all_dbs(); #every task should leave with closed connections. + destroy_all_dbs(); return $result; } diff --git a/lib/NGCP/BulkProcessor/Projects/Migration/Teletek/settings.cfg b/lib/NGCP/BulkProcessor/Projects/Migration/Teletek/settings.cfg index 18d54db..ca81476 100644 --- a/lib/NGCP/BulkProcessor/Projects/Migration/Teletek/settings.cfg +++ b/lib/NGCP/BulkProcessor/Projects/Migration/Teletek/settings.cfg @@ -43,6 +43,7 @@ provision_subscriber_multithreading = 1 provision_subscriber_numofthreads = 2 webpassword_length = 8 webusername_length = 8 +sippassword_length = 16 report_filename = provision.txt #report_filename = provision_%s.json #default_channels = 1 diff --git a/lib/NGCP/BulkProcessor/SqlConnector.pm b/lib/NGCP/BulkProcessor/SqlConnector.pm index 7f2236f..cfe7546 100644 --- a/lib/NGCP/BulkProcessor/SqlConnector.pm +++ b/lib/NGCP/BulkProcessor/SqlConnector.pm @@ -993,4 +993,13 @@ sub db_finish { } +sub ping { + + my $self = shift; + + #notimplementederror((ref $self) . ': ' . (caller(0))[3] . ' not implemented',getlogger(__PACKAGE__)); + return 1; + +} + 1; diff --git a/lib/NGCP/BulkProcessor/SqlConnectors/MySQLDB.pm b/lib/NGCP/BulkProcessor/SqlConnectors/MySQLDB.pm index 9a3360c..9e9e2dc 100644 --- a/lib/NGCP/BulkProcessor/SqlConnectors/MySQLDB.pm +++ b/lib/NGCP/BulkProcessor/SqlConnectors/MySQLDB.pm @@ -39,11 +39,13 @@ my $texttable_charset = 'latin1'; my $texttable_collation = 'latin1_swedish_ci'; my $default_texttable_engine = 'MyISAM'; #InnoDB'; # ... provide transactions y/n? -my $session_charset = 'latin1'; +#my $session_charset = 'latin1'; +my $session_charset = 'utf8mb4'; my $LongReadLen = $LongReadLen_limit; #bytes my $LongTruncOk = 0; +my $net_read_timeout = 300; #my $logger = getlogger(__PACKAGE__); #my $lock_do_chunk = 0; #1; @@ -234,6 +236,9 @@ sub db_connect { #$self->db_do('SET SESSION time_format = \'%H:%i:%s\''); #$self->db_do('SET SESSION time_zone = \'Europe/Paris\''); #$self->db_do('SET SESSION datetime_format = \'%Y-%m-%d %H:%i:%s\''); + if (defined $net_read_timeout) { + $self->db_do('SET SESSION net_read_timeout = ' . $net_read_timeout); + } if (length($serialization_level) > 0) { $self->db_do('SET SESSION TRANSACTION ISOLATION LEVEL ' . $serialization_level); @@ -555,4 +560,13 @@ sub _split_indexcol { return ($indexcol, ''); } +sub ping { + + my $self = shift; + + $self->{dbh}->ping() if $self->{dbh}; + return 1; + +} + 1;