diff --git a/lib/NGCP/BulkProcessor/Dao/Trunk/billing/contracts.pm b/lib/NGCP/BulkProcessor/Dao/Trunk/billing/contracts.pm index 173b6ce..0c89fd8 100755 --- a/lib/NGCP/BulkProcessor/Dao/Trunk/billing/contracts.pm +++ b/lib/NGCP/BulkProcessor/Dao/Trunk/billing/contracts.pm @@ -34,6 +34,7 @@ our @EXPORT_OK = qw( countby_status_resellerid findby_contactid + findby_externalid findby_id forupdate_id @@ -108,6 +109,23 @@ sub findby_contactid { } +sub findby_externalid { + + my ($external_id,$load_recursive) = @_; + + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + + my $stmt = 'SELECT * FROM ' . $table . ' WHERE ' . + $db->columnidentifier('external_id') . ' = ?'; + my @params = ($external_id); + my $rows = $db->db_get_all_arrayref($stmt,@params); + + return buildrecords_fromrows($rows,$load_recursive); + +} + sub findby_id { my ($id,$load_recursive) = @_; diff --git a/lib/NGCP/BulkProcessor/Dao/Trunk/provisioning/voip_preferences.pm b/lib/NGCP/BulkProcessor/Dao/Trunk/provisioning/voip_preferences.pm index 61cd5c0..089f852 100755 --- a/lib/NGCP/BulkProcessor/Dao/Trunk/provisioning/voip_preferences.pm +++ b/lib/NGCP/BulkProcessor/Dao/Trunk/provisioning/voip_preferences.pm @@ -27,8 +27,11 @@ our @EXPORT_OK = qw( $CC_ATTRIBUTE $ACCOUNT_ID_ATTRIBUTE + $NCOS_ID_ATTRIBUTE $ADM_NCOS_ID_ATTRIBUTE + $GPPx_ATTRIBUTE + $PEER_AUTH_USER $PEER_AUTH_PASS $PEER_AUTH_REALM @@ -76,7 +79,9 @@ our $AC_ATTRIBUTE = 'ac'; our $CC_ATTRIBUTE = 'cc'; our $ACCOUNT_ID_ATTRIBUTE = 'account_id'; +our $NCOS_ID_ATTRIBUTE = 'ncos_id'; our $ADM_NCOS_ID_ATTRIBUTE = 'adm_ncos_id'; +our $GPPx_ATTRIBUTE = 'gpp'; our $PEER_AUTH_USER = 'peer_auth_user'; our $PEER_AUTH_PASS = 'peer_auth_pass'; diff --git a/lib/NGCP/BulkProcessor/FileProcessors/CSVFileSimple.pm b/lib/NGCP/BulkProcessor/FileProcessors/CSVFileSimple.pm index 10f7664..83e7131 100755 --- a/lib/NGCP/BulkProcessor/FileProcessors/CSVFileSimple.pm +++ b/lib/NGCP/BulkProcessor/FileProcessors/CSVFileSimple.pm @@ -13,7 +13,7 @@ require Exporter; our @ISA = qw(Exporter NGCP::BulkProcessor::FileProcessor); our @EXPORT_OK = qw(); -my $default_lineseparator = '\\n\\r|\\r|\\n'; +my $default_lineseparator = '\\r\\n|\\r|\\n'; #\\n\\r my $default_fieldseparator = ","; my $default_encoding = 'UTF-8'; diff --git a/lib/NGCP/BulkProcessor/Projects/ETL/SetProfilePackage/process.pl b/lib/NGCP/BulkProcessor/Projects/ETL/SetProfilePackage/process.pl index 40eb4b0..591f596 100755 --- a/lib/NGCP/BulkProcessor/Projects/ETL/SetProfilePackage/process.pl +++ b/lib/NGCP/BulkProcessor/Projects/ETL/SetProfilePackage/process.pl @@ -48,6 +48,7 @@ use NGCP::BulkProcessor::LoadConfig qw( ); use NGCP::BulkProcessor::Array qw(removeduplicates); use NGCP::BulkProcessor::Utils qw(getscriptpath prompt cleanupdir); +use NGCP::BulkProcessor::RestConnectors::NGCPRestApi qw(cleanupcertfiles); use NGCP::BulkProcessor::Mail qw( cleanupmsgfiles ); @@ -166,6 +167,7 @@ sub cleanup_task { #cleanupdbfiles() if $clean_generated; cleanuplogfiles(\&fileerror,\&filewarn,($currentlogfile,$attachmentlogfile)); cleanupmsgfiles(\&fileerror,\&filewarn); + cleanupcertfiles(); #cleanupdir($output_path,1,\&filewarn,getlogger(getscriptpath())) if $clean_generated; #cleanupdir($rollback_path,1,\&filewarn,getlogger(getscriptpath())) if $clean_generated; $result = 1; diff --git a/lib/NGCP/BulkProcessor/Projects/Migration/Teletek/Import.pm b/lib/NGCP/BulkProcessor/Projects/Migration/Teletek/Import.pm index 1ced891..eaf94f6 100755 --- a/lib/NGCP/BulkProcessor/Projects/Migration/Teletek/Import.pm +++ b/lib/NGCP/BulkProcessor/Projects/Migration/Teletek/Import.pm @@ -394,7 +394,7 @@ sub _unfold_number_ranges { my ($context,$record,$rows) = @_; - sub create_new_record_code{} + #sub create_new_record_code{} my $result = 0; my @fieldnames = @{$context->{fieldnames}}; diff --git a/lib/NGCP/BulkProcessor/Projects/Migration/UPCAT/Check.pm b/lib/NGCP/BulkProcessor/Projects/Migration/UPCAT/Check.pm new file mode 100755 index 0000000..3d59972 --- /dev/null +++ b/lib/NGCP/BulkProcessor/Projects/Migration/UPCAT/Check.pm @@ -0,0 +1,244 @@ +package NGCP::BulkProcessor::Projects::Migration::UPCAT::Check; +use strict; + +## no critic + +no strict 'refs'; + +use NGCP::BulkProcessor::Dao::Trunk::billing::billing_profiles qw(); +#use NGCP::BulkProcessor::Dao::Trunk::billing::billing_mappings qw(); +use NGCP::BulkProcessor::Dao::Trunk::billing::contracts_billing_profile_network qw(); +use NGCP::BulkProcessor::Dao::Trunk::billing::contracts_billing_profile_network_schedule qw(); +use NGCP::BulkProcessor::Dao::Trunk::billing::contracts qw(); +use NGCP::BulkProcessor::Dao::Trunk::billing::contract_balances qw(); +use NGCP::BulkProcessor::Dao::Trunk::billing::contacts qw(); +use NGCP::BulkProcessor::Dao::Trunk::billing::voip_subscribers qw(); +use NGCP::BulkProcessor::Dao::Trunk::billing::ncos_levels qw(); +use NGCP::BulkProcessor::Dao::Trunk::billing::voip_numbers qw(); +use NGCP::BulkProcessor::Dao::Trunk::billing::products qw(); +use NGCP::BulkProcessor::Dao::Trunk::billing::domains qw(); +use NGCP::BulkProcessor::Dao::Trunk::billing::domain_resellers qw(); + +use NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_domains qw(); +use NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_subscribers qw(); +use NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_preferences qw(); +use NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_usr_preferences qw(); +use NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_aig_sequence qw(); +use NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_allowed_ip_groups qw(); +use NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_dbaliases qw(); +#use NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_cf_mappings qw(); +#use NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_cf_destination_sets qw(); +#use NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_cf_destinations qw(); +#use NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_trusted_sources qw(); + +use NGCP::BulkProcessor::Dao::Trunk::kamailio::voicemail_users qw(); +#use NGCP::BulkProcessor::Dao::Trunk::kamailio::location qw(); + +use NGCP::BulkProcessor::Projects::Migration::UPCAT::Dao::import::Subscriber qw(); + +use NGCP::BulkProcessor::RestRequests::Trunk::Resellers qw(); +use NGCP::BulkProcessor::RestRequests::Trunk::Domains qw(); +use NGCP::BulkProcessor::RestRequests::Trunk::BillingProfiles qw(); + +require Exporter; +our @ISA = qw(Exporter); +our @EXPORT_OK = qw( + check_billing_db_tables + check_provisioning_db_tables + check_kamailio_db_tables + check_import_db_tables + check_rest_get_items +); + +my $NOK = 'NOK'; +my $OK = 'ok'; + +sub check_billing_db_tables { + + my ($messages) = @_; + + my $result = 1; + my $check_result; + my $message; + + my $message_prefix = 'NGCP billing db tables - '; + + ($check_result,$message) = _check_table($message_prefix,'NGCP::BulkProcessor::Dao::Trunk::billing::billing_profiles'); + $result &= $check_result; push(@$messages,$message); + + ($check_result,$message) = _check_table($message_prefix,'NGCP::BulkProcessor::Dao::Trunk::billing::products'); + $result &= $check_result; push(@$messages,$message); + + ($check_result,$message) = _check_table($message_prefix,'NGCP::BulkProcessor::Dao::Trunk::billing::domains'); + $result &= $check_result; push(@$messages,$message); + + ($check_result,$message) = _check_table($message_prefix,'NGCP::BulkProcessor::Dao::Trunk::billing::domain_resellers'); + $result &= $check_result; push(@$messages,$message); + + ($check_result,$message) = _check_table($message_prefix,'NGCP::BulkProcessor::Dao::Trunk::billing::contacts'); + $result &= $check_result; push(@$messages,$message); + + ($check_result,$message) = _check_table($message_prefix,'NGCP::BulkProcessor::Dao::Trunk::billing::contracts'); + $result &= $check_result; push(@$messages,$message); + + ($check_result,$message) = _check_table($message_prefix,'NGCP::BulkProcessor::Dao::Trunk::billing::contract_balances'); + $result &= $check_result; push(@$messages,$message); + + #($check_result,$message) = _check_table($message_prefix,'NGCP::BulkProcessor::Dao::Trunk::billing::billing_mappings'); + #$result &= $check_result; push(@$messages,$message); + + ($check_result,$message) = _check_table($message_prefix,'NGCP::BulkProcessor::Dao::Trunk::billing::contracts_billing_profile_network'); + $result &= $check_result; push(@$messages,$message); + + ($check_result,$message) = _check_table($message_prefix,'NGCP::BulkProcessor::Dao::Trunk::billing::contracts_billing_profile_network_schedule'); + $result &= $check_result; push(@$messages,$message); + + ($check_result,$message) = _check_table($message_prefix,'NGCP::BulkProcessor::Dao::Trunk::billing::voip_subscribers'); + $result &= $check_result; push(@$messages,$message); + + ($check_result,$message) = _check_table($message_prefix,'NGCP::BulkProcessor::Dao::Trunk::billing::ncos_levels'); + $result &= $check_result; push(@$messages,$message); + + ($check_result,$message) = _check_table($message_prefix,'NGCP::BulkProcessor::Dao::Trunk::billing::voip_numbers'); + $result &= $check_result; push(@$messages,$message); + + return $result; + +} + +sub check_import_db_tables { + + my ($messages) = @_; + + my $result = 1; + my $check_result; + my $message; + + my $message_prefix = 'import db tables - '; + + ($check_result,$message) = _check_table($message_prefix,'NGCP::BulkProcessor::Projects::Migration::UPCAT::Dao::import::Subscriber'); + $result &= $check_result; push(@$messages,$message); + + return $result; + +} + +sub check_provisioning_db_tables { + + my ($messages) = @_; + + my $result = 1; + my $check_result; + my $message; + + my $message_prefix = 'NGCP provisioning db tables - '; + + ($check_result,$message) = _check_table($message_prefix,'NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_domains'); + $result &= $check_result; push(@$messages,$message); + + ($check_result,$message) = _check_table($message_prefix,'NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_subscribers'); + $result &= $check_result; push(@$messages,$message); + + ($check_result,$message) = _check_table($message_prefix,'NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_preferences'); + $result &= $check_result; push(@$messages,$message); + + ($check_result,$message) = _check_table($message_prefix,'NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_usr_preferences'); + $result &= $check_result; push(@$messages,$message); + + ($check_result,$message) = _check_table($message_prefix,'NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_aig_sequence'); + $result &= $check_result; push(@$messages,$message); + + ($check_result,$message) = _check_table($message_prefix,'NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_allowed_ip_groups'); + $result &= $check_result; push(@$messages,$message); + + ($check_result,$message) = _check_table($message_prefix,'NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_dbaliases'); + $result &= $check_result; push(@$messages,$message); + + #($check_result,$message) = _check_table($message_prefix,'NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_cf_mappings'); + #$result &= $check_result; push(@$messages,$message); + + #($check_result,$message) = _check_table($message_prefix,'NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_cf_destination_sets'); + #$result &= $check_result; push(@$messages,$message); + + #($check_result,$message) = _check_table($message_prefix,'NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_cf_destinations'); + #$result &= $check_result; push(@$messages,$message); + + #($check_result,$message) = _check_table($message_prefix,'NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_trusted_sources'); + #$result &= $check_result; push(@$messages,$message); + + return $result; + +} + +sub check_kamailio_db_tables { + + my ($messages) = @_; + + my $result = 1; + my $check_result; + my $message; + + my $message_prefix = 'NGCP kamailio db tables - '; + + ($check_result,$message) = _check_table($message_prefix,'NGCP::BulkProcessor::Dao::Trunk::kamailio::voicemail_users'); + $result &= $check_result; push(@$messages,$message); + + #($check_result,$message) = _check_table($message_prefix,'NGCP::BulkProcessor::Dao::Trunk::kamailio::location'); + #$result &= $check_result; push(@$messages,$message); + + return $result; + +} + +sub _check_table { + + my ($message_prefix,$module) = @_; + my $result = 0; + my $message = ($message_prefix // '') . &{$module . '::gettablename'}() . ': '; + eval { + $result = &{$module . '::check_table'}(); + }; + if (@$ or not $result) { + return (0,$message . $NOK); + } else { + return (1,$message . $OK); + } + +} + +sub check_rest_get_items { + + my ($messages) = @_; + + my $result = 1; + my $check_result; + my $message; + + my $message_prefix = 'NGCP id\'s/constants - '; + + return $result; +} + + +sub _check_rest_get_item { + + my ($message_prefix,$module,$id,$item_name_field,$get_method,$item_path_method) = @_; + my $item = undef; + $get_method //= 'get_item'; + $item_path_method //= 'get_item_path'; + my $message = ($message_prefix // '') . &{$module . '::' . $item_path_method}($id) . ': '; + return (0,$message . $NOK,$item) unless $id; + eval { + $item = &{$module . '::' . $get_method}($id); + }; + + if (@$ or not defined $item or ('ARRAY' eq ref $item and (scalar @$item) != 1)) { + return (0,$message . $NOK,$item); + } else { + $item = $item->[0] if ('ARRAY' eq ref $item and (scalar @$item) == 1); + return (1,$message . "'" . $item->{$item_name_field} . "' " . $OK,$item); + } + +} + +1; diff --git a/lib/NGCP/BulkProcessor/Projects/Migration/UPCAT/Dao/import/Subscriber.pm b/lib/NGCP/BulkProcessor/Projects/Migration/UPCAT/Dao/import/Subscriber.pm new file mode 100755 index 0000000..9740ae0 --- /dev/null +++ b/lib/NGCP/BulkProcessor/Projects/Migration/UPCAT/Dao/import/Subscriber.pm @@ -0,0 +1,470 @@ +package NGCP::BulkProcessor::Projects::Migration::UPCAT::Dao::import::Subscriber; +use strict; + +## no critic + +use NGCP::BulkProcessor::Projects::Migration::UPCAT::ProjectConnectorPool qw( + get_import_db + destroy_all_dbs +); +#import_db_tableidentifier + +use NGCP::BulkProcessor::SqlProcessor qw( + registertableinfo + create_targettable + checktableinfo + copy_row + + insert_stmt + + process_table +); +use NGCP::BulkProcessor::SqlRecord qw(); + +#use NGCP::BulkProcessor::Projects::Migration::UPCAT::Dao::import::Subscriber qw(); + +require Exporter; +our @ISA = qw(Exporter NGCP::BulkProcessor::SqlRecord); +our @EXPORT_OK = qw( + create_table + gettablename + check_table + getinsertstatement + getupsertstatement + + findby_ccacsn + countby_ccacsn + + findby_domain_sipusername + findby_domain_webusername + list_domain_billingprofilename_resellernames + findby_sipusername + list_barring_resellernames + + update_delta + findby_delta + countby_delta + + $deleted_delta + $updated_delta + $added_delta + + process_records + + @fieldnames +); +# findby_ccacsn +# countby_ccacsn + +# findby_domain_sipusername +# findby_domain_webusername +# list_domain_billingprofilename_resellernames +# findby_sipusername +# list_barring_resellernames + +my $tablename = 'subscriber'; +my $get_db = \&get_import_db; +#my $get_tablename = \&import_db_tableidentifier; + +my @csv_cols = ( + # fields in order of cols from .csv + "_rownum", + "_dn", + "_txt_sw_username", + "sip_password", #"Subscriber sip password", + "_len", + "_cpe_mta_mac_address", + "_cpe_model", + "_cpe_vendor", + "customer_id", #"Customer ID", +); + +our @fieldnames = ( + @csv_cols, + "reseller_name", #"Reseller name", + "domain", #"Sip domain name", + "billing_profile_name", #"Billing profile name", + "sip_username", #"Subscriber sip username", + "cc", #"Subscriber primary number - country code (cc)", + "ac", #"Subscriber primary number - country code (ac)", + "sn", #"Subscriber primary number - country code (sn)", + "web_username", #"Subscriber web username", + "web_password", #"Subscriber web password", + "barring", + #"allowed_ips", + #"channels", + #"voicemail", + + #calculated fields at the end! + 'rownum', + #'range', + #'contact_hash', + 'filenum', + 'filename', +); +my $expected_fieldnames = [ + @fieldnames, + 'delta', +]; + +# table creation: +my $primarykey_fieldnames = [ 'cc','ac','sn' ]; +my $indexes = { + + $tablename . '_domain_web_username' => [ 'domain(32)','web_username(32)' ], + $tablename . '_domain_sip_username' => [ 'domain(32)','sip_username(32)' ], + + $tablename . '_rownum' => [ 'rownum(11)' ], + $tablename . '_delta' => [ 'delta(7)' ],}; +#my $fixtable_statements = []; + +our $deleted_delta = 'DELETED'; +our $updated_delta = 'UPDATED'; +our $added_delta = 'ADDED'; + +sub new { + + my $class = shift; + my $self = NGCP::BulkProcessor::SqlRecord->new($class,$get_db, + $tablename,$expected_fieldnames,$indexes); + + copy_row($self,shift,$expected_fieldnames); + + return $self; + +} + +sub create_table { + + my ($truncate) = @_; + + my $db = &$get_db(); + + registertableinfo($db,__PACKAGE__,$tablename,$expected_fieldnames,$indexes,$primarykey_fieldnames); + return create_targettable($db,__PACKAGE__,$db,__PACKAGE__,$tablename,$truncate,0,undef); + +} + +sub findby_delta { + + my ($delta,$load_recursive) = @_; + + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + + return [] unless defined $delta; + + my $rows = $db->db_get_all_arrayref( + 'SELECT * FROM ' . + $table . + ' WHERE ' . + $db->columnidentifier('delta') . ' = ?' + ,$delta); + + return buildrecords_fromrows($rows,$load_recursive); + +} + +sub findby_ccacsn { + + my ($cc,$ac,$sn,$load_recursive) = @_; + + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + + return [] unless (defined $cc or defined $ac or defined $sn); + + my $rows = $db->db_get_all_arrayref( + 'SELECT * FROM ' . + $table . + ' WHERE ' . + $db->columnidentifier('cc') . ' = ?' . + ' AND ' . $db->columnidentifier('ac') . ' = ?' . + ' AND ' . $db->columnidentifier('sn') . ' = ?' + ,$cc,$ac,$sn); + + return buildrecords_fromrows($rows,$load_recursive)->[0]; + +} + +sub findby_domain_sipusername { + + my ($domain,$sip_username,$load_recursive) = @_; + + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + + #return [] unless (defined $cc or defined $ac or defined $sn); + + my $rows = $db->db_get_all_arrayref( + $db->paginate_sort_query('SELECT * FROM ' . + $table . + ' WHERE ' . + $db->columnidentifier('domain') . ' = ?' . + ' AND ' . $db->columnidentifier('sip_username') . ' = ?', + undef,undef,[{ + column => 'filenum', + numeric => 1, + dir => 1, + },{ + column => 'rownum', + numeric => 1, + dir => 1, + }]) + ,$domain,$sip_username); + + return buildrecords_fromrows($rows,$load_recursive); + +} + +sub findby_domain_webusername { + + my ($domain,$web_username,$load_recursive) = @_; + + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + + #return [] unless (defined $cc or defined $ac or defined $sn); + + my $rows = $db->db_get_all_arrayref( + $db->paginate_sort_query('SELECT * FROM ' . + $table . + ' WHERE ' . + $db->columnidentifier('domain') . ' = ?' . + ' AND ' . $db->columnidentifier('web_username') . ' = ?', + undef,undef,[{ + column => 'filenum', + numeric => 1, + dir => 1, + },{ + column => 'rownum', + numeric => 1, + dir => 1, + }]) + ,$domain,$web_username); + + return buildrecords_fromrows($rows,$load_recursive); + +} + +sub update_delta { + + my ($cc,$ac,$sn,$delta) = @_; + + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + + my $stmt = 'UPDATE ' . $table . ' SET delta = ?'; + my @params = (); + push(@params,$delta); + if (defined $cc or defined $ac or defined $sn) { + $stmt .= ' WHERE ' . + $db->columnidentifier('cc') . ' = ?' . + ' AND ' . $db->columnidentifier('ac') . ' = ?' . + ' AND ' . $db->columnidentifier('sn') . ' = ?'; + push(@params,$cc,$ac,$sn); + } + + return $db->db_do($stmt,@params); + +} + +sub countby_ccacsn { + + my ($cc,$ac,$sn) = @_; + + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + + my $stmt = 'SELECT COUNT(*) FROM ' . $table; + my @params = (); + if (defined $cc or defined $ac or defined $sn) { + $stmt .= ' WHERE ' . + $db->columnidentifier('cc') . ' = ?' . + ' AND ' . $db->columnidentifier('ac') . ' = ?' . + ' AND ' . $db->columnidentifier('sn') . ' = ?'; + push(@params,$cc,$ac,$sn); + } + + return $db->db_get_value($stmt,@params); + +} + +sub countby_delta { + + my ($deltas) = @_; + + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + + my $stmt = 'SELECT COUNT(*) FROM ' . $table . ' WHERE 1=1'; + my @params = (); + if (defined $deltas and 'HASH' eq ref $deltas) { + foreach my $in (keys %$deltas) { + my @values = (defined $deltas->{$in} and 'ARRAY' eq ref $deltas->{$in} ? @{$deltas->{$in}} : ($deltas->{$in})); + $stmt .= ' AND ' . $db->columnidentifier('delta') . ' ' . $in . ' (' . substr(',?' x scalar @values,1) . ')'; + push(@params,@values); + } + } elsif (defined $deltas and length($deltas) > 0) { + $stmt .= ' AND ' . $db->columnidentifier('delta') . ' = ?'; + push(@params,$deltas); + } + + return $db->db_get_value($stmt,@params); + +} + +sub list_domain_billingprofilename_resellernames { + + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + + my @cols = map { $db->columnidentifier($_); } qw/domain billing_profile_name reseller_name/; + my $stmt = 'SELECT ' . join(',',@cols) . ' FROM ' . $table . ' GROUP BY ' . join(',',@cols); + my @params = (); + + return $db->db_get_all_arrayref($stmt,@params); + +} + +sub list_barring_resellernames { + + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + + my @cols = map { $db->columnidentifier($_); } qw/barring reseller_name/; + my $stmt = 'SELECT ' . join(',',@cols) . ' FROM ' . $table . ' GROUP BY ' . join(',',@cols); + my @params = (); + + return $db->db_get_all_arrayref($stmt,@params); + +} + +sub buildrecords_fromrows { + + my ($rows,$load_recursive) = @_; + + my @records = (); + my $record; + + if (defined $rows and ref $rows eq 'ARRAY') { + foreach my $row (@$rows) { + $record = __PACKAGE__->new($row); + + # transformations go here ... + + push @records,$record; + } + } + + return \@records; + +} + +sub process_records { + + my %params = @_; + my ($process_code, + $static_context, + $init_process_context_code, + $uninit_process_context_code, + $multithreading, + $numofthreads) = @params{qw/ + process_code + static_context + init_process_context_code + uninit_process_context_code + multithreading + numofthreads + /}; + + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + + my @cols = map { $db->columnidentifier($_); } qw/domain sip_username/; + + return process_table( + get_db => $get_db, + class => __PACKAGE__, + process_code => sub { + my ($context,$rowblock,$row_offset) = @_; + return &$process_code($context,$rowblock,$row_offset); + }, + static_context => $static_context, + init_process_context_code => $init_process_context_code, + uninit_process_context_code => $uninit_process_context_code, + destroy_reader_dbs_code => \&destroy_all_dbs, + multithreading => $multithreading, + tableprocessing_threads => $numofthreads, + #'select' => 'SELECT ' . join(',',@cols) . ' FROM ' . $table . ' GROUP BY ' . join(',',@cols), + 'select' => $db->paginate_sort_query('SELECT ' . join(',',@cols) . ' FROM ' . $table . ' GROUP BY ' . join(',',@cols),undef,undef,[{ + column => 'filenum', + numeric => 1, + dir => 1, + },{ + column => 'rownum', + numeric => 1, + dir => 1, + }]), + 'selectcount' => 'SELECT COUNT(*) FROM (SELECT ' . join(',',@cols) . ' FROM ' . $table . ' GROUP BY ' . join(',',@cols) . ') AS g', + ); +} + +sub getinsertstatement { + + my ($insert_ignore) = @_; + check_table(); + return insert_stmt($get_db,__PACKAGE__,$insert_ignore); + +} + +sub getupsertstatement { + + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + my $upsert_stmt = 'INSERT OR REPLACE INTO ' . $table . ' (' . + join(', ',map { local $_ = $_; $_ = $db->columnidentifier($_); $_; } @$expected_fieldnames) . ')'; + my @values = (); + foreach my $fieldname (@$expected_fieldnames) { + if ('delta' eq $fieldname) { + my $stmt = 'SELECT \'' . $updated_delta . '\' FROM ' . $table . ' WHERE ' . + $db->columnidentifier('cc') . ' = ?' . + ' AND ' . $db->columnidentifier('ac') . ' = ?' . + ' AND ' . $db->columnidentifier('sn') . ' = ?'; + push(@values,'COALESCE((' . $stmt . '), \'' . $added_delta . '\')'); + } else { + push(@values,'?'); + } + } + $upsert_stmt .= ' VALUES (' . join(',',@values) . ')'; + return $upsert_stmt; + +} + +sub gettablename { + + return $tablename; + +} + +sub check_table { + + return checktableinfo($get_db, + __PACKAGE__,$tablename, + $expected_fieldnames, + $indexes); + +} + +1; diff --git a/lib/NGCP/BulkProcessor/Projects/Migration/UPCAT/Import.pm b/lib/NGCP/BulkProcessor/Projects/Migration/UPCAT/Import.pm new file mode 100755 index 0000000..c064e4c --- /dev/null +++ b/lib/NGCP/BulkProcessor/Projects/Migration/UPCAT/Import.pm @@ -0,0 +1,290 @@ +package NGCP::BulkProcessor::Projects::Migration::UPCAT::Import; +use strict; + +## no critic + +use threads::shared qw(); + +use NGCP::BulkProcessor::Projects::Migration::UPCAT::Settings qw( + $provision_subscriber_rownum_start + $import_multithreading + $subscriber_import_numofthreads + $ignore_subscriber_unique + $subscriber_import_single_row_txn + + $skip_errors + + $default_domain + $default_reseller_name + $default_billing_profile_name + $default_barring + $cc_ac_map + $default_cc + $cc_len_min + $cc_len_max + $ac_len +); +use NGCP::BulkProcessor::Logging qw ( + getlogger + processing_info + processing_debug +); +use NGCP::BulkProcessor::LogError qw( + fileprocessingwarn + fileprocessingerror +); + +#use NGCP::BulkProcessor::Projects::Migration::UPCAT::FileProcessors::CSVFile qw(); +use NGCP::BulkProcessor::FileProcessors::CSVFileSimple qw(); + +use NGCP::BulkProcessor::Projects::Migration::UPCAT::ProjectConnectorPool qw( + get_import_db + destroy_all_dbs +); + +use NGCP::BulkProcessor::Projects::Migration::UPCAT::Dao::import::Subscriber qw(); + +use NGCP::BulkProcessor::Array qw(removeduplicates); +use NGCP::BulkProcessor::Utils qw(threadid zerofill trim); +use NGCP::BulkProcessor::Table qw(get_rowhash); + +require Exporter; +our @ISA = qw(Exporter); +our @EXPORT_OK = qw( + import_subscriber + +); + +sub import_subscriber { + + my (@files) = @_; + + my $result = NGCP::BulkProcessor::Projects::Migration::UPCAT::Dao::import::Subscriber::create_table(0); + + foreach my $file (@files) { + $result &= _import_subscriber_checks($file); + } + + #my $importer = NGCP::BulkProcessor::Projects::Migration::UPCAT::FileProcessors::CSVFile->new($subscriber_import_numofthreads); + my $importer = NGCP::BulkProcessor::FileProcessors::CSVFileSimple->new($subscriber_import_numofthreads); + + my $upsert = _import_subscriber_reset_delta(); + + destroy_all_dbs(); #close all db connections before forking.. + my $warning_count :shared = 0; + my $filenum = 0; + foreach my $file (@files) { + $filenum++; + $result &= $importer->process( + file => $file, + process_code => sub { + my ($context,$rows,$row_offset) = @_; + my $rownum = $row_offset; + my @subscriber_rows = (); + foreach my $row (@$rows) { + $rownum++; + next if (defined $provision_subscriber_rownum_start and $rownum < $provision_subscriber_rownum_start); + next if (scalar @$row) == 0; + $row = [ map { local $_ = $_; trim($_); $_ =~ s/^"//; $_ =~ s/"$//r; } @$row ]; + my $record = NGCP::BulkProcessor::Projects::Migration::UPCAT::Dao::import::Subscriber->new($row); + $record->{reseller_name} = $default_reseller_name; + ($record->{sip_username},$record->{domain}) = split('@',$record->{_txt_sw_username},2); + $record->{domain} //= $default_domain; + $record->{billing_profile_name} = $default_billing_profile_name; + ($record->{cc},$record->{ac},$record->{sn}) = _split_dn($record->{_dn}); + $record->{web_username} = undef; + $record->{web_password} = undef; + $record->{barring} = $default_barring; + #$record->{allowed_ips} + #"channels", + #"voicemail", + + $record->{rownum} = $rownum; + $record->{filenum} = $filenum; + $record->{filename} = $file; + + my %r = %$record; my @row_ext = @r{@NGCP::BulkProcessor::Projects::Migration::UPCAT::Dao::import::Subscriber::fieldnames}; + if ($context->{upsert}) { + push(@row_ext,$record->{cc},$record->{ac},$record->{sn}); + } else { + push(@row_ext,$NGCP::BulkProcessor::Projects::Migration::UPCAT::Dao::import::Subscriber::added_delta); + } + push(@subscriber_rows,\@row_ext); # if &{$context->{check_number_code}}($context,$record); + + #my %r = %$record; + #$record->{contact_hash} = get_rowhash([@r{@NGCP::BulkProcessor::Projects::Migration::UPCAT::Dao::import::Subscriber::contact_fieldnames}]); + #next unless _unfold_number_ranges($context,$record,\@subscriber_rows); + if ($subscriber_import_single_row_txn and (scalar @subscriber_rows) > 0) { + while (defined (my $subscriber_row = shift @subscriber_rows)) { + if ($skip_errors) { + eval { _insert_subscriber_rows($context,[$subscriber_row]); }; + _warn($context,$@) if $@; + } else { + _insert_subscriber_rows($context,[$subscriber_row]); + } + } + } + } + + if (not $subscriber_import_single_row_txn and (scalar @subscriber_rows) > 0) { + if ($skip_errors) { + eval { _insert_subscriber_rows($context,\@subscriber_rows); }; + _warn($context,$@) if $@; + } else { + _insert_subscriber_rows($context,\@subscriber_rows); + } + } + #use Data::Dumper; + #print Dumper(\@subscriber_rows); + return 1; + }, + init_process_context_code => sub { + my ($context)= @_; + $context->{db} = &get_import_db(); # keep ref count low.. + $context->{upsert} = $upsert; + #$context->{unfold_ranges} = $subscriber_import_unfold_ranges; + #$context->{fieldnames} = \@NGCP::BulkProcessor::Projects::Migration::UPCAT::Dao::import::Subscriber::fieldnames; + #$context->{added_delta} = $NGCP::BulkProcessor::Projects::Migration::UPCAT::Dao::import::Subscriber::added_delta; + #$context->{create_new_record_code} = sub { + # return NGCP::BulkProcessor::Projects::Migration::UPCAT::Dao::import::Subscriber->new(shift); + #}; + + #$context->{check_number_code} = sub { + # my ($context,$record) = @_; + # my $result = 1; + # my $number = $record->{dn}; + # my $number = $record->{cc} . $record->{ac} . $record->{sn}; + # if (NGCP::BulkProcessor::Projects::Migration::UPCAT::Dao::import::Subscriber::findby_ccacsn($record->{cc},$record->{ac},$record->{sn})) { + # if ($skip_errors) { + # _warn($context,"$record->{sip_username}: duplicate number $number"); + # } else { + # _error($context,"$record->{sip_username}: duplicate number $number"); + # } + # $result = 0; + # } + # return $result; + #}; + $context->{error_count} = 0; + $context->{warning_count} = 0; + }, + uninit_process_context_code => sub { + my ($context)= @_; + undef $context->{db}; + destroy_all_dbs(); + { + lock $warning_count; + $warning_count += $context->{warning_count}; + } + }, + multithreading => $import_multithreading + ); + } + + return ($result,$warning_count); + +} + +sub _import_subscriber_checks { + my ($file) = @_; + my $result = 1; + + return $result; +} + +sub _import_subscriber_reset_delta { + my $upsert = 0; + if (NGCP::BulkProcessor::Projects::Migration::UPCAT::Dao::import::Subscriber::countby_ccacsn() > 0) { + processing_info(threadid(),'resetting delta of ' . + NGCP::BulkProcessor::Projects::Migration::UPCAT::Dao::import::Subscriber::update_delta(undef,undef,undef, + $NGCP::BulkProcessor::Projects::Migration::UPCAT::Dao::import::Subscriber::deleted_delta) . + ' subscriber records',getlogger(__PACKAGE__)); + $upsert |= 1; + } + return $upsert; +} + +sub _insert_subscriber_rows { + my ($context,$subscriber_rows) = @_; + $context->{db}->db_do_begin( + ($context->{upsert} ? + NGCP::BulkProcessor::Projects::Migration::UPCAT::Dao::import::Subscriber::getupsertstatement() + : NGCP::BulkProcessor::Projects::Migration::UPCAT::Dao::import::Subscriber::getinsertstatement($ignore_subscriber_unique)), + #NGCP::BulkProcessor::Projects::Migration::UPCAT::Dao::import::Subscriber::gettablename(), + #lock + ); + eval { + $context->{db}->db_do_rowblock($subscriber_rows); + $context->{db}->db_finish(); + }; + my $err = $@; + if ($err) { + eval { + $context->{db}->db_finish(1); + }; + die($err); + } +} + +sub _split_dn { + my ($dn) = @_; + my ($cc,$ac,$sn) = ('','',$dn); + if ($cc_ac_map) { + if ($default_cc) { + $cc = $default_cc; + $dn =~ s/^0//; + $sn = $dn; + } else { + foreach my $cc_length ($cc_len_min .. $cc_len_max) { + my ($_cc,$_dn) = (substr($dn,0,$cc_length), substr($dn,$cc_length)); + if (exists $cc_ac_map->{$_cc}) { + $cc = $_cc; + $sn = $_dn; + $dn = $_dn; + last; + } + } + } + if (exists $cc_ac_map->{$cc}) { + my $ac_map = $cc_ac_map->{$cc}; + foreach my $ac_length ($ac_len->{$cc}->{min} .. $ac_len->{$cc}->{max}) { + my ($_ac,$_sn) = (substr($dn,0,$ac_length), substr($dn,$ac_length)); + if (exists $ac_map->{$_ac}) { + $ac = $_ac; + $sn = $_sn; + #$dn = ''; + last; + } + } + } + } + return ($cc,$ac,$sn); + +} + +sub _error { + + my ($context,$message) = @_; + $context->{error_count} = $context->{error_count} + 1; + fileprocessingerror($context->{filename},$message,getlogger(__PACKAGE__)); + +} + +sub _warn { + + my ($context,$message) = @_; + $context->{warning_count} = $context->{warning_count} + 1; + fileprocessingwarn($context->{filename},$message,getlogger(__PACKAGE__)); + +} + +sub _info { + + my ($context,$message,$debug) = @_; + if ($debug) { + processing_debug($context->{tid},$message,getlogger(__PACKAGE__)); + } else { + processing_info($context->{tid},$message,getlogger(__PACKAGE__)); + } +} + +1; diff --git a/lib/NGCP/BulkProcessor/Projects/Migration/UPCAT/Preferences.pm b/lib/NGCP/BulkProcessor/Projects/Migration/UPCAT/Preferences.pm new file mode 100755 index 0000000..393a4e7 --- /dev/null +++ b/lib/NGCP/BulkProcessor/Projects/Migration/UPCAT/Preferences.pm @@ -0,0 +1,182 @@ +package NGCP::BulkProcessor::Projects::Migration::UPCAT::Preferences; +use strict; + +## no critic + +no strict 'refs'; + +use threads::shared qw(); +#use List::Util qw(); + +use NGCP::BulkProcessor::Projects::Migration::UPCAT::Settings qw( + $dry + $skip_errors + +); + +use NGCP::BulkProcessor::Logging qw ( + getlogger + processing_info + processing_debug +); +use NGCP::BulkProcessor::LogError qw( + rowprocessingerror + rowprocessingwarn +); + +use NGCP::BulkProcessor::Dao::Trunk::billing::ncos_levels qw(); +use NGCP::BulkProcessor::Dao::Trunk::billing::domains qw(); +use NGCP::BulkProcessor::Dao::Trunk::billing::domain_resellers qw(); +use NGCP::BulkProcessor::Dao::Trunk::billing::voip_subscribers qw(); + +use NGCP::BulkProcessor::Projects::Migration::UPCAT::Dao::import::Subscriber qw(); + +use NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_subscribers qw(); +use NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_preferences qw(); +use NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_usr_preferences qw(); + +use NGCP::BulkProcessor::ConnectorPool qw( + get_xa_db +); + +use NGCP::BulkProcessor::Projects::Migration::UPCAT::ProjectConnectorPool qw( + destroy_all_dbs +); + +use NGCP::BulkProcessor::Utils qw(threadid); + +require Exporter; +our @ISA = qw(Exporter); +our @EXPORT_OK = qw( + + clear_subscriber_preferences + delete_subscriber_preference + set_subscriber_preference + get_subscriber_preference + +); + +my %get_preference_sub_names = ( + voip_usr_preferences => 'findby_subscriberid_attributeid', +); +my %preference_id_cols = ( + voip_usr_preferences => 'subscriber_id', +); + +sub clear_subscriber_preferences { + my ($context,$subscriber_id,$attribute,$except_value) = @_; + return _clear_preferences($context,'voip_usr_preferences',$subscriber_id,$attribute,$except_value); +} +sub delete_subscriber_preference { + my ($context,$subscriber_id,$attribute,$value) = @_; + return _delete_preference($context,'voip_usr_preferences',$subscriber_id,$attribute,$value); +} +sub set_subscriber_preference { + my ($context,$subscriber_id,$attribute,$value) = @_; + return _set_preference($context,'voip_usr_preferences',$subscriber_id,$attribute,$value); +} +sub get_subscriber_preference { + my ($context,$subscriber_id,$attribute) = @_; + return _get_preference($context,'voip_usr_preferences',$subscriber_id,$attribute); +} + +sub _clear_preferences { + my ($context,$pref_type,$id,$attribute,$except_value) = @_; + + return &{'NGCP::BulkProcessor::Dao::Trunk::provisioning::' . $pref_type . '::delete_preferences'}($context->{db}, + $id, $attribute->{id}, defined $except_value ? { 'NOT IN' => $except_value } : undef); + +} + +sub _delete_preference { + my ($context,$pref_type,$id,$attribute,$value) = @_; + + return &{'NGCP::BulkProcessor::Dao::Trunk::provisioning::' . $pref_type . '::delete_preferences'}($context->{db}, + $id, $attribute->{id}, { 'IN' => $value } ); + +} + +sub _set_preference { + my ($context,$pref_type,$id,$attribute,$value) = @_; + + if ($attribute->{max_occur} == 1) { + my $old_preferences = &{'NGCP::BulkProcessor::Dao::Trunk::provisioning::' . $pref_type . '::' . $get_preference_sub_names{$pref_type}}($context->{db}, + $id,$attribute->{id}); + if (defined $value) { + if ((scalar @$old_preferences) == 1) { + &{'NGCP::BulkProcessor::Dao::Trunk::provisioning::' . $pref_type . '::update_row'}($context->{db},{ + id => $old_preferences->[0]->{id}, + value => $value, + }); + return $old_preferences->[0]->{id}; + } else { + if ((scalar @$old_preferences) > 1) { + &{'NGCP::BulkProcessor::Dao::Trunk::provisioning::' . $pref_type . '::delete_preferences'}($context->{db}, + $id,$attribute->{id}); + } + return &{'NGCP::BulkProcessor::Dao::Trunk::provisioning::' . $pref_type . '::insert_row'}($context->{db}, + attribute_id => $attribute->{id}, + $preference_id_cols{$pref_type} => $id, + value => $value, + ); + } + } else { + &{'NGCP::BulkProcessor::Dao::Trunk::provisioning::' . $pref_type . '::delete_preferences'}($context->{db}, + $id,$attribute->{id}); + return undef; + } + } else { + if (defined $value) { + return &{'NGCP::BulkProcessor::Dao::Trunk::provisioning::' . $pref_type . '::insert_row'}($context->{db}, + attribute_id => $attribute->{id}, + $preference_id_cols{$pref_type} => $id, + value => $value, + ); + } else { + return undef; + } + } + +} + +sub _get_preference { + my ($context,$pref_type,$id,$attribute) = @_; + + my $preferences = &{'NGCP::BulkProcessor::Dao::Trunk::provisioning::' . $pref_type . '::' . $get_preference_sub_names{$pref_type}}($context->{db}, + $id,$attribute->{id}); + + if ($attribute->{max_occur} == 1) { + return $preferences->[0]; + } else { + return $preferences; + } + +} + +sub _error { + + my ($context,$message) = @_; + $context->{error_count} = $context->{error_count} + 1; + rowprocessingerror($context->{tid},$message,getlogger(__PACKAGE__)); + +} + +sub _warn { + + my ($context,$message) = @_; + $context->{warning_count} = $context->{warning_count} + 1; + rowprocessingwarn($context->{tid},$message,getlogger(__PACKAGE__)); + +} + +sub _info { + + my ($context,$message,$debug) = @_; + if ($debug) { + processing_debug($context->{tid},$message,getlogger(__PACKAGE__)); + } else { + processing_info($context->{tid},$message,getlogger(__PACKAGE__)); + } +} + +1; diff --git a/lib/NGCP/BulkProcessor/Projects/Migration/UPCAT/ProjectConnectorPool.pm b/lib/NGCP/BulkProcessor/Projects/Migration/UPCAT/ProjectConnectorPool.pm new file mode 100755 index 0000000..0d1be6e --- /dev/null +++ b/lib/NGCP/BulkProcessor/Projects/Migration/UPCAT/ProjectConnectorPool.pm @@ -0,0 +1,101 @@ +package NGCP::BulkProcessor::Projects::Migration::UPCAT::ProjectConnectorPool; +use strict; + +## no critic + +use File::Basename; +use Cwd; +use lib Cwd::abs_path(File::Basename::dirname(__FILE__) . '/../../../'); + +use NGCP::BulkProcessor::Projects::Migration::UPCAT::Settings qw( + $import_db_file +); + +use NGCP::BulkProcessor::ConnectorPool qw( + get_connectorinstancename + ping +); + +#use NGCP::BulkProcessor::SqlConnectors::MySQLDB; +#use NGCP::BulkProcessor::SqlConnectors::OracleDB; +#use NGCP::BulkProcessor::SqlConnectors::PostgreSQLDB; +use NGCP::BulkProcessor::SqlConnectors::SQLiteDB qw( + $staticdbfilemode + cleanupdbfiles +); +#use NGCP::BulkProcessor::SqlConnectors::CSVDB; +#use NGCP::BulkProcessor::SqlConnectors::SQLServerDB; +#use NGCP::BulkProcessor::RestConnectors::NGCPRestApi; + +use NGCP::BulkProcessor::SqlProcessor qw(cleartableinfo); + +require Exporter; +our @ISA = qw(Exporter); +our @EXPORT_OK = qw( + get_import_db + import_db_tableidentifier + + destroy_dbs + destroy_all_dbs + + ping_dbs + ping_all_dbs +); + +# thread connector pools: +my $import_dbs = {}; + + +sub get_import_db { + + my ($instance_name,$reconnect) = @_; + my $name = get_connectorinstancename($instance_name); #threadid(); #shift; + + if (not defined $import_dbs->{$name}) { + $import_dbs->{$name} = NGCP::BulkProcessor::SqlConnectors::SQLiteDB->new($instance_name); #$name); + if (not defined $reconnect) { + $reconnect = 1; + } + } + if ($reconnect) { + $import_dbs->{$name}->db_connect($staticdbfilemode,$import_db_file); + } + + return $import_dbs->{$name}; + +} + +sub import_db_tableidentifier { + + my ($get_target_db,$tablename) = @_; + my $target_db = (ref $get_target_db eq 'CODE') ? &$get_target_db() : $get_target_db; + return $target_db->getsafetablename(NGCP::BulkProcessor::SqlConnectors::SQLiteDB::get_tableidentifier($tablename,$staticdbfilemode,$import_db_file)); + +} + +sub ping_dbs { + ping($import_dbs); +} + +sub ping_all_dbs { + ping_dbs(); + NGCP::BulkProcessor::ConnectorPool::ping_dbs(); +} + +sub destroy_dbs { + + + foreach my $name (keys %$import_dbs) { + cleartableinfo($import_dbs->{$name}); + undef $import_dbs->{$name}; + delete $import_dbs->{$name}; + } + +} + +sub destroy_all_dbs() { + destroy_dbs(); + NGCP::BulkProcessor::ConnectorPool::destroy_dbs(); +} + +1; diff --git a/lib/NGCP/BulkProcessor/Projects/Migration/UPCAT/Provisioning.pm b/lib/NGCP/BulkProcessor/Projects/Migration/UPCAT/Provisioning.pm new file mode 100755 index 0000000..aab30aa --- /dev/null +++ b/lib/NGCP/BulkProcessor/Projects/Migration/UPCAT/Provisioning.pm @@ -0,0 +1,1110 @@ +package NGCP::BulkProcessor::Projects::Migration::UPCAT::Provisioning; +use strict; + +## no critic + +use threads::shared qw(); +use String::MkPasswd qw(); +#use List::Util qw(); + +use JSON -support_by_pp, -no_export; +use Tie::IxHash; + +use NGCP::BulkProcessor::Projects::Migration::UPCAT::Settings qw( + $dry + $skip_errors + $report_filename + + $provision_subscriber_multithreading + $provision_subscriber_numofthreads + $webpassword_length + $webusername_length + $sippassword_length + + $barring_profiles +); + +use NGCP::BulkProcessor::Logging qw ( + getlogger + processing_info + processing_debug +); +use NGCP::BulkProcessor::LogError qw( + rowprocessingerror + rowprocessingwarn + fileerror +); + +use NGCP::BulkProcessor::Projects::Migration::UPCAT::Dao::import::Subscriber qw(); + +use NGCP::BulkProcessor::Dao::Trunk::billing::billing_profiles qw(); +use NGCP::BulkProcessor::Dao::Trunk::billing::products qw(); +use NGCP::BulkProcessor::Dao::Trunk::billing::contacts qw(); +use NGCP::BulkProcessor::Dao::Trunk::billing::contracts qw(); +#use NGCP::BulkProcessor::Dao::Trunk::billing::billing_mappings qw(); +use NGCP::BulkProcessor::Dao::Trunk::billing::contracts_billing_profile_network qw(); +use NGCP::BulkProcessor::Dao::Trunk::billing::contracts_billing_profile_network_schedule qw(); +use NGCP::BulkProcessor::Dao::Trunk::billing::contract_balances qw(); +use NGCP::BulkProcessor::Dao::Trunk::billing::domains qw(); +use NGCP::BulkProcessor::Dao::Trunk::billing::resellers qw(); +use NGCP::BulkProcessor::Dao::Trunk::billing::voip_subscribers qw(); +use NGCP::BulkProcessor::Dao::Trunk::billing::voip_numbers qw(); +use NGCP::BulkProcessor::Dao::Trunk::billing::domain_resellers qw(); +use NGCP::BulkProcessor::Dao::Trunk::billing::ncos_levels qw(); + +use NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_domains qw(); +use NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_dbaliases qw(); +use NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_subscribers qw(); +use NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_preferences qw(); +use NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_usr_preferences qw(); +#use NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_cf_mappings qw(); +#use NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_cf_destination_sets qw(); +#use NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_cf_destinations qw(); +#use NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_trusted_sources qw(); + +use NGCP::BulkProcessor::Dao::Trunk::kamailio::voicemail_users qw(); +#use NGCP::BulkProcessor::Dao::Trunk::kamailio::location qw(); + +use NGCP::BulkProcessor::RestRequests::Trunk::Subscribers qw(); +use NGCP::BulkProcessor::RestRequests::Trunk::Customers qw(); + +use NGCP::BulkProcessor::Projects::Migration::UPCAT::Preferences qw( + set_subscriber_preference + get_subscriber_preference + clear_subscriber_preferences + delete_subscriber_preference +); +#set_allowed_ips_preferences +#cleanup_aig_sequence_ids + +use NGCP::BulkProcessor::ConnectorPool qw( + get_xa_db +); + +use NGCP::BulkProcessor::Projects::Migration::UPCAT::ProjectConnectorPool qw( + destroy_all_dbs + ping_all_dbs +); + +use NGCP::BulkProcessor::Utils qw(create_uuid threadid timestamp stringtobool trim); #check_ipnet +use NGCP::BulkProcessor::DSSorter qw(sort_by_configs); +use NGCP::BulkProcessor::RandomString qw(createtmpstring); + +require Exporter; +our @ISA = qw(Exporter); +our @EXPORT_OK = qw( + provision_subscribers + +); + +my $split_ipnets_pattern = join('|',( + quotemeta(','), + quotemeta(';'), + #quotemeta('/') +)); + +my $db_lock :shared = undef; +my $file_lock :shared = undef; + +my $default_barring = 'default'; + +sub provision_subscribers { + + my $static_context = { now => timestamp(), _rowcount => undef }; + my $result = _provision_subscribers_checks($static_context); + + destroy_all_dbs(); + my $warning_count :shared = 0; + my %nonunique_contacts :shared = (); + return ($result && NGCP::BulkProcessor::Projects::Migration::UPCAT::Dao::import::Subscriber::process_records( + static_context => $static_context, + process_code => sub { + my ($context,$records,$row_offset) = @_; + ping_all_dbs(); + $context->{_rowcount} = $row_offset; + my @report_data = (); + foreach my $domain_sipusername (@$records) { + $context->{_rowcount} += 1; + next unless _provision_susbcriber($context, + NGCP::BulkProcessor::Projects::Migration::UPCAT::Dao::import::Subscriber::findby_domain_sipusername(@$domain_sipusername)); + push(@report_data,_get_report_obj($context)); + } + #cleanup_aig_sequence_ids($context); + if (defined $report_filename) { + lock $file_lock; + open(my $fh, '>>', $report_filename) or fileerror('cannot open file ' . $report_filename . ': ' . $!,getlogger(__PACKAGE__)); + binmode($fh); + print $fh JSON::to_json(\@report_data,{ allow_nonref => 1, allow_blessed => 1, convert_blessed => 1, pretty => 1, }); + close $fh; + } + return 1; + }, + init_process_context_code => sub { + my ($context)= @_; + $context->{db} = &get_xa_db(); + $context->{error_count} = 0; + $context->{warning_count} = 0; + $context->{nonunique_contacts} = {}; + + }, + uninit_process_context_code => sub { + my ($context)= @_; + undef $context->{db}; + destroy_all_dbs(); + { + lock $warning_count; + $warning_count += $context->{warning_count}; + } + { + lock %nonunique_contacts; + foreach my $sip_username (keys %{$context->{nonunique_contacts}}) { + $nonunique_contacts{$sip_username} = $context->{nonunique_contacts}->{$sip_username}; + } + } + }, + load_recursive => 0, + multithreading => $provision_subscriber_multithreading, + numofthreads => $provision_subscriber_numofthreads, + ),$warning_count,\%nonunique_contacts); + +} + +sub _get_report_obj { + my ($context) = @_; + my %dump = (); + tie(%dump, 'Tie::IxHash'); + foreach my $key (sort keys %$context) { + $dump{$key} = $context->{$key} if 'CODE' ne ref $context->{$key}; + } + foreach my $key (qw/ + sip_account_product + reseller + billing_profile + reseller_map + domain_map + domain + now + error_count + warning_count + attributes + ncos_level_map + ncos_level + nonunique_contacts + tid + db + blocksize + errorstates + queue + readertid + /) { + delete $dump{$key}; + } + return \%dump; +} + +sub _provision_susbcriber { + my ($context,$subscriber_group) = @_; + + return 0 unless _provision_susbcriber_init_context($context,$subscriber_group); + + eval { + lock $db_lock; + $context->{db}->db_begin(); + #_warn($context,'AutoCommit is on') if $context->{db}->{drh}->{AutoCommit}; + + my $existing_billing_voip_subscribers = NGCP::BulkProcessor::Dao::Trunk::billing::voip_subscribers::findby_domainid_username_states( + $context->{db}, + $context->{domain}->{id}, + $context->{prov_subscriber}->{username}, + { 'NOT IN' => $NGCP::BulkProcessor::Dao::Trunk::billing::voip_subscribers::TERMINATED_STATE} + ); + + if ((scalar @$existing_billing_voip_subscribers) == 0) { + + _update_contact($context); + _update_contract($context); + #{ + # lock $db_lock; #concurrent writes to voip_numbers causes deadlocks + _update_subscriber($context); + _create_aliases($context); + #} + _update_preferences($context); + #_set_registrations($context); + #_set_callforwards($context); + #todo: additional prefs, AllowedIPs, NCOS, Callforwards. still thinking wether to integrate it + #in this main provisioning loop, or align it in separate run-modes, according to the files given. + + } else { + _warn($context,(scalar @$existing_billing_voip_subscribers) . ' existing billing subscribers found, skipping'); + } + + if ($dry) { + $context->{db}->db_rollback(0); + } else { + $context->{db}->db_commit(); + } + + }; + my $err = $@; + if ($err) { + eval { + $context->{db}->db_rollback(1); + }; + if ($skip_errors) { + _warn($context, $err); + } else { + _error($context, $err); + } + } + + return 1; + +} + +sub _provision_subscribers_checks { + my ($context) = @_; + + my $result = 1; + + my $subscribercount = 0; + eval { + $subscribercount = NGCP::BulkProcessor::Projects::Migration::UPCAT::Dao::import::Subscriber::countby_ccacsn(); + }; + if ($@ or $subscribercount == 0) { + rowprocessingerror(threadid(),'please import subscribers first',getlogger(__PACKAGE__)); + $result = 0; #even in skip-error mode.. + } else { + processing_info(threadid(),"$subscribercount subscriber found",getlogger(__PACKAGE__)); + } + + my $domain_billingprofilename_resellernames = []; + eval { + $domain_billingprofilename_resellernames = NGCP::BulkProcessor::Projects::Migration::UPCAT::Dao::import::Subscriber::list_domain_billingprofilename_resellernames(); + }; + if ($@ or (scalar @$domain_billingprofilename_resellernames) == 0) { + rowprocessingerror(threadid(),"no domains/billing profile names/reseller names",getlogger(__PACKAGE__)); + $result = 0; #even in skip-error mode.. + } else { + $context->{domain_map} = {}; + $context->{reseller_map} = {}; + foreach my $domain_billingprofilename_resellername (@$domain_billingprofilename_resellernames) { + my $domain = $domain_billingprofilename_resellername->{domain}; + unless ($domain) { + rowprocessingerror(threadid(),"empty domain detected",getlogger(__PACKAGE__)); + $result = 0; #even in skip-error mode.. + } + my $billingprofilename = $domain_billingprofilename_resellername->{billing_profile_name}; + unless ($billingprofilename) { + rowprocessingerror(threadid(),"empty billing profile name detected",getlogger(__PACKAGE__)); + $result = 0; #even in skip-error mode.. + } + my $resellername = _apply_reseller_mapping($domain_billingprofilename_resellername->{reseller_name}); + unless ($resellername) { + rowprocessingerror(threadid(),"empty reseller name detected",getlogger(__PACKAGE__)); + $result = 0; #even in skip-error mode.. + } + if (not exists $context->{reseller_map}->{$resellername}) { + eval { + $context->{reseller_map}->{$resellername} = NGCP::BulkProcessor::Dao::Trunk::billing::resellers::findby_name($resellername); + }; + if ($@ or not $context->{reseller_map}->{$resellername}) { + rowprocessingerror(threadid(),"cannot find reseller '$resellername'",getlogger(__PACKAGE__)); + $result = 0; #even in skip-error mode.. + } else { + $context->{reseller_map}->{$resellername}->{billingprofile_map} = {}; + processing_info(threadid(),"reseller '$resellername' found",getlogger(__PACKAGE__)); + } + } + if (not exists $context->{domain_map}->{$domain}) { + eval { + $context->{domain_map}->{$domain} = NGCP::BulkProcessor::Dao::Trunk::billing::domains::findby_domain($domain); + }; + if ($@ or not $context->{domain_map}->{$domain}) { + rowprocessingerror(threadid(),"cannot find billing domain '$domain'",getlogger(__PACKAGE__)); + $result = 0; #even in skip-error mode.. + } else { + processing_info(threadid(),"billing domain '$domain' found",getlogger(__PACKAGE__)); + eval { + $context->{domain_map}->{$domain}->{prov_domain} = + NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_domains::findby_domain($domain); + }; + if ($@ or not $context->{domain_map}->{$domain}->{prov_domain}) { + rowprocessingerror(threadid(),"cannot find provisioning domain '$domain'",getlogger(__PACKAGE__)); + $result = 0; #even in skip-error mode.. + } else { + processing_info(threadid(),"provisioning domain '$domain' found",getlogger(__PACKAGE__)); + } + } + } + my $domain_reseller; + eval { + $domain_reseller = NGCP::BulkProcessor::Dao::Trunk::billing::domain_resellers::findby_domainid_resellerid( + $context->{domain_map}->{$domain}->{id}, + $context->{reseller_map}->{$resellername}->{id})->[0]; + }; + if ($@ or not $domain_reseller) { + rowprocessingerror(threadid(),"domain $domain does not belong to reseller $resellername",getlogger(__PACKAGE__)); + $result = 0; #even in skip-error mode.. + } else { + processing_info(threadid(),"domain $domain belongs to reseller $resellername",getlogger(__PACKAGE__)); + } + + if ($context->{reseller_map}->{$resellername}->{billingprofile_map} and + not exists $context->{reseller_map}->{$resellername}->{billingprofile_map}->{$billingprofilename}) { + + eval { + $context->{reseller_map}->{$resellername}->{billingprofile_map}->{$billingprofilename} = + NGCP::BulkProcessor::Dao::Trunk::billing::billing_profiles::findby_resellerid_name_handle( + $context->{reseller_map}->{$resellername}->{id}, + $billingprofilename, + )->[0]; + }; + if ($@ or not $context->{reseller_map}->{$resellername}->{billingprofile_map}->{$billingprofilename}) { + rowprocessingerror(threadid(),"cannot find billing profile '$billingprofilename' of reseller '$resellername'",getlogger(__PACKAGE__)); + $result = 0; #even in skip-error mode.. + } else { + processing_info(threadid(),"billing profile '$billingprofilename' of reseller '$resellername' found",getlogger(__PACKAGE__)); + } + } + } + } + + eval { + $context->{sip_account_product} = NGCP::BulkProcessor::Dao::Trunk::billing::products::findby_resellerid_handle(undef, + $NGCP::BulkProcessor::Dao::Trunk::billing::products::SIP_ACCOUNT_HANDLE)->[0]; + }; + if ($@ or not defined $context->{sip_account_product}) { + rowprocessingerror(threadid(),"cannot find $NGCP::BulkProcessor::Dao::Trunk::billing::products::SIP_ACCOUNT_HANDLE product",getlogger(__PACKAGE__)); + $result = 0; #even in skip-error mode.. + } else { + processing_info(threadid(),"$NGCP::BulkProcessor::Dao::Trunk::billing::products::SIP_ACCOUNT_HANDLE product found",getlogger(__PACKAGE__)); + } + + $context->{attributes} = {}; + + eval { + $context->{attributes}->{allowed_clis} = NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_preferences::findby_attribute( + $NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_preferences::ALLOWED_CLIS_ATTRIBUTE); + }; + if ($@ or not defined $context->{attributes}->{allowed_clis}) { + rowprocessingerror(threadid(),'cannot find allowed_clis attribute',getlogger(__PACKAGE__)); + $result = 0; #even in skip-error mode.. + } else { + processing_info(threadid(),"allowed_clis attribute found",getlogger(__PACKAGE__)); + } + + eval { + $context->{attributes}->{cli} = NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_preferences::findby_attribute( + $NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_preferences::CLI_ATTRIBUTE); + }; + if ($@ or not defined $context->{attributes}->{cli}) { + rowprocessingerror(threadid(),'cannot find cli attribute',getlogger(__PACKAGE__)); + $result = 0; #even in skip-error mode.. + } else { + processing_info(threadid(),"cli attribute found",getlogger(__PACKAGE__)); + } + + eval { + $context->{attributes}->{ac} = NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_preferences::findby_attribute( + $NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_preferences::AC_ATTRIBUTE); + }; + if ($@ or not defined $context->{attributes}->{ac}) { + rowprocessingerror(threadid(),'cannot find ac attribute',getlogger(__PACKAGE__)); + $result = 0; #even in skip-error mode.. + } else { + processing_info(threadid(),"ac attribute found",getlogger(__PACKAGE__)); + } + + eval { + $context->{attributes}->{cc} = NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_preferences::findby_attribute( + $NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_preferences::CC_ATTRIBUTE); + }; + if ($@ or not defined $context->{attributes}->{cc}) { + rowprocessingerror(threadid(),'cannot find cc attribute',getlogger(__PACKAGE__)); + $result = 0; #even in skip-error mode.. + } else { + processing_info(threadid(),"cc attribute found",getlogger(__PACKAGE__)); + } + + eval { + $context->{attributes}->{account_id} = NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_preferences::findby_attribute( + $NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_preferences::ACCOUNT_ID_ATTRIBUTE); + }; + if ($@ or not defined $context->{attributes}->{account_id}) { + rowprocessingerror(threadid(),'cannot find account_id attribute',getlogger(__PACKAGE__)); + $result = 0; #even in skip-error mode.. + } else { + processing_info(threadid(),"account_id attribute found",getlogger(__PACKAGE__)); + } + + my $barring_resellernames = []; + eval { + $barring_resellernames = NGCP::BulkProcessor::Projects::Migration::UPCAT::Dao::import::Subscriber::list_barring_resellernames(); + }; + if ($@) { + rowprocessingerror(threadid(),'error retrieving barrings',getlogger(__PACKAGE__)); + $result = 0; #even in skip-error mode.. + } else { + $context->{ncos_level_map} = {}; + foreach my $barring_resellername (@$barring_resellernames) { + my $resellername = _apply_reseller_mapping($barring_resellername->{reseller_name}); + #unless ($resellername) { + # rowprocessingerror(threadid(),"empty reseller name detected",getlogger(__PACKAGE__)); + # $result = 0; #even in skip-error mode.. + #} + my $barring = $barring_resellername->{barrings}; + $barring = $default_barring unless ($barring); + $result &= _check_ncos_level($context,$resellername,$barring); + } + } + + eval { + $context->{attributes}->{ncos_id} = NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_preferences::findby_attribute( + $NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_preferences::NCOS_ID_ATTRIBUTE); + }; + if ($@ or not defined $context->{attributes}->{ncos_id}) { + rowprocessingerror(threadid(),'cannot find ncos_id attribute',getlogger(__PACKAGE__)); + $result = 0; #even in skip-error mode.. + } else { + processing_info(threadid(),"ncos_id attribute found",getlogger(__PACKAGE__)); + } + + foreach my $gpp_idx (0..9) { + my $gpp_attr = $NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_preferences::GPPx_ATTRIBUTE . $gpp_idx; + eval { + $context->{attributes}->{$gpp_attr} = NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_preferences::findby_attribute($gpp_attr); + }; + if ($@ or not defined $context->{attributes}->{$gpp_attr}) { + rowprocessingerror(threadid(),"cannot find $gpp_attr attribute",getlogger(__PACKAGE__)); + $result = 0; #even in skip-error mode.. + } else { + processing_info(threadid(),"$gpp_attr attribute found",getlogger(__PACKAGE__)); + } + } + + return $result; +} + +sub _check_ncos_level { + my ($context,$resellername,$barring) = @_; + my $result = 1; + if ($barring ne $default_barring and not exists $barring_profiles->{$resellername}) { + rowprocessingerror(threadid(),"barring mappings for reseller $resellername missing",getlogger(__PACKAGE__)); + $result = 0; #even in skip-error mode.. + } elsif ($barring ne $default_barring and not exists $barring_profiles->{$resellername}->{$barring}) { + rowprocessingerror(threadid(),"mappings for barring '" . $barring . "' of reseller $resellername missing",getlogger(__PACKAGE__)); + $result = 0; #even in skip-error mode.. + } else { + my $reseller_id = $context->{reseller_map}->{$resellername}->{id}; + $context->{ncos_level_map}->{$reseller_id} = {} unless exists $context->{ncos_level_map}->{$reseller_id}; + my $level = $barring_profiles->{$resellername}->{$barring}; + unless (exists $context->{ncos_level_map}->{$reseller_id}->{$barring}) { + if (not defined $level or length($level) == 0) { + $context->{ncos_level_map}->{$reseller_id}->{$barring} = undef; + } else { + eval { + $context->{ncos_level_map}->{$reseller_id}->{$barring} = NGCP::BulkProcessor::Dao::Trunk::billing::ncos_levels::findby_resellerid_level( + $reseller_id,$level); + }; + if ($@ or not defined $context->{ncos_level_map}->{$reseller_id}->{$barring}) { + my $err = "cannot find ncos level '$level' of reseller $resellername"; + if (not defined $context->{_rowcount}) { + if ($barring ne $default_barring) { + rowprocessingerror(threadid(),$err,getlogger(__PACKAGE__)); + $result = 0; #even in skip-error mode.. + } else { + rowprocessingwarn(threadid(),$err,getlogger(__PACKAGE__)); + } + } elsif ($skip_errors) { + _warn($context, $err); + } else { + _error($context, $err); + $result = 0; #even in skip-error mode.. + } + } else { + processing_info(threadid(),"ncos level '$level' of reseller $resellername found",getlogger(__PACKAGE__)); + } + } + } + } + return $result; +} + +sub _update_contact { + + my ($context) = @_; + + my $existing_contracts = NGCP::BulkProcessor::Dao::Trunk::billing::contracts::findby_externalid($context->{contract}->{external_id}); + if ((scalar @$existing_contracts) > 0) { + my $existing_contract = $existing_contracts->[0]; + if ((scalar @$existing_contracts) > 1) { + _warn($context,(scalar @$existing_contracts) . " existing contracts found, using first contact id $existing_contract->{id}"); + } else { + _info($context,"existing contract id $existing_contract->{id} found",1); + } + $context->{contract}->{id} = $existing_contract->{id}; + $context->{bill_subscriber}->{contract_id} = $context->{contract}->{id}; + $context->{prov_subscriber}->{account_id} = $context->{contract}->{id}; + } else { + #_warn($context,"no existing contract of contact id $existing_contact->{id} found, will be created"); + + $context->{contract}->{contact}->{id} = NGCP::BulkProcessor::Dao::Trunk::billing::contacts::insert_row($context->{db}, + $context->{contract}->{contact}, + ); + $context->{contract}->{contact_id} = $context->{contract}->{contact}->{id}; + _info($context,"contact id $context->{contract}->{contact}->{id} created",1); + } + + return 1; + +} + +sub _update_contract { + + my ($context) = @_; + + if ($context->{bill_subscriber}->{contract_id}) { + #todo: the update case + } else { + #the insert case + $context->{contract}->{id} = NGCP::BulkProcessor::Dao::Trunk::billing::contracts::insert_row($context->{db}, + $context->{contract} + ); + $context->{bill_subscriber}->{contract_id} = $context->{contract}->{id}; + $context->{prov_subscriber}->{account_id} = $context->{contract}->{id}; + + NGCP::BulkProcessor::Dao::Trunk::billing::contracts_billing_profile_network_schedule::append_billing_mappings($context->{db}, + $context->{contract}->{id}, + [{ billing_profile_id => $context->{billing_profile}->{id}, }], + ); + + $context->{contract}->{contract_balance_id} = NGCP::BulkProcessor::Dao::Trunk::billing::contract_balances::insert_row($context->{db}, + contract_id => $context->{contract}->{id}, + ); + + _info($context,"contract id $context->{contract}->{id} created",1); + } + return 1; + +} + +sub _update_subscriber { + + my ($context) = @_; + + my $result = 1; + + if ($context->{bill_subscriber}->{id}) { + #todo: the update case + } else { + #the insert case + $context->{bill_subscriber}->{id} = NGCP::BulkProcessor::Dao::Trunk::billing::voip_subscribers::insert_row($context->{db}, + $context->{bill_subscriber}, + ); + + $context->{prov_subscriber}->{id} = NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_subscribers::insert_row($context->{db}, + $context->{prov_subscriber}, + ); + + my $number = $context->{numbers}->{primary}; + $context->{voip_numbers}->{primary} = NGCP::BulkProcessor::Dao::Trunk::billing::voip_numbers::forupdate_cc_ac_sn_subscriberid($context->{db}, + $number->{cc}, + $number->{ac}, + $number->{sn}, + $context->{bill_subscriber}->{id}); + + if (defined $context->{voip_numbers}->{primary}) { + NGCP::BulkProcessor::Dao::Trunk::billing::voip_numbers::update_row($context->{db},{ + id => $context->{voip_numbers}->{primary}->{id}, + reseller_id => $context->{reseller}->{id}, + subscriber_id => $context->{bill_subscriber}->{id}, + status => 'active', + }); + } else { + $context->{voip_numbers}->{primary}->{id} = NGCP::BulkProcessor::Dao::Trunk::billing::voip_numbers::insert_row($context->{db}, + cc => $number->{cc}, + ac => $number->{ac}, + sn => $number->{sn}, + reseller_id => $context->{reseller}->{id}, + subscriber_id => $context->{bill_subscriber}->{id}, + ); + } + + $context->{preferences}->{cli} = { id => set_subscriber_preference($context, + $context->{prov_subscriber}->{id}, + $context->{attributes}->{cli}, + $number->{number}), value => $number->{number} }; + + NGCP::BulkProcessor::Dao::Trunk::billing::voip_subscribers::update_row($context->{db},{ + id => $context->{bill_subscriber}->{id}, + primary_number_id => $context->{voip_numbers}->{primary}->{id}, + }); + + _info($context,"subscriber uuid $context->{prov_subscriber}->{uuid} created",1); + + #primary alias + $context->{aliases}->{primary}->{id} = NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_dbaliases::insert_row($context->{db}, + domain_id => $context->{prov_subscriber}->{domain_id}, + subscriber_id => $context->{prov_subscriber}->{id}, + username => $number->{number}, + ); + + my @allowed_clis = (); + push(@allowed_clis,{ id => set_subscriber_preference($context, + $context->{prov_subscriber}->{id}, + $context->{attributes}->{allowed_clis}, + $number->{number}), value => $number->{number}}); + $context->{preferences}->{allowed_clis} = \@allowed_clis; + + NGCP::BulkProcessor::Dao::Trunk::billing::voip_numbers::release_subscriber_numbers($context->{db}, + $context->{bill_subscriber}->{id},{ 'NOT IN' => $context->{voip_numbers}->{primary}->{id} }); + + NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_dbaliases::delete_dbaliases($context->{db}, + $context->{prov_subscriber}->{id},{ 'NOT IN' => $number->{number} }); + + clear_subscriber_preferences($context, + $context->{prov_subscriber}->{id}, + $context->{attributes}->{allowed_clis}, + $number->{number}); + + _info($context,"primary alias $number->{number} created",1); + + $context->{voicemail_user}->{id} = NGCP::BulkProcessor::Dao::Trunk::kamailio::voicemail_users::insert_row($context->{db}, + $context->{voicemail_user}, + ); + + $context->{preferences}->{account_id} = { id => set_subscriber_preference($context, + $context->{prov_subscriber}->{id}, + $context->{attributes}->{account_id}, + $context->{contract}->{id}), value => $context->{contract}->{id} }; + + if (length($number->{ac}) > 0) { + $context->{preferences}->{ac} = { id => set_subscriber_preference($context, + $context->{prov_subscriber}->{id}, + $context->{attributes}->{ac}, + $number->{ac}), value => $number->{ac} }; + } + if (length($number->{cc}) > 0) { + $context->{preferences}->{cc} = { id => set_subscriber_preference($context, + $context->{prov_subscriber}->{id}, + $context->{attributes}->{cc}, + $number->{cc}), value => $number->{cc} }; + } + + } + + return $result; + +} + +sub _update_preferences { + + my ($context) = @_; + + my $result = 1; + + if (defined $context->{ncos_level}) { + $context->{preferences}->{ncos_id} = { id => set_subscriber_preference($context, + $context->{prov_subscriber}->{id}, + $context->{attributes}->{ncos_id}, + $context->{ncos_level}->{id}), value => $context->{ncos_level}->{id} }; + _info($context,"ncos_id preference set to $context->{ncos_level}->{id} - $context->{ncos_level}->{level}",1); + } + + if (defined $context->{preferences}->{gpp}) { + my $gpp_idx = 0; + foreach my $gpp_val (@{$context->{preferences}->{gpp}}) { + my $gpp_attr = 'gpp' . $gpp_idx; + $context->{preferences}->{$gpp_attr} = { id => set_subscriber_preference($context, + $context->{prov_subscriber}->{id}, + $context->{attributes}->{$gpp_attr}, + $gpp_attr), value => $gpp_attr }; + _info($context,"$gpp_attr preference set to $gpp_val",1); + $gpp_idx++; + } + } + + return $result; + +} + +sub _create_aliases { + + my ($context) = @_; + my $result = 1; + + if ((scalar @{$context->{numbers}->{other}}) > 0) { + + my @voip_number_ids = (); + my @usernames = (); + + foreach my $number (@{$context->{numbers}->{other}}) { + + my $voip_number = NGCP::BulkProcessor::Dao::Trunk::billing::voip_numbers::forupdate_cc_ac_sn_subscriberid($context->{db}, + $number->{cc}, + $number->{ac}, + $number->{sn}, + $context->{bill_subscriber}->{id}); + + if (defined $voip_number) { + NGCP::BulkProcessor::Dao::Trunk::billing::voip_numbers::update_row($context->{db},{ + id => $voip_number->{id}, + reseller_id => $context->{reseller}->{id}, + subscriber_id => $context->{bill_subscriber}->{id}, + status => 'active', + }); + } else { + $voip_number->{id} = NGCP::BulkProcessor::Dao::Trunk::billing::voip_numbers::insert_row($context->{db}, + cc => $number->{cc}, + ac => $number->{ac}, + sn => $number->{sn}, + reseller_id => $context->{reseller}->{id}, + subscriber_id => $context->{bill_subscriber}->{id}, + ); + } + + push(@{$context->{voip_numbers}->{other}}, $voip_number); + push(@voip_number_ids, $voip_number->{id}); + + my $alias; + if ($alias = NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_dbaliases::findby_subscriberid_username($context->{db}, + $context->{prov_subscriber}->{id}, + $number->{number}, + )->[0]) { + NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_dbaliases::update_row($context->{db},{ + id => $alias->{id}, + is_primary => '0', + }); + $alias->{is_primary} = '0'; + } else { + $alias->{id} = NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_dbaliases::insert_row($context->{db},{ + domain_id => $context->{prov_subscriber}->{domain_id}, + subscriber_id => $context->{prov_subscriber}->{id}, + is_primary => '0', + username => $number->{number}, + }); + } + + push(@{$context->{aliases}->{other}},$alias); + push(@usernames,$number->{number}); + + delete_subscriber_preference($context, + $context->{prov_subscriber}->{id}, + $context->{attributes}->{allowed_clis}, + $number->{number}); + push(@{$context->{preferences}->{allowed_clis}},{ id => set_subscriber_preference($context, + $context->{prov_subscriber}->{id}, + $context->{attributes}->{allowed_clis}, + $number->{number}), value => $number->{number}}); + + _info($context,"alias $number->{number} created",1); + } + + push(@voip_number_ids,$context->{voip_numbers}->{primary}->{id}); + push(@usernames,$context->{numbers}->{primary}->{number}); + + NGCP::BulkProcessor::Dao::Trunk::billing::voip_numbers::release_subscriber_numbers($context->{db}, + $context->{bill_subscriber}->{id},{ 'NOT IN' => \@voip_number_ids }); + + NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_dbaliases::delete_dbaliases($context->{db},$context->{prov_subscriber}->{id}, + { 'NOT IN' => \@usernames }); + + clear_subscriber_preferences($context, + $context->{prov_subscriber}->{id}, + $context->{attributes}->{allowed_clis}, + \@usernames ); + + #test: + #my $allowed_clis = get_subscriber_preference($context, + # $context->{prov_subscriber}->{id}, + # $context->{attributes}->{allowed_clis}); + + #my $voip_numbers = NGCP::BulkProcessor::Dao::Trunk::billing::voip_numbers::findby_subscriberid($context->{db}, + # $context->{bill_subscriber}->{id}); + + #my $aliases = NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_dbaliases::findby_subscriberid_username($context->{db}, + # $context->{prov_subscriber}->{id},undef); + + #_info($context,(scalar @{$context->{numbers}->{other}}) . " aliases created: " . join(',',(map { $_->{number}; } @{$context->{numbers}->{other}}))); + } + return $result; +} + +sub _provision_susbcriber_init_context { + + my ($context,$subscriber_group) = @_; + + my $result = 1; + + $context->{log_info} = []; + $context->{log_warning} = []; + $context->{log_error} = []; + + my $first = $subscriber_group->[0]; + + unless (defined $first->{sip_username} and length($first->{sip_username}) > 0) { + _warn($context,'empty sip_username ignored'); + $result = 0; + } + + $context->{domain} = $context->{domain_map}->{$first->{domain}}; + my $resellername = _apply_reseller_mapping($first->{reseller_name}); + $context->{reseller} = $context->{reseller_map}->{$first->{reseller_name}}; + $context->{billing_profile} = $context->{reseller}->{billingprofile_map}->{$first->{billing_profile_name}}; + + $context->{prov_subscriber} = {}; + $context->{prov_subscriber}->{username} = $first->{sip_username}; + $context->{prov_subscriber}->{password} = $first->{sip_password}; + $context->{prov_subscriber}->{webusername} = $first->{web_username}; + $context->{prov_subscriber}->{webpassword} = $first->{web_password}; + my $webusername = $first->{web_username}; + + $context->{prov_subscriber}->{uuid} = create_uuid(); + $context->{prov_subscriber}->{domain_id} = $context->{domain}->{prov_domain}->{id}; + + $context->{bill_subscriber} = {}; + $context->{bill_subscriber}->{username} = $first->{sip_username}; + $context->{bill_subscriber}->{domain_id} = $context->{domain}->{id}; + $context->{bill_subscriber}->{uuid} = $context->{prov_subscriber}->{uuid}; + + undef $context->{contract}; + #undef $context->{channels}; + + my @numbers = (); + my %number_dupes = (); + my %contract_dupes = (); + my %barrings = (); + #my $voicemail = 0; + foreach my $subscriber (@$subscriber_group) { + my $number = $subscriber->{cc} . $subscriber->{ac} . $subscriber->{sn}; + if (not exists $number_dupes{$number}) { + push(@numbers,{ + cc => $subscriber->{cc}, + ac => $subscriber->{ac}, + sn => $subscriber->{sn}, + number => $number, + #delta => $subscriber->{delta}, + additional => 0, + filename => $subscriber->{filename}, + }); + $number_dupes{$number} = 1; + } else { + _warn($context,"duplicate number $number ($subscriber->{filename}) ignored"); + } + + if (not exists $contract_dupes{$subscriber->{customer_id}}) { + if (not $context->{contract}) { + $context->{contract} = { + external_id => $subscriber->{customer_id}, + create_timestamp => $context->{now}, + product_id => $context->{sip_account_product}->{id}, + contact => { + reseller_id => $context->{reseller}->{id}, + + firstname => $subscriber->{first_name}, + lastname => $subscriber->{last_name}, + compregnum => $subscriber->{company_registration_number}, + company => $subscriber->{company}, + street => $subscriber->{street}, + postcode => $subscriber->{postal_code}, + city => $subscriber->{city_name}, + #country => $context->{contract}->{contact}->{country}, + phonenumber => $subscriber->{phone_number}, + email => $subscriber->{email}, + vatnum => $subscriber->{vat_number}, + #$contact_hash_field => $subscriber->{contact_hash}, + }, + }; + $contract_dupes{$subscriber->{customer_id}} = 1; + } else { + _warn($context,'non-unique contact data, skipped'); + $context->{nonunique_contacts}->{$context->{prov_subscriber}->{username}} += 1; + $result = 0; + } + } + + unless (defined $context->{prov_subscriber}->{password} and length($context->{prov_subscriber}->{password}) > 0) { + $context->{prov_subscriber}->{password} = $subscriber->{sip_password}; + } + + unless (defined $context->{prov_subscriber}->{webusername} and length($context->{prov_subscriber}->{webusername}) > 0 + and defined $context->{prov_subscriber}->{webpassword} and length($context->{prov_subscriber}->{webpassword}) > 0) { + $context->{prov_subscriber}->{webusername} = $subscriber->{web_username}; + $context->{prov_subscriber}->{webpassword} = $subscriber->{web_password}; + } + + unless (defined $webusername and length($webusername) > 0) { + $webusername = $subscriber->{web_username}; + } + + if (defined $subscriber->{barrings} and length($subscriber->{barrings}) > 0) { + $barrings{$subscriber->{barrings}} = 1; + } + + } + + unless (defined $context->{prov_subscriber}->{password} and length($context->{prov_subscriber}->{password}) > 0) { + my $generated = _generate_sippassword(); + $context->{prov_subscriber}->{password} = $generated; + _info($context,"empty sip_password, using generated '$generated'",1); + } + + unless (defined $context->{prov_subscriber}->{webusername} and length($context->{prov_subscriber}->{webusername}) > 0) { + $context->{prov_subscriber}->{webusername} = $webusername; + $context->{prov_subscriber}->{webpassword} = undef; + } + + if (not (defined $context->{prov_subscriber}->{webusername} and length($context->{prov_subscriber}->{webusername}) > 0)) { + $context->{prov_subscriber}->{webusername} = undef; + $context->{prov_subscriber}->{webpassword} = undef; + _info($context,"empty web_username for sip_username '$first->{sip_username}'",1); + } else { + $webusername = $context->{prov_subscriber}->{webusername}; + my %webusername_dupes = map { $_->{sip_username} => 1; } + @{NGCP::BulkProcessor::Projects::Migration::UPCAT::Dao::import::Subscriber::findby_domain_webusername( + $first->{domain},$webusername)}; + if ((scalar keys %webusername_dupes) > 1) { + my $generated = _generate_webusername(); #$first->{sip_username}; + _info($context,"duplicate web_username '$webusername', using generated '$generated'",1); + $context->{prov_subscriber}->{webusername} = $generated; + } + + #$context->{prov_subscriber}->{webpassword} = $first->{web_password}; + if (not (defined $context->{prov_subscriber}->{webpassword} and length($context->{prov_subscriber}->{webpassword}) > 0)) { + my $generated = _generate_webpassword(); + _info($context,"empty web_password for web_username '$webusername', using generated '$generated'",1); + $context->{prov_subscriber}->{webpassword} = $generated; + #} elsif (defined $first->{web_password} and length($first->{web_password}) < 8) { + # $context->{prov_subscriber}->{webpassword} = _generate_webpassword(); + # _info($context,"web_password for web_username '$first->{web_username}' is too short, using '$context->{prov_subscriber}->{webpassword}'"); + } + } + + $context->{ncos_level} = undef; + if ((scalar keys %barrings) > 1) { + my $combined_barring = join('_',sort keys %barrings); + #$result &= + _check_ncos_level($context,$resellername,$combined_barring); + _info($context,"barrings combination $combined_barring"); + $context->{ncos_level} = $context->{ncos_level_map}->{$context->{reseller}->{id}}->{$combined_barring}; + } elsif ((scalar keys %barrings) == 1) { + my ($barring) = keys %barrings; + $context->{ncos_level} = $context->{ncos_level_map}->{$context->{reseller}->{id}}->{$barring}; + } else { + if (exists $context->{ncos_level_map}->{$context->{reseller}->{id}}->{$default_barring}) { + $context->{ncos_level} = $context->{ncos_level_map}->{$context->{reseller}->{id}}->{$default_barring}; + _info($context,"no ncos level, using default '$context->{ncos_level}->{level}'",1); + } + } + + $context->{numbers} = {}; + $context->{numbers}->{other} = sort_by_configs(\@numbers,[ + { numeric => 1, + dir => 1, #-1, + memberchain => [ 'additional' ], + }, + { numeric => 0, + dir => 1, #-1, + memberchain => [ 'cc' ], + }, + { numeric => 0, + dir => 1, #-1, + memberchain => [ 'ac' ], + }, + { numeric => 0, + dir => 1, #-1, + memberchain => [ 'sn' ], + }, + ]); + $context->{numbers}->{primary} = shift(@{$context->{numbers}->{other}}); + #return 0 unless scalar @{$context->{numbers}->{other}}; + + $context->{voip_numbers} = {}; + $context->{voip_numbers}->{primary} = undef; + $context->{voip_numbers}->{other} = []; + $context->{aliases} = {}; + $context->{aliases}->{primary} = undef; + $context->{aliases}->{other} = []; + + $context->{voicemail_user} = {}; + $context->{voicemail_user}->{customer_id} = $context->{prov_subscriber}->{uuid}; + $context->{voicemail_user}->{mailbox} = $context->{numbers}->{primary}->{number}; + $context->{voicemail_user}->{password} = sprintf("%04d", int(rand 10000)); + + $context->{preferences} = {}; + + $context->{preferences}->{gpp} = [ + $first->{"_len"}, + $first->{"_cpe_mta_mac_address"}, + $first->{"_cpe_model"}, + $first->{"_cpe_vendor"}, + ]; + + return $result; + +} + + +sub _generate_webpassword { + return String::MkPasswd::mkpasswd( + -length => $webpassword_length, + -minnum => 1, -minlower => 1, -minupper => 1, -minspecial => 1, + -distribute => 1, -fatal => 1, + ); +} + +sub _generate_sippassword { + return createtmpstring($sippassword_length); +} + +sub _generate_webusername { + return createtmpstring($webusername_length); +} + +sub _apply_reseller_mapping { + my $reseller_name = shift; + #if (defined $reseller_name and exists $reseller_mapping->{$reseller_name}) { + # return $reseller_mapping->{$reseller_name}; + #} + return $reseller_name; +} + +sub _error { + + my ($context,$message) = @_; + $context->{error_count} = $context->{error_count} + 1; + push(@{$context->{log_error}},$message) if exists $context->{log_error}; + if ($context->{prov_subscriber}) { + $message = ($context->{prov_subscriber}->{username} ? $context->{prov_subscriber}->{username} : '') . ': ' . $message; + } + rowprocessingerror($context->{tid},$message,getlogger(__PACKAGE__)); + +} + +sub _warn { + + my ($context,$message) = @_; + $context->{warning_count} = $context->{warning_count} + 1; + push(@{$context->{log_warning}},$message) if exists $context->{log_warning}; + if ($context->{prov_subscriber}) { + $message = ($context->{prov_subscriber}->{username} ? $context->{prov_subscriber}->{username} : '') . ': ' . $message; + } + rowprocessingwarn($context->{tid},$message,getlogger(__PACKAGE__)); + +} + +sub _info { + + my ($context,$message,$debug) = @_; + push(@{$context->{log_info}},$message) if exists $context->{log_info}; + if ($context->{prov_subscriber}) { + $message = ($context->{prov_subscriber}->{username} ? $context->{prov_subscriber}->{username} : '') . ': ' . $message; + } + if ($debug) { + processing_debug($context->{tid},$message,getlogger(__PACKAGE__)); + } else { + processing_info($context->{tid},$message,getlogger(__PACKAGE__)); + } +} + +1; diff --git a/lib/NGCP/BulkProcessor/Projects/Migration/UPCAT/Settings.pm b/lib/NGCP/BulkProcessor/Projects/Migration/UPCAT/Settings.pm new file mode 100755 index 0000000..8626a78 --- /dev/null +++ b/lib/NGCP/BulkProcessor/Projects/Migration/UPCAT/Settings.pm @@ -0,0 +1,325 @@ +package NGCP::BulkProcessor::Projects::Migration::UPCAT::Settings; +use strict; + +## no critic + +use NGCP::BulkProcessor::Globals qw( + $working_path + $enablemultithreading + $cpucount + create_path +); + +use NGCP::BulkProcessor::Logging qw( + getlogger + scriptinfo + configurationinfo +); + +use NGCP::BulkProcessor::LogError qw( + fileerror + filewarn + configurationwarn + configurationerror +); + +use NGCP::BulkProcessor::LoadConfig qw( + split_tuple + parse_regexp +); +use NGCP::BulkProcessor::Utils qw(prompt timestampdigits); +#format_number check_ipnet + +require Exporter; +our @ISA = qw(Exporter); +our @EXPORT_OK = qw( + update_settings + update_cc_ac_map + update_barring_profiles + check_dry + + $input_path + $output_path + $report_filename + + $defaultsettings + $defaultconfig + + $import_multithreading + $run_id + $dry + $skip_errors + $force + $import_db_file + + @subscriber_filenames + $subscriber_import_numofthreads + $ignore_subscriber_unique + $subscriber_import_single_row_txn + + $provision_subscriber_rownum_start + $provision_subscriber_multithreading + $provision_subscriber_numofthreads + $webpassword_length + $webusername_length + $sippassword_length + + $default_domain + $default_reseller_name + $default_billing_profile_name + $default_barring + + $cc_ac_map_yml + $cc_ac_map + $default_cc + $cc_len_min + $cc_len_max + $ac_len + + $barring_profiles_yml + $barring_profiles +); +#$default_channels_map + +our $defaultconfig = 'config.cfg'; +our $defaultsettings = 'settings.cfg'; + +our $input_path = $working_path . 'input/'; +our $output_path = $working_path . 'output/'; +our $report_filename = undef; + +our $force = 0; +our $dry = 0; +our $skip_errors = 0; +our $run_id = ''; +our $import_db_file = _get_import_db_file($run_id,'import'); +our $import_multithreading = 0; #$enablemultithreading; + +our @subscriber_filenames = (); +our $subscriber_import_numofthreads = $cpucount; +our $ignore_subscriber_unique = 0; +our $subscriber_import_single_row_txn = 1; + +our $provision_subscriber_rownum_start = 0; #all lines +our $provision_subscriber_multithreading = $enablemultithreading; +our $provision_subscriber_numofthreads = $cpucount; +our $webpassword_length = 8; +our $webusername_length = 8; +our $sippassword_length = 16; + +our $default_domain = undef; +our $default_reseller_name = 'default'; +our $default_billing_profile_name = 'Default Billing Profile'; +our $default_barring = undef; + +our $cc_ac_map_yml = 'cc_ac.yml'; +our $cc_ac_map = {}; +our $default_cc = undef; +our $cc_len_min = ~0; +our $cc_len_max = 0; +our $ac_len = {}; + +our $barring_profiles_yml = undef; +our $barring_profiles = {}; + +sub update_settings { + + my ($data,$configfile) = @_; + + if (defined $data) { + + my $result = 1; + my $regexp_result; + + #&$configurationinfocode("testinfomessage",$configlogger); + + $result &= _prepare_working_paths(1); + if ($data->{report_filename}) { + $report_filename = $output_path . sprintf('/' . $data->{report_filename},timestampdigits()); + if (-e $report_filename and (unlink $report_filename) == 0) { + filewarn('cannot remove ' . $report_filename . ': ' . $!,getlogger(__PACKAGE__)); + $report_filename = undef; + } + } else { + $report_filename = undef; + } + + $dry = $data->{dry} if exists $data->{dry}; + $skip_errors = $data->{skip_errors} if exists $data->{skip_errors}; + $import_db_file = _get_import_db_file($run_id,'import'); + $import_multithreading = $data->{import_multithreading} if exists $data->{import_multithreading}; + #if ($import_multithreading) { + # configurationerror($configfile,"import_multithreading must be disabled to preserve record order",getlogger(__PACKAGE__)); + #} + + @subscriber_filenames = _get_import_filenames(\@subscriber_filenames,$data,'subscriber_filenames'); + $subscriber_import_numofthreads = _get_numofthreads($cpucount,$data,'subscriber_import_numofthreads'); + $ignore_subscriber_unique = $data->{ignore_subscriber_unique} if exists $data->{ignore_subscriber_unique}; + $subscriber_import_single_row_txn = $data->{subscriber_import_single_row_txn} if exists $data->{subscriber_import_single_row_txn}; + + $provision_subscriber_rownum_start = $data->{provision_subscriber_rownum_start} if exists $data->{provision_subscriber_rownum_start}; + $provision_subscriber_multithreading = $data->{provision_subscriber_multithreading} if exists $data->{provision_subscriber_multithreading}; + $provision_subscriber_numofthreads = _get_numofthreads($cpucount,$data,'provision_subscriber_numofthreads'); + $webpassword_length = $data->{webpassword_length} if exists $data->{webpassword_length}; + if (not defined $webpassword_length or $webpassword_length <= 7) { + configurationerror($configfile,'webpassword_length greater than 7 required',getlogger(__PACKAGE__)); + $result = 0; + } + $webusername_length = $data->{webusername_length} if exists $data->{webusername_length}; + if (not defined $webusername_length or $webusername_length <= 7) { + configurationerror($configfile,'webusername_length greater than 7 required',getlogger(__PACKAGE__)); + $result = 0; + } + $sippassword_length = $data->{sippassword_length} if exists $data->{sippassword_length}; + if (not defined $sippassword_length or $sippassword_length <= 7) { + configurationerror($configfile,'sippassword_length greater than 7 required',getlogger(__PACKAGE__)); + $result = 0; + } + #$default_channels = $data->{default_channels} if exists $data->{default_channels}; + + $default_domain = $data->{default_domain} if exists $data->{default_domain}; + $default_reseller_name = $data->{default_reseller_name} if exists $data->{default_reseller_name}; + $default_billing_profile_name = $data->{default_billing_profile_name} if exists $data->{default_billing_profile_name}; + $default_barring = $data->{default_barring} if exists $data->{default_barring}; + $cc_ac_map_yml = $data->{cc_ac_map_yml} if exists $data->{cc_ac_map_yml}; + $default_cc = $data->{default_cc} if exists $data->{default_cc}; + + $barring_profiles_yml = $data->{barring_profiles_yml} if exists $data->{barring_profiles_yml}; + + return $result; + + } + return 0; + +} + +sub update_cc_ac_map { + + my ($data,$configfile) = @_; + + if (defined $data) { + + my $result = 1; + + eval { + $cc_ac_map = $data; + }; + if ($@ or 'HASH' ne ref $cc_ac_map) { + $cc_ac_map //= {}; + configurationerror($configfile,'invalid cc ac map',getlogger(__PACKAGE__)); + $result = 0; + } else { + foreach my $cc (keys %$cc_ac_map) { + my $ac_map = $cc_ac_map->{$cc}; + $cc_len_min = length($cc) if length($cc) < $cc_len_min; + $cc_len_max = length($cc) if length($cc) > $cc_len_max; + $ac_len->{$cc} = { min => ~0, max => 0, }; + if ('HASH' ne ref $ac_map) { + configurationerror($configfile,"invalid $cc ac map",getlogger(__PACKAGE__)); + $result = 0; + } else { + foreach my $ac (keys %$ac_map) { + if ($ac_map->{$ac}) { # ac enabled + $ac_len->{$cc}->{min} = length($ac) if length($ac) < $ac_len->{$cc}->{min}; + $ac_len->{$cc}->{max} = length($ac) if length($ac) > $ac_len->{$cc}->{max}; + } else { + delete $ac_map->{$ac}; + } + } + } + } + } + + return $result; + } + return 0; + +} + +sub update_barring_profiles { + + my ($data,$configfile) = @_; + + if (defined $data) { + + my $result = 1; + + eval { + $barring_profiles = $data; #->{'mapping'}; + }; + if ($@ or 'HASH' ne ref $barring_profiles or (scalar keys %$barring_profiles) == 0) { + $barring_profiles //= {}; + configurationerror($configfile,'no barring mappings found',getlogger(__PACKAGE__)); + $result = 0; + } + + return $result; + } + return 0; + +} + +sub _prepare_working_paths { + + my ($create) = @_; + my $result = 1; + my $path_result; + + ($path_result,$input_path) = create_path($working_path . 'input',$input_path,$create,\&fileerror,getlogger(__PACKAGE__)); + $result &= $path_result; + ($path_result,$output_path) = create_path($working_path . 'output',$output_path,$create,\&fileerror,getlogger(__PACKAGE__)); + $result &= $path_result; + + return $result; + +} + +sub _get_numofthreads { + my ($default_value,$data,$key) = @_; + my $numofthreads = $default_value; + $numofthreads = $data->{$key} if exists $data->{$key}; + $numofthreads = $cpucount if $numofthreads > $cpucount; + return $numofthreads; +} + +sub _get_import_db_file { + my ($run,$name) = @_; + return ((defined $run and length($run) > 0) ? $run . '_' : '') . $name; +} + +sub _get_import_filenames { + my ($old_value,$data,$key) = @_; + my @import_filenames = @$old_value; + @import_filenames = split_tuple($data->{$key}) if exists $data->{$key}; + my @result = (); + foreach my $import_filename (@import_filenames) { + if (defined $import_filename and length($import_filename) > 0) { + $import_filename = $input_path . $import_filename unless -e $import_filename; + push(@result,$import_filename); + } + } + return @result; +} + +sub check_dry { + + if ($dry) { + scriptinfo('running in dry mode - NGCP databases will not be modified',getlogger(__PACKAGE__)); + return 1; + } else { + scriptinfo('NO DRY MODE - NGCP DATABASES WILL BE MODIFIED!',getlogger(__PACKAGE__)); + if (!$force) { + if ('yes' eq lc(prompt("Type 'yes' to proceed: "))) { + return 1; + } else { + return 0; + } + } else { + scriptinfo('force option applied',getlogger(__PACKAGE__)); + return 1; + } + } + +} + +1; diff --git a/lib/NGCP/BulkProcessor/Projects/Migration/UPCAT/barring_profiles.yml b/lib/NGCP/BulkProcessor/Projects/Migration/UPCAT/barring_profiles.yml new file mode 100755 index 0000000..31631dd --- /dev/null +++ b/lib/NGCP/BulkProcessor/Projects/Migration/UPCAT/barring_profiles.yml @@ -0,0 +1,2 @@ +default: + default: 'NCOS 0' diff --git a/lib/NGCP/BulkProcessor/Projects/Migration/UPCAT/cc_ac.yml b/lib/NGCP/BulkProcessor/Projects/Migration/UPCAT/cc_ac.yml new file mode 100644 index 0000000..a3f271b --- /dev/null +++ b/lib/NGCP/BulkProcessor/Projects/Migration/UPCAT/cc_ac.yml @@ -0,0 +1,1070 @@ +43: + 1: 1 + 2142: 1 + 2143: 1 + 2144: 1 + 2145: 1 + 2146: 1 + 2147: 1 + 2160: 1 + 2162: 1 + 2163: 1 + 2164: 1 + 2165: 1 + 2166: 1 + 2167: 1 + 2168: 1 + 2169: 1 + 2172: 1 + 2173: 1 + 2174: 1 + 2175: 1 + 2176: 1 + 2177: 1 + 2212: 1 + 2213: 1 + 2214: 1 + 2215: 1 + 2216: 1 + 2230: 1 + 2231: 1 + 2232: 1 + 2233: 1 + 2234: 1 + 2235: 1 + 2236: 1 + 2237: 1 + 2238: 1 + 2239: 1 + 2242: 1 + 2243: 1 + 2244: 1 + 2245: 1 + 2246: 1 + 2247: 1 + 2248: 1 + 2249: 1 + 2252: 1 + 2253: 1 + 2254: 1 + 2255: 1 + 2256: 1 + 2257: 1 + 2258: 1 + 2259: 1 + 2262: 1 + 2263: 1 + 2264: 1 + 2265: 1 + 2266: 1 + 2267: 1 + 2268: 1 + 2269: 1 + 2271: 1 + 2272: 1 + 2273: 1 + 2274: 1 + 2275: 1 + 2276: 1 + 2277: 1 + 2278: 1 + 2279: 1 + 2282: 1 + 2283: 1 + 2284: 1 + 2285: 1 + 2286: 1 + 2287: 1 + 2288: 1 + 2289: 1 + 2522: 1 + 2523: 1 + 2524: 1 + 2525: 1 + 2526: 1 + 2527: 1 + 2532: 1 + 2533: 1 + 2534: 1 + 2535: 1 + 2536: 1 + 2538: 1 + 2552: 1 + 2554: 1 + 2555: 1 + 2556: 1 + 2557: 1 + 2572: 1 + 2573: 1 + 2574: 1 + 2575: 1 + 2576: 1 + 2577: 1 + 2610: 1 + 2611: 1 + 2612: 1 + 2613: 1 + 2614: 1 + 2615: 1 + 2616: 1 + 2617: 1 + 2618: 1 + 2619: 1 + 2620: 1 + 2621: 1 + 2622: 1 + 2623: 1 + 2624: 1 + 2625: 1 + 2626: 1 + 2627: 1 + 2628: 1 + 2629: 1 + 2630: 1 + 2631: 1 + 2632: 1 + 2633: 1 + 2634: 1 + 2635: 1 + 2636: 1 + 2637: 1 + 2638: 1 + 2639: 1 + 2641: 1 + 2642: 1 + 2643: 1 + 2644: 1 + 2645: 1 + 2646: 1 + 2647: 1 + 2648: 1 + 2649: 1 + 2662: 1 + 2663: 1 + 2664: 1 + 2665: 1 + 2666: 1 + 2667: 1 + 2672: 1 + 2673: 1 + 2674: 1 + 2680: 1 + 2682: 1 + 2683: 1 + 2684: 1 + 2685: 1 + 2686: 1 + 2687: 1 + 2688: 1 + 2689: 1 + 2711: 1 + 2712: 1 + 2713: 1 + 2714: 1 + 2715: 1 + 2716: 1 + 2717: 1 + 2718: 1 + 2719: 1 + 2722: 1 + 2723: 1 + 2724: 1 + 2725: 1 + 2726: 1 + 2728: 1 + 2731: 1 + 2732: 1 + 2733: 1 + 2734: 1 + 2735: 1 + 2736: 1 + 2738: 1 + 2739: 1 + 2741: 1 + 2742: 1 + 2743: 1 + 2744: 1 + 2745: 1 + 2746: 1 + 2747: 1 + 2748: 1 + 2749: 1 + 2752: 1 + 2753: 1 + 2754: 1 + 2755: 1 + 2756: 1 + 2757: 1 + 2758: 1 + 2762: 1 + 2763: 1 + 2764: 1 + 2765: 1 + 2766: 1 + 2767: 1 + 2768: 1 + 2769: 1 + 2772: 1 + 2773: 1 + 2774: 1 + 2782: 1 + 2783: 1 + 2784: 1 + 2786: 1 + 2812: 1 + 2813: 1 + 2814: 1 + 2815: 1 + 2816: 1 + 2822: 1 + 2823: 1 + 2824: 1 + 2825: 1 + 2826: 1 + 2827: 1 + 2828: 1 + 2829: 1 + 2841: 1 + 2842: 1 + 2843: 1 + 2844: 1 + 2845: 1 + 2846: 1 + 2847: 1 + 2848: 1 + 2849: 1 + 2852: 1 + 2853: 1 + 2854: 1 + 2855: 1 + 2856: 1 + 2857: 1 + 2858: 1 + 2859: 1 + 2862: 1 + 2863: 1 + 2864: 1 + 2865: 1 + 2872: 1 + 2873: 1 + 2874: 1 + 2875: 1 + 2876: 1 + 2877: 1 + 2878: 1 + 2912: 1 + 2913: 1 + 2914: 1 + 2915: 1 + 2916: 1 + 2942: 1 + 2943: 1 + 2944: 1 + 2945: 1 + 2946: 1 + 2947: 1 + 2948: 1 + 2949: 1 + 2951: 1 + 2952: 1 + 2953: 1 + 2954: 1 + 2955: 1 + 2956: 1 + 2957: 1 + 2958: 1 + 2959: 1 + 2982: 1 + 2983: 1 + 2984: 1 + 2985: 1 + 2986: 1 + 2987: 1 + 2988: 1 + 2989: 1 + 3112: 1 + 3113: 1 + 3114: 1 + 3115: 1 + 3116: 1 + 3117: 1 + 3118: 1 + 3119: 1 + 3123: 1 + 3124: 1 + 3125: 1 + 3126: 1 + 3127: 1 + 3132: 1 + 3133: 1 + 3134: 1 + 3135: 1 + 3136: 1 + 3137: 1 + 3140: 1 + 3141: 1 + 3142: 1 + 3143: 1 + 3144: 1 + 3145: 1 + 3146: 1 + 3147: 1 + 3148: 1 + 3149: 1 + 3150: 1 + 3151: 1 + 3152: 1 + 3153: 1 + 3155: 1 + 3157: 1 + 3158: 1 + 3159: 1 + 316: 1 + 3170: 1 + 3171: 1 + 3172: 1 + 3173: 1 + 3174: 1 + 3175: 1 + 3176: 1 + 3177: 1 + 3178: 1 + 3179: 1 + 3182: 1 + 3183: 1 + 3184: 1 + 3185: 1 + 3322: 1 + 3323: 1 + 3324: 1 + 3325: 1 + 3326: 1 + 3327: 1 + 3328: 1 + 3329: 1 + 3331: 1 + 3332: 1 + 3333: 1 + 3334: 1 + 3335: 1 + 3336: 1 + 3337: 1 + 3338: 1 + 3339: 1 + 3352: 1 + 3353: 1 + 3354: 1 + 3355: 1 + 3356: 1 + 3357: 1 + 3358: 1 + 3359: 1 + 3362: 1 + 3363: 1 + 3364: 1 + 3365: 1 + 3366: 1 + 3382: 1 + 3383: 1 + 3385: 1 + 3386: 1 + 3387: 1 + 3452: 1 + 3453: 1 + 3454: 1 + 3455: 1 + 3456: 1 + 3457: 1 + 3460: 1 + 3461: 1 + 3462: 1 + 3463: 1 + 3464: 1 + 3465: 1 + 3466: 1 + 3467: 1 + 3468: 1 + 3469: 1 + 3472: 1 + 3473: 1 + 3474: 1 + 3475: 1 + 3476: 1 + 3477: 1 + 3512: 1 + 3513: 1 + 3514: 1 + 3515: 1 + 3516: 1 + 3532: 1 + 3533: 1 + 3534: 1 + 3535: 1 + 3536: 1 + 3537: 1 + 3571: 1 + 3572: 1 + 3573: 1 + 3574: 1 + 3575: 1 + 3576: 1 + 3577: 1 + 3578: 1 + 3579: 1 + 3581: 1 + 3582: 1 + 3583: 1 + 3584: 1 + 3585: 1 + 3586: 1 + 3587: 1 + 3588: 1 + 3611: 1 + 3612: 1 + 3613: 1 + 3614: 1 + 3615: 1 + 3616: 1 + 3617: 1 + 3618: 1 + 3619: 1 + 3622: 1 + 3623: 1 + 3624: 1 + 3631: 1 + 3632: 1 + 3633: 1 + 3634: 1 + 3635: 1 + 3636: 1 + 3637: 1 + 3638: 1 + 3680: 1 + 3682: 1 + 3683: 1 + 3684: 1 + 3685: 1 + 3686: 1 + 3687: 1 + 3688: 1 + 3689: 1 + 3832: 1 + 3833: 1 + 3834: 1 + 3842: 1 + 3843: 1 + 3844: 1 + 3845: 1 + 3846: 1 + 3847: 1 + 3848: 1 + 3849: 1 + 3852: 1 + 3853: 1 + 3854: 1 + 3855: 1 + 3856: 1 + 3857: 1 + 3858: 1 + 3859: 1 + 3861: 1 + 3862: 1 + 3863: 1 + 3864: 1 + 3865: 1 + 3866: 1 + 3867: 1 + 3868: 1 + 3869: 1 + 3882: 1 + 3883: 1 + 3884: 1 + 3885: 1 + 3886: 1 + 4212: 1 + 4213: 1 + 4214: 1 + 4215: 1 + 4220: 1 + 4221: 1 + 4223: 1 + 4224: 1 + 4225: 1 + 4226: 1 + 4227: 1 + 4228: 1 + 4229: 1 + 4230: 1 + 4231: 1 + 4232: 1 + 4233: 1 + 4234: 1 + 4235: 1 + 4236: 1 + 4237: 1 + 4238: 1 + 4239: 1 + 4240: 1 + 4242: 1 + 4243: 1 + 4244: 1 + 4245: 1 + 4246: 1 + 4247: 1 + 4248: 1 + 4252: 1 + 4253: 1 + 4254: 1 + 4255: 1 + 4256: 1 + 4257: 1 + 4258: 1 + 4262: 1 + 4263: 1 + 4264: 1 + 4265: 1 + 4266: 1 + 4267: 1 + 4268: 1 + 4269: 1 + 4271: 1 + 4272: 1 + 4273: 1 + 4274: 1 + 4275: 1 + 4276: 1 + 4277: 1 + 4278: 1 + 4279: 1 + 4282: 1 + 4283: 1 + 4284: 1 + 4285: 1 + 4286: 1 + 4350: 1 + 4352: 1 + 4353: 1 + 4354: 1 + 4355: 1 + 4356: 1 + 4357: 1 + 4358: 1 + 4359: 1 + 463: 1 + 4710: 1 + 4712: 1 + 4713: 1 + 4714: 1 + 4715: 1 + 4716: 1 + 4717: 1 + 4718: 1 + 4732: 1 + 4733: 1 + 4734: 1 + 4735: 1 + 4736: 1 + 4761: 1 + 4762: 1 + 4766: 1 + 4767: 1 + 4768: 1 + 4769: 1 + 4782: 1 + 4783: 1 + 4784: 1 + 4785: 1 + 4822: 1 + 4823: 1 + 4824: 1 + 4825: 1 + 4826: 1 + 4842: 1 + 4843: 1 + 4846: 1 + 4847: 1 + 4848: 1 + 4852: 1 + 4853: 1 + 4855: 1 + 4858: 1 + 4872: 1 + 4873: 1 + 4874: 1 + 4875: 1 + 4876: 1 + 4877: 1 + 4879: 1 + 501: 1 + 502: 1 + 503: 1 + 504: 1 + 505: 1 + 506: 1 + 507: 1 + 508: 1 + 509: 1 + 512: 1 + 517: 1 + 5212: 1 + 5213: 1 + 5214: 1 + 5223: 1 + 5224: 1 + 5225: 1 + 5226: 1 + 5230: 1 + 5232: 1 + 5234: 1 + 5236: 1 + 5238: 1 + 5239: 1 + 5242: 1 + 5243: 1 + 5244: 1 + 5245: 1 + 5246: 1 + 5248: 1 + 5252: 1 + 5253: 1 + 5254: 1 + 5255: 1 + 5256: 1 + 5262: 1 + 5263: 1 + 5264: 1 + 5265: 1 + 5266: 1 + 5272: 1 + 5273: 1 + 5274: 1 + 5275: 1 + 5276: 1 + 5278: 1 + 5279: 1 + 5280: 1 + 5282: 1 + 5283: 1 + 5284: 1 + 5285: 1 + 5286: 1 + 5287: 1 + 5288: 1 + 5289: 1 + 5331: 1 + 5332: 1 + 5333: 1 + 5334: 1 + 5335: 1 + 5336: 1 + 5337: 1 + 5338: 1 + 5339: 1 + 5352: 1 + 5353: 1 + 5354: 1 + 5355: 1 + 5356: 1 + 5357: 1 + 5358: 1 + 5359: 1 + 5372: 1 + 5373: 1 + 5374: 1 + 5375: 1 + 5376: 1 + 5412: 1 + 5413: 1 + 5414: 1 + 5417: 1 + 5418: 1 + 5441: 1 + 5442: 1 + 5443: 1 + 5444: 1 + 5445: 1 + 5446: 1 + 5447: 1 + 5448: 1 + 5449: 1 + 5472: 1 + 5473: 1 + 5474: 1 + 5475: 1 + 5476: 1 + 5477: 1 + 5510: 1 + 5512: 1 + 5513: 1 + 5514: 1 + 5515: 1 + 5516: 1 + 5517: 1 + 5518: 1 + 5519: 1 + 5522: 1 + 5523: 1 + 5524: 1 + 5525: 1 + 5526: 1 + 5550: 1 + 5552: 1 + 5553: 1 + 5554: 1 + 5556: 1 + 5557: 1 + 5558: 1 + 5559: 1 + 5572: 1 + 5573: 1 + 5574: 1 + 5575: 1 + 5576: 1 + 5577: 1 + 5578: 1 + 5579: 1 + 5582: 1 + 5583: 1 + 5585: 1 + 5632: 1 + 5633: 1 + 5634: 1 + 5635: 1 + 5672: 1 + 5673: 1 + 5674: 1 + 5675: 1 + 5676: 1 + 5677: 1 + 5678: 1 + 57: 1 + 59: 1 + 6131: 1 + 6132: 1 + 6133: 1 + 6134: 1 + 6135: 1 + 6136: 1 + 6137: 1 + 6138: 1 + 6212: 1 + 6213: 1 + 6214: 1 + 6215: 1 + 6216: 1 + 6217: 1 + 6219: 1 + 6221: 1 + 6223: 1 + 6224: 1 + 6225: 1 + 6226: 1 + 6227: 1 + 6228: 1 + 6229: 1 + 6232: 1 + 6233: 1 + 6234: 1 + 6235: 1 + 6240: 1 + 6241: 1 + 6242: 1 + 6243: 1 + 6244: 1 + 6245: 1 + 6246: 1 + 6247: 1 + 6272: 1 + 6274: 1 + 6276: 1 + 6277: 1 + 6278: 1 + 6412: 1 + 6413: 1 + 6414: 1 + 6415: 1 + 6416: 1 + 6417: 1 + 6418: 1 + 6432: 1 + 6433: 1 + 6434: 1 + 6452: 1 + 6453: 1 + 6454: 1 + 6455: 1 + 6456: 1 + 6457: 1 + 6458: 1 + 6461: 1 + 6462: 1 + 6463: 1 + 6466: 1 + 6467: 1 + 6468: 1 + 6470: 1 + 6471: 1 + 6472: 1 + 6473: 1 + 6474: 1 + 6475: 1 + 6476: 1 + 6477: 1 + 6478: 1 + 6479: 1 + 6483: 1 + 6484: 1 + 650: 1 + 651: 1 + 652: 1 + 653: 1 + 6541: 1 + 6542: 1 + 6543: 1 + 6544: 1 + 6545: 1 + 6546: 1 + 6547: 1 + 6548: 1 + 6549: 1 + 655: 1 + 6562: 1 + 6563: 1 + 6564: 1 + 6565: 1 + 6566: 1 + 657: 1 + 6582: 1 + 6583: 1 + 6584: 1 + 6588: 1 + 6589: 1 + 659: 1 + 660: 1 + 661: 1 + 662: 1 + 663: 1 + 664: 1 + 665: 1 + 666: 1 + 667: 1 + 668: 1 + 669: 1 + 67: 1 + 68: 1 + 69: 1 + 711: 1 + 718: 1 + 720: 1 + 7211: 1 + 7212: 1 + 7213: 1 + 7214: 1 + 7215: 1 + 7216: 1 + 7217: 1 + 7218: 1 + 7219: 1 + 7221: 1 + 7223: 1 + 7224: 1 + 7225: 1 + 7226: 1 + 7227: 1 + 7228: 1 + 7229: 1 + 7230: 1 + 7231: 1 + 7232: 1 + 7233: 1 + 7234: 1 + 7235: 1 + 7236: 1 + 7237: 1 + 7238: 1 + 7239: 1 + 7240: 1 + 7241: 1 + 7242: 1 + 7243: 1 + 7244: 1 + 7245: 1 + 7246: 1 + 7247: 1 + 7248: 1 + 7249: 1 + 7250: 1 + 7251: 1 + 7252: 1 + 7253: 1 + 7254: 1 + 7255: 1 + 7256: 1 + 7257: 1 + 7258: 1 + 7259: 1 + 7260: 1 + 7261: 1 + 7262: 1 + 7263: 1 + 7264: 1 + 7265: 1 + 7266: 1 + 7267: 1 + 7268: 1 + 7269: 1 + 7272: 1 + 7273: 1 + 7274: 1 + 7276: 1 + 7277: 1 + 7278: 1 + 7279: 1 + 7280: 1 + 7281: 1 + 7282: 1 + 7283: 1 + 7284: 1 + 7285: 1 + 7286: 1 + 7287: 1 + 7288: 1 + 7289: 1 + 732: 1 + 7353: 1 + 7355: 1 + 7357: 1 + 7412: 1 + 7413: 1 + 7414: 1 + 7415: 1 + 7416: 1 + 7432: 1 + 7433: 1 + 7434: 1 + 7435: 1 + 7442: 1 + 7443: 1 + 7444: 1 + 7445: 1 + 7448: 1 + 7471: 1 + 7472: 1 + 7473: 1 + 7474: 1 + 7475: 1 + 7476: 1 + 7477: 1 + 7478: 1 + 7479: 1 + 7480: 1 + 7482: 1 + 7483: 1 + 7484: 1 + 7485: 1 + 7486: 1 + 7487: 1 + 7488: 1 + 7489: 1 + 7562: 1 + 7563: 1 + 7564: 1 + 7565: 1 + 7566: 1 + 7582: 1 + 7583: 1 + 7584: 1 + 7585: 1 + 7586: 1 + 7587: 1 + 7588: 1 + 7612: 1 + 7613: 1 + 7614: 1 + 7615: 1 + 7616: 1 + 7617: 1 + 7618: 1 + 7619: 1 + 7662: 1 + 7663: 1 + 7664: 1 + 7665: 1 + 7666: 1 + 7667: 1 + 7672: 1 + 7673: 1 + 7674: 1 + 7675: 1 + 7676: 1 + 7682: 1 + 7683: 1 + 7684: 1 + 7711: 1 + 7712: 1 + 7713: 1 + 7714: 1 + 7716: 1 + 7717: 1 + 7718: 1 + 7719: 1 + 7722: 1 + 7723: 1 + 7724: 1 + 7727: 1 + 7728: 1 + 7729: 1 + 7732: 1 + 7733: 1 + 7734: 1 + 7735: 1 + 7736: 1 + 7742: 1 + 7743: 1 + 7744: 1 + 7745: 1 + 7746: 1 + 7747: 1 + 7748: 1 + 7750: 1 + 7751: 1 + 7752: 1 + 7753: 1 + 7754: 1 + 7755: 1 + 7757: 1 + 7758: 1 + 7759: 1 + 7762: 1 + 7763: 1 + 7764: 1 + 7765: 1 + 7766: 1 + 7767: 1 + 780: 1 + 7941: 1 + 7942: 1 + 7943: 1 + 7944: 1 + 7945: 1 + 7946: 1 + 7947: 1 + 7948: 1 + 7949: 1 + 7952: 1 + 7953: 1 + 7954: 1 + 7955: 1 + 7956: 1 + 800: 1 + 804: 1 + 810: 1 + 820: 1 + 821: 1 + 828: 1 + 89: 1 + 900: 1 + 901: 1 + 930: 1 + 931: 1 + 939: 1 \ No newline at end of file diff --git a/lib/NGCP/BulkProcessor/Projects/Migration/UPCAT/config.cfg b/lib/NGCP/BulkProcessor/Projects/Migration/UPCAT/config.cfg new file mode 100755 index 0000000..bd050a0 --- /dev/null +++ b/lib/NGCP/BulkProcessor/Projects/Migration/UPCAT/config.cfg @@ -0,0 +1,61 @@ +##general settings: +working_path = /home/rkrenn/temp/upcat +cpucount = 4 +enablemultithreading = 0 + +##gearman/service listener config: +jobservers = 127.0.0.1:4730 + +##NGCP MySQL connectivity - "accounting" db: +accounting_host = 192.168.0.29 +accounting_port = 3306 +accounting_databasename = accounting +accounting_username = root +accounting_password = + +##NGCP MySQL connectivity - "billing" db: +billing_host = 192.168.0.29 +billing_port = 3306 +billing_databasename = billing +billing_username = root +billing_password = + +##NGCP MySQL connectivity - "provisioning" db: +provisioning_host = 192.168.0.29 +provisioning_port = 3306 +provisioning_databasename = provisioning +provisioning_username = root +provisioning_password = + +##NGCP MySQL connectivity - "kamailio" db: +kamailio_host = 192.168.0.29 +kamailio_port = 3306 +kamailio_databasename = kamailio +kamailio_username = root +kamailio_password = + +##NGCP MySQL connectivity - default db for distributed transactions (XA) to connect to: +xa_host = 192.168.0.29 +xa_port = 3306 +xa_databasename = ngcp +xa_username = root +xa_password = + +##NGCP REST-API connectivity: +ngcprestapi_uri = https://127.0.0.1:1443 +ngcprestapi_username = administrator +ngcprestapi_password = administrator +ngcprestapi_realm = api_admin_http + +##sending email: +emailenable = 0 +erroremailrecipient = +warnemailrecipient = +completionemailrecipient = rkrenn@sipwise.com +doneemailrecipient = + +##logging: +fileloglevel = DEBUG +screenloglevel = INFO +#INFO +emailloglevel = OFF diff --git a/lib/NGCP/BulkProcessor/Projects/Migration/UPCAT/config.debug.cfg b/lib/NGCP/BulkProcessor/Projects/Migration/UPCAT/config.debug.cfg new file mode 100644 index 0000000..bd050a0 --- /dev/null +++ b/lib/NGCP/BulkProcessor/Projects/Migration/UPCAT/config.debug.cfg @@ -0,0 +1,61 @@ +##general settings: +working_path = /home/rkrenn/temp/upcat +cpucount = 4 +enablemultithreading = 0 + +##gearman/service listener config: +jobservers = 127.0.0.1:4730 + +##NGCP MySQL connectivity - "accounting" db: +accounting_host = 192.168.0.29 +accounting_port = 3306 +accounting_databasename = accounting +accounting_username = root +accounting_password = + +##NGCP MySQL connectivity - "billing" db: +billing_host = 192.168.0.29 +billing_port = 3306 +billing_databasename = billing +billing_username = root +billing_password = + +##NGCP MySQL connectivity - "provisioning" db: +provisioning_host = 192.168.0.29 +provisioning_port = 3306 +provisioning_databasename = provisioning +provisioning_username = root +provisioning_password = + +##NGCP MySQL connectivity - "kamailio" db: +kamailio_host = 192.168.0.29 +kamailio_port = 3306 +kamailio_databasename = kamailio +kamailio_username = root +kamailio_password = + +##NGCP MySQL connectivity - default db for distributed transactions (XA) to connect to: +xa_host = 192.168.0.29 +xa_port = 3306 +xa_databasename = ngcp +xa_username = root +xa_password = + +##NGCP REST-API connectivity: +ngcprestapi_uri = https://127.0.0.1:1443 +ngcprestapi_username = administrator +ngcprestapi_password = administrator +ngcprestapi_realm = api_admin_http + +##sending email: +emailenable = 0 +erroremailrecipient = +warnemailrecipient = +completionemailrecipient = rkrenn@sipwise.com +doneemailrecipient = + +##logging: +fileloglevel = DEBUG +screenloglevel = INFO +#INFO +emailloglevel = OFF diff --git a/lib/NGCP/BulkProcessor/Projects/Migration/UPCAT/process.pl b/lib/NGCP/BulkProcessor/Projects/Migration/UPCAT/process.pl new file mode 100755 index 0000000..132d18f --- /dev/null +++ b/lib/NGCP/BulkProcessor/Projects/Migration/UPCAT/process.pl @@ -0,0 +1,377 @@ +use strict; + +## no critic + +use File::Basename; +use Cwd; +use lib Cwd::abs_path(File::Basename::dirname(__FILE__) . '/../../../../../'); + +use Getopt::Long qw(GetOptions); +use Fcntl qw(LOCK_EX LOCK_NB); + +use NGCP::BulkProcessor::Globals qw(); +use NGCP::BulkProcessor::Projects::Migration::UPCAT::Settings qw( + update_settings + update_cc_ac_map + update_barring_profiles + + check_dry + $output_path + $defaultsettings + $defaultconfig + $dry + $skip_errors + $force + $run_id + + @subscriber_filenames + $cc_ac_map_yml + + $barring_profiles_yml + +); +#$allowed_ips + +use NGCP::BulkProcessor::Logging qw( + init_log + getlogger + $attachmentlogfile + scriptinfo + cleanuplogfiles + $currentlogfile +); +use NGCP::BulkProcessor::LogError qw ( + completion + done + scriptwarn + scripterror + filewarn + fileerror +); +use NGCP::BulkProcessor::LoadConfig qw( + load_config + $SIMPLE_CONFIG_TYPE + $YAML_CONFIG_TYPE + $ANY_CONFIG_TYPE +); +use NGCP::BulkProcessor::Array qw(removeduplicates); +use NGCP::BulkProcessor::Utils qw(getscriptpath prompt cleanupdir); +use NGCP::BulkProcessor::Mail qw( + cleanupmsgfiles +); +use NGCP::BulkProcessor::SqlConnectors::CSVDB qw(cleanupcvsdirs); +use NGCP::BulkProcessor::SqlConnectors::SQLiteDB qw(cleanupdbfiles); +use NGCP::BulkProcessor::RestConnectors::NGCPRestApi qw(cleanupcertfiles); + +use NGCP::BulkProcessor::Projects::Migration::UPCAT::ProjectConnectorPool qw(destroy_all_dbs); + +use NGCP::BulkProcessor::Projects::Migration::UPCAT::Dao::import::Subscriber qw(); + +use NGCP::BulkProcessor::Dao::Trunk::billing::contracts qw(); +use NGCP::BulkProcessor::Dao::Trunk::billing::voip_subscribers qw(); +#use NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_dbaliases qw(); +#use NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_cf_mappings qw(); +#use NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_trusted_sources qw(); + +#use NGCP::BulkProcessor::Dao::Trunk::kamailio::location qw(); + +use NGCP::BulkProcessor::Projects::Migration::UPCAT::Check qw( + check_billing_db_tables + check_provisioning_db_tables + check_kamailio_db_tables + check_import_db_tables + check_rest_get_items +); + +use NGCP::BulkProcessor::Projects::Migration::UPCAT::Import qw( + import_subscriber +); + +use NGCP::BulkProcessor::Projects::Migration::UPCAT::Provisioning qw( + provision_subscribers +); + +scripterror(getscriptpath() . ' already running',getlogger(getscriptpath())) unless flock DATA, LOCK_EX | LOCK_NB; # not tested on windows yet + +my @TASK_OPTS = (); + +my $tasks = []; + +my $check_task_opt = 'check'; +push(@TASK_OPTS,$check_task_opt); + +my $cleanup_task_opt = 'cleanup'; +push(@TASK_OPTS,$cleanup_task_opt); +my $cleanup_all_task_opt = 'cleanup_all'; +push(@TASK_OPTS,$cleanup_all_task_opt); + +my $import_subscriber_task_opt = 'import_subscriber'; +push(@TASK_OPTS,$import_subscriber_task_opt); +my $import_truncate_subscriber_task_opt = 'truncate_subscriber'; +push(@TASK_OPTS,$import_truncate_subscriber_task_opt); + +my $create_subscriber_task_opt = 'create_subscriber'; +push(@TASK_OPTS,$create_subscriber_task_opt); + +if (init()) { + main(); + exit(0); +} else { + exit(1); +} + +sub init { + + my $configfile = $defaultconfig; + my $settingsfile = $defaultsettings; + + return 0 unless GetOptions( + "config=s" => \$configfile, + "settings=s" => \$settingsfile, + "task=s" => $tasks, + "run=s" => \$run_id, + "dry" => \$dry, + "skip-errors" => \$skip_errors, + "force" => \$force, + ); # or scripterror('error in command line arguments',getlogger(getscriptpath())); + + $tasks = removeduplicates($tasks,1); + + my $result = load_config($configfile); + init_log(); + $result &= load_config($settingsfile,\&update_settings,$SIMPLE_CONFIG_TYPE); + $result &= load_config($cc_ac_map_yml,\&update_cc_ac_map,$YAML_CONFIG_TYPE); + $result &= load_config($barring_profiles_yml,\&update_barring_profiles,$YAML_CONFIG_TYPE); + return $result; + +} + +sub main() { + + my @messages = (); + my @attachmentfiles = (); + my $result = 1; + my $completion = 0; + + if (defined $tasks and 'ARRAY' eq ref $tasks and (scalar @$tasks) > 0) { + scriptinfo('skip-errors: processing won\'t stop upon errors',getlogger(__PACKAGE__)) if $skip_errors; + foreach my $task (@$tasks) { + + if (lc($check_task_opt) eq lc($task)) { + $result &= check_task(\@messages) if taskinfo($check_task_opt,$result); + + } elsif (lc($cleanup_task_opt) eq lc($task)) { + $result &= cleanup_task(\@messages,0) if taskinfo($cleanup_task_opt,$result); + } elsif (lc($cleanup_all_task_opt) eq lc($task)) { + $result &= cleanup_task(\@messages,1) if taskinfo($cleanup_all_task_opt,$result); + + } elsif (lc($import_subscriber_task_opt) eq lc($task)) { + $result &= import_subscriber_task(\@messages) if taskinfo($import_subscriber_task_opt,$result); + } elsif (lc($import_truncate_subscriber_task_opt) eq lc($task)) { + $result &= import_truncate_subscriber_task(\@messages) if taskinfo($import_truncate_subscriber_task_opt,$result); + + } elsif (lc($create_subscriber_task_opt) eq lc($task)) { + if (taskinfo($create_subscriber_task_opt,$result,1)) { + next unless check_dry(); + $result &= create_subscriber_task(\@messages); + $completion |= 1; + } + + + } else { + $result = 0; + scripterror("unknow task option '" . $task . "', must be one of " . join(', ',@TASK_OPTS),getlogger(getscriptpath())); + last; + } + } + } else { + $result = 0; + scripterror('at least one task option is required. supported tasks: ' . join(', ',@TASK_OPTS),getlogger(getscriptpath())); + } + + push(@attachmentfiles,$attachmentlogfile); + if ($completion) { + completion(join("\n\n",@messages),\@attachmentfiles,getlogger(getscriptpath())); + } else { + done(join("\n\n",@messages),\@attachmentfiles,getlogger(getscriptpath())); + } + + return $result; +} + +sub taskinfo { + my ($task,$result) = @_; + scriptinfo($result ? "starting task: '$task'" : "skipping task '$task' due to previous problems",getlogger(getscriptpath())); + return $result; +} + +sub check_task { + my ($messages) = @_; + my @check_messages = (); + my $result = check_billing_db_tables(\@check_messages); + #$result &= .. + push(@$messages,join("\n",@check_messages)); + + @check_messages = (); + $result = check_provisioning_db_tables(\@check_messages); + #$result &= .. + push(@$messages,join("\n",@check_messages)); + + @check_messages = (); + $result = check_kamailio_db_tables(\@check_messages); + #$result &= .. + push(@$messages,join("\n",@check_messages)); + + @check_messages = (); + $result = check_rest_get_items(\@check_messages); + #$result &= .. + push(@$messages,join("\n",@check_messages)); + + @check_messages = (); + $result = check_import_db_tables(\@check_messages); + #$result &= .. + push(@$messages,join("\n",@check_messages)); + + + destroy_all_dbs(); + return $result; +} + +sub cleanup_task { + my ($messages,$clean_generated) = @_; + my $result = 0; + if (!$clean_generated or $force or 'yes' eq lc(prompt("Type 'yes' to proceed: "))) { + eval { + cleanupcvsdirs() if $clean_generated; + cleanupdbfiles() if $clean_generated; + cleanuplogfiles(\&fileerror,\&filewarn,($currentlogfile,$attachmentlogfile)); + cleanupmsgfiles(\&fileerror,\&filewarn); + cleanupcertfiles(); + cleanupdir($output_path,1,\&filewarn,getlogger(getscriptpath())) if $clean_generated; + $result = 1; + }; + } + if ($@ or !$result) { + push(@$messages,'working directory cleanup INCOMPLETE'); + return 0; + } else { + push(@$messages,'working directory folders cleaned up'); + return 1; + } +} + +sub import_subscriber_task { + + my ($messages) = @_; + my ($result,$warning_count) = (0,0); + eval { + ($result,$warning_count) = import_subscriber(@subscriber_filenames); + }; + my $err = $@; + my $stats = ": $warning_count warnings"; + eval { + $stats .= "\n total subscriber records: " . + NGCP::BulkProcessor::Projects::Migration::UPCAT::Dao::import::Subscriber::countby_ccacsn() . ' rows'; + my $added_count = NGCP::BulkProcessor::Projects::Migration::UPCAT::Dao::import::Subscriber::countby_delta( + $NGCP::BulkProcessor::Projects::Migration::UPCAT::Dao::import::Subscriber::added_delta + ); + $stats .= "\n new: $added_count rows"; + my $existing_count = NGCP::BulkProcessor::Projects::Migration::UPCAT::Dao::import::Subscriber::countby_delta( + $NGCP::BulkProcessor::Projects::Migration::UPCAT::Dao::import::Subscriber::updated_delta + ); + $stats .= "\n existing: $existing_count rows"; + my $deleted_count = NGCP::BulkProcessor::Projects::Migration::UPCAT::Dao::import::Subscriber::countby_delta( + $NGCP::BulkProcessor::Projects::Migration::UPCAT::Dao::import::Subscriber::deleted_delta + ); + $stats .= "\n removed: $deleted_count rows"; + }; + if ($err or !$result) { + push(@$messages,"importing subscribers INCOMPLETE$stats"); + } else { + push(@$messages,"importing subscribers completed$stats"); + } + destroy_all_dbs(); #every task should leave with closed connections. + return $result; + +} + +sub import_truncate_subscriber_task { + + my ($messages) = @_; + my $result = 0; + eval { + $result = NGCP::BulkProcessor::Projects::Migration::UPCAT::Dao::import::Subscriber::create_table(1); + }; + my $err = $@; + my $stats = ''; + eval { + $stats .= "\n total subscriber records: " . + NGCP::BulkProcessor::Projects::Migration::UPCAT::Dao::import::Subscriber::countby_dn() . ' rows'; + }; + if ($err or !$result) { + push(@$messages,"truncating imported subscribers INCOMPLETE$stats"); + } else { + push(@$messages,"truncating imported subscribers completed$stats"); + } + destroy_all_dbs(); #every task should leave with closed connections. + return $result; + +} + + + +sub create_subscriber_task { + + my ($messages) = @_; + my ($result,$warning_count,$nonunique_contacts) = (0,0,{}); + eval { + ($result,$warning_count,$nonunique_contacts) = provision_subscribers(); + }; + my $err = $@; + my $stats = ": $warning_count warnings"; + eval { + $stats .= "\n total contracts: " . + NGCP::BulkProcessor::Dao::Trunk::billing::contracts::countby_status_resellerid(undef,undef) . ' rows'; + $stats .= "\n total subscribers: " . + NGCP::BulkProcessor::Dao::Trunk::billing::voip_subscribers::countby_status_resellerid(undef,undef) . ' rows'; + + $stats .= "\n total aliases: " . + NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_dbaliases::countby_subscriberidisprimary(undef,undef) . ' rows'; + $stats .= "\n primary aliases: " . + NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_dbaliases::countby_subscriberidisprimary(undef,1) . ' rows'; + + #$stats .= "\n call forwards: " . + # NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_cf_mappings::countby_subscriberid_type(undef,undef) . ' rows'; + + #$stats .= "\n registrations: " . + # NGCP::BulkProcessor::Dao::Trunk::kamailio::location::countby_usernamedomain(undef,undef) . ' rows'; + + #$stats .= "\n trusted sources: " . + # NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_trusted_sources::countby_subscriberid(undef) . ' rows'; + + $stats .= "\n non-unique contacts skipped:\n " . join("\n ",keys %$nonunique_contacts) + if (scalar keys %$nonunique_contacts) > 0; + }; + if ($err or !$result) { + push(@$messages,"create subscribers INCOMPLETE$stats"); + } else { + push(@$messages,"create subscribers completed$stats"); + #if (not $dry) { + # push(@$messages,"YOU MIGHT WANT TO RESTART KAMAILIO FOR PERMANENT REGISTRATIONS TO COME INTO EFFECT"); + #} + } + destroy_all_dbs(); + return $result; + +} + + +#END { +# # this should not be required explicitly, but prevents Log4Perl's +# # "rootlogger not initialized error upon exit.. +# destroy_all_dbs +#} + +__DATA__ +This exists to allow the locking code at the beginning of the file to work. +DO NOT REMOVE THESE LINES! diff --git a/lib/NGCP/BulkProcessor/Projects/Migration/UPCAT/settings.cfg b/lib/NGCP/BulkProcessor/Projects/Migration/UPCAT/settings.cfg new file mode 100755 index 0000000..d30fb01 --- /dev/null +++ b/lib/NGCP/BulkProcessor/Projects/Migration/UPCAT/settings.cfg @@ -0,0 +1,28 @@ + +#dry=0 +#skip_errors=0 +import_multithreading = 0 + +#subscriber_filenames = /home/rkrenn/temp/upcat/export1.csv,/home/rkrenn/temp/upcat/export2.csv +subscriber_filenames = /home/rkrenn/temp/upcat/ExportTestMigration.csv +subscriber_import_numofthreads = 2 +ignore_subscriber_unique = 0 +subscriber_import_single_row_txn = 1 + +provision_subscriber_rownum_start = 2 +provision_subscriber_multithreading = 1 +provision_subscriber_numofthreads = 2 +webpassword_length = 8 +webusername_length = 8 +sippassword_length = 16 +report_filename = provision.txt +#report_filename = provision_%s.json +#default_channels = 1 + +default_domain = d20.upc.at +default_reseller_name = default +default_billing_profile_name = Default Billing Profile +default_barring = +cc_ac_map_yml = cc_ac.yml +default_cc = 43 +barring_profiles_yml = barring_profiles.yml \ No newline at end of file diff --git a/lib/NGCP/BulkProcessor/Projects/Migration/UPCAT/settings.debug.cfg b/lib/NGCP/BulkProcessor/Projects/Migration/UPCAT/settings.debug.cfg new file mode 100644 index 0000000..d30fb01 --- /dev/null +++ b/lib/NGCP/BulkProcessor/Projects/Migration/UPCAT/settings.debug.cfg @@ -0,0 +1,28 @@ + +#dry=0 +#skip_errors=0 +import_multithreading = 0 + +#subscriber_filenames = /home/rkrenn/temp/upcat/export1.csv,/home/rkrenn/temp/upcat/export2.csv +subscriber_filenames = /home/rkrenn/temp/upcat/ExportTestMigration.csv +subscriber_import_numofthreads = 2 +ignore_subscriber_unique = 0 +subscriber_import_single_row_txn = 1 + +provision_subscriber_rownum_start = 2 +provision_subscriber_multithreading = 1 +provision_subscriber_numofthreads = 2 +webpassword_length = 8 +webusername_length = 8 +sippassword_length = 16 +report_filename = provision.txt +#report_filename = provision_%s.json +#default_channels = 1 + +default_domain = d20.upc.at +default_reseller_name = default +default_billing_profile_name = Default Billing Profile +default_barring = +cc_ac_map_yml = cc_ac.yml +default_cc = 43 +barring_profiles_yml = barring_profiles.yml \ No newline at end of file