TT#18881 implement teletek importer #4

+ "allowed_cli" import gets a different meaning,
  it are additional "numbers" (alias+allowed_cli) to import

+ unique cc+ac+sn in subscriber file
+ unique cc+ac+sn in allowed_cli file

+ logging of cc+ac+sn of allowed_cli records
  already found in subscriber file
+ logging of sip_usernames of allowed_cli records
  not found in subscriber file

+ reseller mapping

Change-Id: I620db415f5dab91e8d5c6336b7a45b3a0ffeacb9

lib/NGCP/BulkProcessor/Projects/Migration/Teletek/Dao/import/AllowedCli.pm

@@ -158,7 +158,7 @@ sub findby_ccacsn {
 
 }
 
-sub findby_domain_sipusername {
+sub findby_sipusername {
 
     my ($sip_username,$load_recursive) = @_;
 

lib/NGCP/BulkProcessor/Projects/Migration/Teletek/Dao/import/Subscriber.pm

@@ -38,6 +38,7 @@ our @EXPORT_OK = qw(
     findby_domain_sipusername
     findby_domain_webusername
     list_domain_billingprofilename_resellernames
+    findby_sipusername
 
     update_delta
     findby_delta
@@ -204,6 +205,26 @@ sub findby_domain_sipusername {
 
 }
 
+sub findby_sipusername {
+
+    my ($sip_username,$load_recursive) = @_;
+
+    check_table();
+    my $db = &$get_db();
+    my $table = $db->tableidentifier($tablename);
+
+    #return [] unless (defined $cc or defined $ac or defined $sn);
+
+    my $rows = $db->db_get_all_arrayref(
+        'SELECT * FROM ' .
+            $table .
+        ' WHERE ' . $db->columnidentifier('sip_username') . ' = ?'
+    ,$sip_username);
+
+    return buildrecords_fromrows($rows,$load_recursive);
+
+}
+
 sub findby_domain_webusername {
 
     my ($domain,$web_username,$load_recursive) = @_;

lib/NGCP/BulkProcessor/Projects/Migration/Teletek/Import.pm

@@ -273,6 +273,26 @@ sub import_allowedcli {
                     $rownum++;
                     next if (scalar @$row) == 0;
                     my $record = NGCP::BulkProcessor::Projects::Migration::Teletek::Dao::import::AllowedCli->new($row);
+
+                    if (NGCP::BulkProcessor::Projects::Migration::Teletek::Dao::import::Subscriber::findby_ccacsn($record->{cc},$record->{ac},$record->{sn})) {
+                        my $number = ($record->{cc} // '') . ($record->{ac} // '') . ($record->{sn} // '');
+                        if ($skip_errors) {
+                            _warn($context,"duplicate number: $number");
+                        } else {
+                            _error($context,"duplicate number: $number");
+                        }
+                        next;
+                    }
+
+                    if ((scalar @{NGCP::BulkProcessor::Projects::Migration::Teletek::Dao::import::Subscriber::findby_sipusername($record->{sip_username})}) == 0) {
+                        if ($skip_errors) {
+                            _warn($context,"sip username $record->{sip_username} not found");
+                        } else {
+                            _error($context,"sip username $record->{sip_username} not found");
+                        }
+                        next;
+                    }
+
                     $record->{rownum} = $rownum;
                     my %r = %$record; my @allowedcli_row = @r{@NGCP::BulkProcessor::Projects::Migration::Teletek::Dao::import::AllowedCli::fieldnames};
                     if ($context->{upsert}) {
@@ -333,14 +353,14 @@ sub import_allowedcli {
 sub _import_allowedcli_checks {
     my ($file) = @_;
     my $result = 1;
-    #my $optioncount = 0;
+    my $subscribercount = 0;
-    #eval {
+    eval {
-    #    $optioncount = NGCP::BulkProcessor::Projects::Migration::Teletek::Dao::import::FeatureOption::countby_subscribernumber_option();
+        $subscribercount = NGCP::BulkProcessor::Projects::Migration::Teletek::Dao::import::Subscriber::countby_ccacsn();
-    #};
+    };
-    #if ($@ or $optioncount == 0) {
+    if ($@ or $subscribercount == 0) {
-    #    fileprocessingerror($file,'please import subscriber features first',getlogger(__PACKAGE__));
+        fileprocessingerror($file,'please import subscribers first',getlogger(__PACKAGE__));
-    #    $result = 0; #even in skip-error mode..
+        $result = 0; #even in skip-error mode..
-    #}
+    }
     #my $userpasswordcount = 0;
     #eval {
     #    $userpasswordcount = NGCP::BulkProcessor::Projects::Migration::Teletek::Dao::import::UsernamePassword::countby_fqdn();

lib/NGCP/BulkProcessor/Projects/Migration/Teletek/Provisioning.pm

@@ -17,6 +17,8 @@ use NGCP::BulkProcessor::Projects::Migration::Teletek::Settings qw(
     $provision_subscriber_multithreading
     $provision_subscriber_numofthreads
 
+    $reseller_mapping
+
 );
 #$batch
 
@@ -42,7 +44,7 @@ use NGCP::BulkProcessor::LogError qw(
 );
 
 use NGCP::BulkProcessor::Projects::Migration::Teletek::Dao::import::Subscriber qw();
-#use NGCP::BulkProcessor::Projects::Migration::Teletek::Dao::import::AllowedCli qw();
+use NGCP::BulkProcessor::Projects::Migration::Teletek::Dao::import::AllowedCli qw();
 
 use NGCP::BulkProcessor::Dao::Trunk::billing::billing_profiles qw();
 use NGCP::BulkProcessor::Dao::Trunk::billing::products qw();
@@ -231,7 +233,7 @@ sub _provision_subscribers_checks {
                 rowprocessingerror(threadid(),"empty billing profile name detected",getlogger(__PACKAGE__));
                 $result = 0; #even in skip-error mode..
             }
-            my $resellername = $domain_billingprofilename_resellername->{reseller_name};
+            my $resellername = _apply_reseller_mapping($domain_billingprofilename_resellername->{reseller_name});
             unless ($resellername) {
                 rowprocessingerror(threadid(),"empty reseller name detected",getlogger(__PACKAGE__));
                 $result = 0; #even in skip-error mode..
@@ -617,7 +619,7 @@ sub _provision_susbcriber_init_context {
     }
 
     $context->{domain} = $context->{domain_map}->{$first->{domain}};
-    $context->{reseller} = $context->{reseller_map}->{$first->{reseller_name}};
+    $context->{reseller} = $context->{reseller_map}->{_apply_reseller_mapping($first->{reseller_name})};
     $context->{billing_profile} = $context->{reseller}->{billingprofile_map}->{$first->{billing_profile_name}};
 
     $context->{prov_subscriber} = {};
@@ -631,7 +633,7 @@ sub _provision_susbcriber_init_context {
             @{NGCP::BulkProcessor::Projects::Migration::Teletek::Dao::import::Subscriber::findby_domain_webusername(
             $first->{domain},$context->{prov_subscriber}->{webusername})};
         if ((scalar keys %webusername_dupes) > 1) {
-            _warn($context,"duplicate web_username $context->{prov_subscriber}->{webusername}, using sip_username");
+            #_warn($context,"duplicate web_username $context->{prov_subscriber}->{webusername}, using sip_username");
             $context->{prov_subscriber}->{webusername} = $first->{sip_username};
         }
     }
@@ -658,10 +660,11 @@ sub _provision_susbcriber_init_context {
                 sn => $subscriber->{sn} // '',
                 number => $number,
                 delta => $subscriber->{delta},
+                additional => 0,
             });
             $number_dupes{$number} = 1;
         } else {
-            _warn($context,'duplicate number $number ignored');
+            _warn($context,'duplicate number $number (subscriber table) ignored');
         }
 
         if (not exists $contact_dupes{$subscriber->{contact_hash}}) {
@@ -692,8 +695,37 @@ sub _provision_susbcriber_init_context {
         }
     }
 
+    foreach my $allowed_cli (@{NGCP::BulkProcessor::Projects::Migration::Teletek::Dao::import::AllowedCli::findby_sipusername($first->{sip_username})}) {
+        my $number = ($allowed_cli->{cc} // '') . ($allowed_cli->{ac} // '') . ($allowed_cli->{sn} // '');
+        if (not exists $number_dupes{$number}) {
+            push(@numbers,{
+                cc => $allowed_cli->{cc} // '',
+                ac => $allowed_cli->{ac} // '',
+                sn => $allowed_cli->{sn} // '',
+                number => $number,
+                delta => $allowed_cli->{delta},
+                additional => 1,
+            });
+            $number_dupes{$number} = 1;
+        } else {
+            _warn($context,'duplicate number $number (allowed_cli table) ignored');
+        }
+    }
+
     $context->{numbers} = {};
     $context->{numbers}->{other} = sort_by_configs(\@numbers,[
+        {   numeric     => 1,
+            dir         => 1, #-1,
+            memberchain => [ 'additional' ],
+        },
+        {   numeric     => 0,
+            dir         => 1, #-1,
+            memberchain => [ 'cc' ],
+        },
+        {   numeric     => 0,
+            dir         => 1, #-1,
+            memberchain => [ 'ac' ],
+        },
         {   numeric     => 0,
             dir         => 1, #-1,
             memberchain => [ 'sn' ],
@@ -727,6 +759,14 @@ sub _provision_susbcriber_init_context {
 #    );
 #}
 
+sub _apply_reseller_mapping {
+    my $reseller_name = shift;
+    if (defined $reseller_name and exists $reseller_mapping->{$reseller_name}) {
+        return $reseller_mapping->{$reseller_name};
+    }
+    return $reseller_name;
+}
+
 sub _error {
 
     my ($context,$message) = @_;

lib/NGCP/BulkProcessor/Projects/Migration/Teletek/Settings.pm

@@ -32,6 +32,7 @@ require Exporter;
 our @ISA = qw(Exporter);
 our @EXPORT_OK = qw(
     update_settings
+    update_reseller_mapping
     check_dry
 
     $input_path
@@ -52,7 +53,8 @@ our @EXPORT_OK = qw(
     $ignore_subscriber_unique
     $subscriber_import_single_row_txn
     $subscriber_import_unfold_ranges
-
+    $reseller_mapping_yml
+    $reseller_mapping
 
     @allowedcli_filenames
     $allowedcli_import_numofthreads
@@ -106,6 +108,8 @@ our $subscriber_import_numofthreads = $cpucount;
 our $ignore_subscriber_unique = 0;
 our $subscriber_import_single_row_txn = 1;
 our $subscriber_import_unfold_ranges = 1;
+our $reseller_mapping_yml = undef;
+our $reseller_mapping = {};
 
 our @allowedcli_filenames = ();
 our $allowedcli_import_numofthreads = $cpucount;
@@ -163,6 +167,7 @@ sub update_settings {
         $ignore_subscriber_unique = $data->{ignore_subscriber_unique} if exists $data->{ignore_subscriber_unique};
         $subscriber_import_single_row_txn = $data->{subscriber_import_single_row_txn} if exists $data->{subscriber_import_single_row_txn};
         $subscriber_import_unfold_ranges = $data->{subscriber_import_unfold_ranges} if exists $data->{subscriber_import_unfold_ranges};
+        $reseller_mapping_yml = $data->{reseller_mapping_yml} if exists $data->{reseller_mapping_yml};
 
         @allowedcli_filenames = _get_import_filenames(\@allowedcli_filenames,$data,'allowedcli_filenames');
         $allowedcli_import_numofthreads = _get_numofthreads($cpucount,$data,'allowedcli_import_numofthreads');
@@ -284,4 +289,27 @@ sub check_dry {
 
 }
 
+sub update_reseller_mapping {
+
+    my ($data,$configfile) = @_;
+
+    if (defined $data) {
+
+        my $result = 1;
+
+        eval {
+            $reseller_mapping = $data->{'mapping'};
+        };
+        if ($@ or 'HASH' ne ref $reseller_mapping) { # or (scalar keys %$reseller_mapping) == 0) {
+            $reseller_mapping //= {};
+            configurationerror($configfile,'invalid reseller mapping',getlogger(__PACKAGE__));
+            $result = 0;
+        }
+
+        return $result;
+    }
+    return 0;
+
+}
+
 1;

lib/NGCP/BulkProcessor/Projects/Migration/Teletek/process.pl

@@ -12,6 +12,7 @@ use Fcntl qw(LOCK_EX LOCK_NB);
 use NGCP::BulkProcessor::Globals qw();
 use NGCP::BulkProcessor::Projects::Migration::Teletek::Settings qw(
     update_settings
+    update_reseller_mapping
     check_dry
     $output_path
     $defaultsettings
@@ -21,7 +22,7 @@ use NGCP::BulkProcessor::Projects::Migration::Teletek::Settings qw(
     $force
     $run_id
     @subscriber_filenames
-
+    $reseller_mapping_yml
 
     @allowedcli_filenames
 );
@@ -161,6 +162,7 @@ sub init {
     my $result = load_config($configfile);
     init_log();
     $result &= load_config($settingsfile,\&update_settings,$SIMPLE_CONFIG_TYPE);
+    $result &= load_config($reseller_mapping_yml,\&update_reseller_mapping,$YAML_CONFIG_TYPE);
     return $result;
 
 }
@@ -316,7 +318,7 @@ sub import_subscriber_task {
         ($result,$warning_count) = import_subscriber(@subscriber_filenames);
     };
     my $err = $@;
-    my $stats = ($skip_errors ? ": $warning_count warnings" : '');
+    my $stats = ": $warning_count warnings";
     eval {
         $stats .= "\n  total subscriber records: " .
             NGCP::BulkProcessor::Projects::Migration::Teletek::Dao::import::Subscriber::countby_ccacsn() . ' rows';
@@ -378,7 +380,7 @@ sub import_allowedcli_task {
         ($result,$warning_count) = import_allowedcli(@allowedcli_filenames);
     };
     my $err = $@;
-    my $stats = ($skip_errors ? ": $warning_count warnings" : '');
+    my $stats = ": $warning_count warnings";
     eval {
         $stats .= "\n  total allowed cli records: " .
             NGCP::BulkProcessor::Projects::Migration::Teletek::Dao::import::AllowedCli::countby_ccacsn() . ' rows';
@@ -439,7 +441,7 @@ sub create_subscriber_task {
         ($result,$warning_count) = provision_subscribers();
     };
     my $err = $@;
-    my $stats = ($skip_errors ? ": $warning_count warnings" : '');
+    my $stats = ": $warning_count warnings";
     eval {
         #$stats .= "\n  total contracts: " .
         #    NGCP::BulkProcessor::Dao::Trunk::billing::contracts::countby_status_resellerid(undef,$reseller_id) . ' rows';

lib/NGCP/BulkProcessor/Projects/Migration/Teletek/reseller_mapping.yml

@@ -0,0 +1,4 @@
+mapping:
+  0700: 'Teleman'
+  Digifon: 'Teleman'
+  RingEtt: 'Teleman'

lib/NGCP/BulkProcessor/Projects/Migration/Teletek/settings.cfg

@@ -10,16 +10,17 @@ import_multithreading = 0
 #subscriber_filename = /home/rkrenn/temp/teletek/export_nummer_sip3.csv
 subscriber_filenames = /home/rkrenn/temp/teletek/export_kundinfo_leica.170823.csv,/home/rkrenn/temp/teletek/export_nummer_sip3.170904.csv
 subscriber_import_numofthreads = 2
-ignore_subscriber_unique = 1
+ignore_subscriber_unique = 0
 subscriber_import_single_row_txn = 1
 subscriber_import_unfold_ranges = 1
+reseller_mapping_yml = reseller_mapping.yml
 
 
 #allowedcli_filename = /home/rkrenn/test/teletek/export_multiple_DID_Leica.csv
 #allowedcli_filename = /home/rkrenn/temp/teletek/export_screeningOnly_170824.csv
-allowedcli_filenames = /home/rkrenn/temp/teletek/export_MultipleDID_170823.csv
+allowedcli_filenames = /home/rkrenn/temp/teletek/export_MultipleDID_170823.csv,/home/rkrenn/temp/teletek/export_screeningOnly_170824.csv
 allowedcli_import_numofthreads = 2
-ignore_allowedcli_unique = 1
+ignore_allowedcli_unique = 0
 allowedcli_import_single_row_txn = 1