TT#21083 implement teletek importer #3

+ allow to specify multiple subscriber files to import
+ allow to specify multiple allowed_cli files to import

Change-Id: I80f83542f541a0d8857d477158dc043cfabb93ba

lib/NGCP/BulkProcessor/Projects/Migration/Teletek/FileProcessors/CSVFile.pm

@@ -66,6 +66,16 @@ sub new {
 
 }
 
+sub process {
+
+    my $self = shift;
+
+    $self->{line_number} = 1;
+
+    return $self->SUPER::process(@_);
+
+}
+
 sub extractlines {
     my ($context,$buffer_ref,$lines) = @_;
 

lib/NGCP/BulkProcessor/Projects/Migration/Teletek/Import.pm

@@ -51,11 +51,13 @@ our @EXPORT_OK = qw(
 
 sub import_subscriber {
 
-    my ($file) = @_;
+    my (@files) = @_;
 
     my $result = NGCP::BulkProcessor::Projects::Migration::Teletek::Dao::import::Subscriber::create_table(0);
 
+    foreach my $file (@files) {
         $result &= _import_subscriber_checks($file);
+    }
 
     my $importer = NGCP::BulkProcessor::Projects::Migration::Teletek::FileProcessors::CSVFile->new($subscriber_import_numofthreads);
 
@@ -63,7 +65,8 @@ sub import_subscriber {
 
     destroy_all_dbs(); #close all db connections before forking..
     my $warning_count :shared = 0;
-    return ($result && $importer->process(
+    foreach my $file (@files) {
+        $result &= $importer->process(
             file => $file,
             process_code => sub {
                 my ($context,$rows,$row_offset) = @_;
@@ -164,7 +167,10 @@ sub import_subscriber {
                 }
             },
             multithreading => $import_multithreading
-    ),$warning_count);
+        );
+    }
+
+    return ($result,$warning_count);
 
 }
 
@@ -242,11 +248,13 @@ sub _insert_subscriber_rows {
 
 sub import_allowedcli {
 
-    my ($file) = @_;
+    my (@files) = @_;
 
     my $result = NGCP::BulkProcessor::Projects::Migration::Teletek::Dao::import::AllowedCli::create_table(0);
 
+    foreach my $file (@files) {
         $result &= _import_allowedcli_checks($file);
+    }
 
     my $importer = NGCP::BulkProcessor::Projects::Migration::Teletek::FileProcessors::CSVFile->new($allowedcli_import_numofthreads);
 
@@ -254,7 +262,8 @@ sub import_allowedcli {
 
     destroy_all_dbs(); #close all db connections before forking..
     my $warning_count :shared = 0;
-    return ($result && $importer->process(
+    foreach my $file (@files) {
+        $result &= $importer->process(
             file => $file,
             process_code => sub {
                 my ($context,$rows,$row_offset) = @_;
@@ -314,7 +323,10 @@ sub import_allowedcli {
                 }
             },
             multithreading => $import_multithreading
-    ),$warning_count);
+        );
+    }
+
+    return ($result,$warning_count);
 
 }
 

lib/NGCP/BulkProcessor/Projects/Migration/Teletek/Settings.pm

@@ -47,14 +47,14 @@ our @EXPORT_OK = qw(
     $force
     $import_db_file
 
-    $subscriber_filename
+    @subscriber_filenames
     $subscriber_import_numofthreads
     $ignore_subscriber_unique
     $subscriber_import_single_row_txn
     $subscriber_import_unfold_ranges
 
 
-    $allowedcli_filename
+    @allowedcli_filenames
     $allowedcli_import_numofthreads
     $ignore_allowedcli_unique
     $allowedcli_import_single_row_txn
@@ -101,13 +101,13 @@ our $run_id = '';
 our $import_db_file = _get_import_db_file($run_id,'import');
 our $import_multithreading = $enablemultithreading;
 
-our $subscriber_filename = undef;
+our @subscriber_filenames = ();
 our $subscriber_import_numofthreads = $cpucount;
 our $ignore_subscriber_unique = 0;
 our $subscriber_import_single_row_txn = 1;
 our $subscriber_import_unfold_ranges = 1;
 
-our $allowedcli_filename = undef;
+our @allowedcli_filenames = ();
 our $allowedcli_import_numofthreads = $cpucount;
 our $ignore_allowedcli_unique = 0;
 our $allowedcli_import_single_row_txn = 1;
@@ -158,13 +158,13 @@ sub update_settings {
         $import_db_file = _get_import_db_file($run_id,'import');
         $import_multithreading = $data->{import_multithreading} if exists $data->{import_multithreading};
 
-        $subscriber_filename = _get_import_filename($subscriber_filename,$data,'subscriber_filename');
+        @subscriber_filenames = _get_import_filenames(\@subscriber_filenames,$data,'subscriber_filenames');
         $subscriber_import_numofthreads = _get_numofthreads($cpucount,$data,'subscriber_import_numofthreads');
         $ignore_subscriber_unique = $data->{ignore_subscriber_unique} if exists $data->{ignore_subscriber_unique};
         $subscriber_import_single_row_txn = $data->{subscriber_import_single_row_txn} if exists $data->{subscriber_import_single_row_txn};
         $subscriber_import_unfold_ranges = $data->{subscriber_import_unfold_ranges} if exists $data->{subscriber_import_unfold_ranges};
 
-        $allowedcli_filename = _get_import_filename($allowedcli_filename,$data,'allowedcli_filename');
+        @allowedcli_filenames = _get_import_filenames(\@allowedcli_filenames,$data,'allowedcli_filenames');
         $allowedcli_import_numofthreads = _get_numofthreads($cpucount,$data,'allowedcli_import_numofthreads');
         $ignore_allowedcli_unique = $data->{ignore_allowedcli_unique} if exists $data->{ignore_allowedcli_unique};
         $allowedcli_import_single_row_txn = $data->{allowedcli_import_single_row_txn} if exists $data->{allowedcli_import_single_row_txn};
@@ -249,14 +249,18 @@ sub _get_import_db_file {
     return ((defined $run and length($run) > 0) ? $run . '_' : '') . $name;
 }
 
-sub _get_import_filename {
+sub _get_import_filenames {
     my ($old_value,$data,$key) = @_;
-    my $import_filename = $old_value;
-    $import_filename = $data->{$key} if exists $data->{$key};
+    my @import_filenames = @$old_value;
+    @import_filenames = split_tuple($data->{$key}) if exists $data->{$key};
+    my @result = ();
+    foreach my $import_filename (@import_filenames) {
         if (defined $import_filename and length($import_filename) > 0) {
             $import_filename = $input_path . $import_filename unless -e $import_filename;
+            push(@result,$import_filename);
         }
-    return $import_filename;
+    }
+    return @result;
 }
 
 sub check_dry {

lib/NGCP/BulkProcessor/Projects/Migration/Teletek/process.pl

@@ -20,10 +20,10 @@ use NGCP::BulkProcessor::Projects::Migration::Teletek::Settings qw(
     $skip_errors
     $force
     $run_id
-    $subscriber_filename
+    @subscriber_filenames
 
 
-    $allowedcli_filename
+    @allowedcli_filenames
 );
 #$allowed_ips
 
@@ -313,7 +313,7 @@ sub import_subscriber_task {
     my ($messages) = @_;
     my ($result,$warning_count) = (0,0);
     eval {
-        ($result,$warning_count) = import_subscriber($subscriber_filename);
+        ($result,$warning_count) = import_subscriber(@subscriber_filenames);
     };
     my $err = $@;
     my $stats = ($skip_errors ? ": $warning_count warnings" : '');
@@ -375,7 +375,7 @@ sub import_allowedcli_task {
     my ($messages) = @_;
     my ($result,$warning_count) = (0,0);
     eval {
-        ($result,$warning_count) = import_allowedcli($allowedcli_filename);
+        ($result,$warning_count) = import_allowedcli(@allowedcli_filenames);
     };
     my $err = $@;
     my $stats = ($skip_errors ? ": $warning_count warnings" : '');

lib/NGCP/BulkProcessor/Projects/Migration/Teletek/settings.cfg

@@ -7,7 +7,8 @@ import_multithreading = 0
 #subscriber_filename = /home/rkrenn/test/teletek/teletek_format_test1.txt
 #subscriber_filename = /home/rkrenn/test/teletek/teletek_format_test2.txt
 #subscriber_filename = /home/rkrenn/test/teletek/export_kundinfo_leica2.csv
-subscriber_filename = /home/rkrenn/temp/teletek/export_nummer_sip3.csv
+#subscriber_filename = /home/rkrenn/temp/teletek/export_nummer_sip3.csv
+subscriber_filenames = /home/rkrenn/temp/teletek/export_kundinfo_leica.170823.csv,/home/rkrenn/temp/teletek/export_nummer_sip3.170904.csv
 subscriber_import_numofthreads = 2
 ignore_subscriber_unique = 1
 subscriber_import_single_row_txn = 1
@@ -16,7 +17,7 @@ subscriber_import_unfold_ranges = 1
 
 #allowedcli_filename = /home/rkrenn/test/teletek/export_multiple_DID_Leica.csv
 #allowedcli_filename = /home/rkrenn/temp/teletek/export_screeningOnly_170824.csv
-allowedcli_filename = /home/rkrenn/temp/teletek/export_MultipleDID_170823.csv
+allowedcli_filenames = /home/rkrenn/temp/teletek/export_MultipleDID_170823.csv
 allowedcli_import_numofthreads = 2
 ignore_allowedcli_unique = 1
 allowedcli_import_single_row_txn = 1