TT#21083 implement Teletek/generic importer #2

+ fix process umask
+ make it work with new files from Teletek

Change-Id: I82ed65d5d6bae9cff911fdbc2d56a5a130d8cb44

lib/NGCP/BulkProcessor/Projects/Migration/Teletek/Provisioning.pm

@@ -122,7 +122,8 @@ sub provision_subscribers {
         uninit_process_context_code => sub {
             my ($context)= @_;
             undef $context->{db};
-            print('non-unique contacts: ' . join("\n",keys %{$context->{nonunique_contacts}}));
+            _warn($context,'non-unique contacts: ' . join("\n",keys %{$context->{nonunique_contacts}}))
+                if (scalar keys %{$context->{nonunique_contacts}}) > 0;
             destroy_all_dbs();
             {
                 lock $warning_count;
@@ -240,7 +241,7 @@ sub _provision_subscribers_checks {
                     $context->{reseller_map}->{$resellername} = NGCP::BulkProcessor::Dao::Trunk::billing::resellers::findby_name($resellername);
                 };
                 if ($@ or not $context->{reseller_map}->{$resellername}) {
-                    rowprocessingerror(threadid(),"cannot find reseller $resellername",getlogger(__PACKAGE__));
+                    rowprocessingerror(threadid(),"cannot find reseller '$resellername'",getlogger(__PACKAGE__));
                     $result = 0; #even in skip-error mode..
                 } else {
                     $context->{reseller_map}->{$resellername}->{billingprofile_map} = {};
@@ -251,7 +252,7 @@ sub _provision_subscribers_checks {
                     $context->{domain_map}->{$domain} = NGCP::BulkProcessor::Dao::Trunk::billing::domains::findby_domain($domain);
                 };
                 if ($@ or not $context->{domain_map}->{$domain}) {
-                    rowprocessingerror(threadid(),"cannot find domain $domain (billing)",getlogger(__PACKAGE__));
+                    rowprocessingerror(threadid(),"cannot find domain '$domain' (billing)",getlogger(__PACKAGE__));
                     $result = 0; #even in skip-error mode..
                 } else {
                     eval {
@@ -259,7 +260,7 @@ sub _provision_subscribers_checks {
                             NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_domains::findby_domain($domain);
                     };
                     if ($@ or not $context->{domain_map}->{$domain}->{prov_domain}) {
-                        rowprocessingerror(threadid(),"cannot find domain $domain (provisioning)",getlogger(__PACKAGE__));
+                        rowprocessingerror(threadid(),"cannot find domain '$domain' (provisioning)",getlogger(__PACKAGE__));
                         $result = 0; #even in skip-error mode..
                     }
                 }
@@ -286,7 +287,7 @@ sub _provision_subscribers_checks {
                         )->[0];
                 };
                 if ($@ or not $context->{reseller_map}->{$resellername}->{billingprofile_map}->{$billingprofilename}) {
-                    rowprocessingerror(threadid(),"cannot find billing profile $billingprofilename of reseller $resellername",getlogger(__PACKAGE__));
+                    rowprocessingerror(threadid(),"cannot find billing profile '$billingprofilename' of reseller '$resellername'",getlogger(__PACKAGE__));
                     $result = 0; #even in skip-error mode..
                 }
             }

lib/NGCP/BulkProcessor/Projects/Migration/Teletek/config.cfg

@@ -1,5 +1,5 @@
 ##general settings:
-working_path = /var/sipwise/Migration/Teletek
+working_path = /home/rkrenn/temp/teletek
 cpucount = 4
 enablemultithreading = 0
 
@@ -9,35 +9,35 @@ jobservers = 127.0.0.1:4730
 #provisioning_conf = /etc/ngcp-panel/provisioning.conf
 
 ##NGCP MySQL connectivity - "accounting" db:
-accounting_host = 127.0.0.1
+accounting_host = 192.168.0.84
 accounting_port = 3306
 accounting_databasename = accounting
 accounting_username = root
 accounting_password =
 
 ##NGCP MySQL connectivity - "billing" db:
-billing_host = 192.168.0.212
+billing_host = 192.168.0.84
 billing_port = 3306
 billing_databasename = billing
 billing_username = root
 billing_password =
 
 ##NGCP MySQL connectivity - "provisioning" db:
-provisioning_host = 192.168.0.212
+provisioning_host = 192.168.0.84
 provisioning_port = 3306
 provisioning_databasename = provisioning
 provisioning_username = root
 provisioning_password =
 
 ##NGCP MySQL connectivity - "kamailio" db:
-kamailio_host = 192.168.0.212
+kamailio_host = 192.168.0.84
 kamailio_port = 3306
 kamailio_databasename = kamailio
 kamailio_username = root
 kamailio_password =
 
 ##NGCP MySQL connectivity - default db for distributed transactions (XA) to connect to:
-xa_host = 192.168.0.212
+xa_host = 192.168.0.84
 xa_port = 3306
 xa_databasename = ngcp
 xa_username = root

lib/NGCP/BulkProcessor/Projects/Migration/Teletek/settings.cfg

@@ -6,14 +6,17 @@ import_multithreading = 0
 #subscriber_filename = /home/rkrenn/test/teletek/teletek_format.txt
 #subscriber_filename = /home/rkrenn/test/teletek/teletek_format_test1.txt
 #subscriber_filename = /home/rkrenn/test/teletek/teletek_format_test2.txt
-subscriber_filename = /home/rkrenn/test/teletek/export_kundinfo_leica2.csv
+#subscriber_filename = /home/rkrenn/test/teletek/export_kundinfo_leica2.csv
+subscriber_filename = /home/rkrenn/temp/teletek/export_nummer_sip3.csv
 subscriber_import_numofthreads = 2
 ignore_subscriber_unique = 1
 subscriber_import_single_row_txn = 1
 subscriber_import_unfold_ranges = 1
 
 
-allowedcli_filename = /home/rkrenn/test/teletek/export_multiple_DID_Leica.csv
+#allowedcli_filename = /home/rkrenn/test/teletek/export_multiple_DID_Leica.csv
+#allowedcli_filename = /home/rkrenn/temp/teletek/export_screeningOnly_170824.csv
+allowedcli_filename = /home/rkrenn/temp/teletek/export_MultipleDID_170823.csv
 allowedcli_import_numofthreads = 2
 ignore_allowedcli_unique = 1
 allowedcli_import_single_row_txn = 1

lib/NGCP/BulkProcessor/Utils.pm

@@ -107,7 +107,7 @@ our @EXPORT_OK = qw(
     check_ipnet
 );
 
-our $chmod_umask = 0644;
+our $chmod_umask = 0777;
 
 my $default_epsilon = 1e-3; #float comparison tolerance