TT#166652 contractexporter: option to omit internal cols in csv exports

Change-Id: Ie132459dca0c44b4336ebbd22b6d867993bb084d

lib/NGCP/BulkProcessor/Projects/ETL/Customer/Dao/Tabular.pm

@@ -10,6 +10,7 @@ use NGCP::BulkProcessor::Projects::ETL::Customer::ProjectConnectorPool qw(
 
 use NGCP::BulkProcessor::Projects::ETL::Customer::Settings qw(
     $tabular_fields
+    $csv_all_expected_fields
 );
 
 use NGCP::BulkProcessor::SqlProcessor qw(
@@ -189,13 +190,16 @@ sub countby_delta {
 sub copy_table {
     
     my ($get_target_db) = @_;
-     
-    check_table();
-    #checktableinfo($get_target_db,
-    #    __PACKAGE__,$tablename,
-    #    get_fieldnames(1),
-    #    $indexes);
 
+    if ($csv_all_expected_fields) {
+        check_table();
+    } else {
+        checktableinfo($get_db,
+            __PACKAGE__,$tablename,
+            get_fieldnames(0),
+            $indexes);
+    }
+    
     return transfer_table(
         get_db => $get_db,
         class => __PACKAGE__,

lib/NGCP/BulkProcessor/Projects/ETL/Customer/Settings.pm

@@ -91,14 +91,11 @@ our @EXPORT_OK = qw(
     $export_customers_numofthreads
     $export_customers_blocksize
    
-
-    $cf_default_priority
-    $cf_default_timeout
-    $cft_default_ringtimeout
-
-    $rollback_sql_export_filename_format
-    $rollback_sql_stmt_format
+    $csv_all_expected_fields
 );
+#$cf_default_priority
+#$cf_default_timeout
+#$cft_default_ringtimeout
 
 our $defaultconfig = 'config.cfg';
 our $defaultsettings = 'settings.cfg';
@@ -122,6 +119,8 @@ our $csv_dir = 'customer';
 
 our $customer_export_filename_format = undef;
 
+our $csv_all_expected_fields = 1;
+
 #our $customer_import_filename = undef;
 #our $customer_import_numofthreads = $cpucount;
 #our $customer_import_multithreading = 1;
@@ -239,6 +238,8 @@ sub update_settings {
         #$cf_default_timeout = $data->{cf_default_timeout} if exists $data->{cf_default_timeout};
         #$cft_default_ringtimeout = $data->{cft_default_ringtimeout} if exists $data->{cft_default_ringtimeout};
 
+        $csv_all_expected_fields = $data->{csv_all_expected_fields} if exists $data->{csv_all_expected_fields};
+        
         $mr = $data->{schema_version};
         if (not defined $mr or not contains($mr,\@supported_mr)) {
             configurationerror($configfile,'schema_version must be one of ' . join(', ', @supported_mr));

lib/NGCP/BulkProcessor/Projects/ETL/Customer/config.debug.cfg

@@ -7,35 +7,35 @@ enablemultithreading = 1
 jobservers = 127.0.0.1:4730
 
 ##NGCP MySQL connectivity - "accounting" db:
-accounting_host = 192.168.0.133
+accounting_host = 192.168.0.178
 accounting_port = 3306
 accounting_databasename = accounting
 accounting_username = root
 accounting_password =
 
 ##NGCP MySQL connectivity - "billing" db:
-billing_host = 192.168.0.133
+billing_host = 192.168.0.178
 billing_port = 3306
 billing_databasename = billing
 billing_username = root
 billing_password =
 
 ##NGCP MySQL connectivity - "provisioning" db:
-provisioning_host = 192.168.0.133
+provisioning_host = 192.168.0.178
 provisioning_port = 3306
 provisioning_databasename = provisioning
 provisioning_username = root
 provisioning_password =
 
 ##NGCP MySQL connectivity - "kamailio" db:
-kamailio_host = 192.168.0.133
+kamailio_host = 192.168.0.178
 kamailio_port = 3306
 kamailio_databasename = kamailio
 kamailio_username = root
 kamailio_password =
 
 ##NGCP MySQL connectivity - default db for distributed transactions (XA) to connect to:
-xa_host = 192.168.0.133
+xa_host = 192.168.0.178
 xa_port = 3306
 xa_databasename = ngcp
 xa_username = root

lib/NGCP/BulkProcessor/Projects/ETL/Customer/settings.cfg

@@ -7,13 +7,15 @@ export_customers_multithreading = 1
 export_customers_numofthreads = 4
 export_customers_blocksize = 1000
 
-customer_export_filename=customer_%s.json
+customer_export_filename=customer_%s.csv
 
 load_yml = load.yml
 tabular_yml = tabular.yml
 graph_yml = graph.yml
 graph_fields_mode = whitelist
 
+csv_all_expected_fields = 0
+