From b464aba51a48267237ac026885e52c64363f4077 Mon Sep 17 00:00:00 2001 From: Rene Krenn Date: Thu, 16 Mar 2023 11:47:13 +0100 Subject: [PATCH] TT#56913 CustomerExporter: option to skip header line Change-Id: I33ce49971f05678da211c3c41644b01aaa3da90f (cherry picked from commit bd0797dd75b3fff5b969b01696da6e45f8adafeb) --- .../BulkProcessor/Projects/ETL/Customer/Settings.pm | 8 ++++++-- .../Projects/ETL/Customer/config.debug.cfg | 10 +++++----- .../BulkProcessor/Projects/ETL/Customer/process.pl | 4 +++- .../BulkProcessor/Projects/ETL/Customer/settings.cfg | 2 ++ lib/NGCP/BulkProcessor/SqlConnectors/CSVDB.pm | 2 ++ 5 files changed, 18 insertions(+), 8 deletions(-) diff --git a/lib/NGCP/BulkProcessor/Projects/ETL/Customer/Settings.pm b/lib/NGCP/BulkProcessor/Projects/ETL/Customer/Settings.pm index 7e75b47..19a115c 100644 --- a/lib/NGCP/BulkProcessor/Projects/ETL/Customer/Settings.pm +++ b/lib/NGCP/BulkProcessor/Projects/ETL/Customer/Settings.pm @@ -93,6 +93,8 @@ our @EXPORT_OK = qw( $export_customers_blocksize $csv_all_expected_fields + + $csv_header_line ); #$cf_default_priority #$cf_default_timeout @@ -122,6 +124,8 @@ our $customer_export_filename_format = undef; our $csv_all_expected_fields = 1; +our $csv_header_line = 1; + #our $customer_import_filename = undef; #our $customer_import_numofthreads = $cpucount; #our $customer_import_multithreading = 1; @@ -247,8 +251,6 @@ sub update_settings { $result = 0; } - $csv_all_expected_fields = $data->{csv_sep_char} if exists $data->{csv_all_expected_fields}; - $default_csv_config = { eol => "\r\n", sep_char => ';', @@ -260,6 +262,8 @@ sub update_settings { $default_csv_config->{quote_char} = unescape($data->{csv_quote_char}) if exists $data->{csv_quote_char}; $default_csv_config->{escape_char} = unescape($data->{csv_escape_char}) if exists $data->{csv_escape_char}; + $csv_header_line = $data->{csv_header_line} if exists $data->{csv_header_line}; + return $result; } return 0; diff --git a/lib/NGCP/BulkProcessor/Projects/ETL/Customer/config.debug.cfg b/lib/NGCP/BulkProcessor/Projects/ETL/Customer/config.debug.cfg index 200aede..504dc89 100644 --- a/lib/NGCP/BulkProcessor/Projects/ETL/Customer/config.debug.cfg +++ b/lib/NGCP/BulkProcessor/Projects/ETL/Customer/config.debug.cfg @@ -7,35 +7,35 @@ enablemultithreading = 1 jobservers = 127.0.0.1:4730 ##NGCP MySQL connectivity - "accounting" db: -accounting_host = 192.168.0.178 +accounting_host = 192.168.0.96 accounting_port = 3306 accounting_databasename = accounting accounting_username = root accounting_password = ##NGCP MySQL connectivity - "billing" db: -billing_host = 192.168.0.178 +billing_host = 192.168.0.96 billing_port = 3306 billing_databasename = billing billing_username = root billing_password = ##NGCP MySQL connectivity - "provisioning" db: -provisioning_host = 192.168.0.178 +provisioning_host = 192.168.0.96 provisioning_port = 3306 provisioning_databasename = provisioning provisioning_username = root provisioning_password = ##NGCP MySQL connectivity - "kamailio" db: -kamailio_host = 192.168.0.178 +kamailio_host = 192.168.0.96 kamailio_port = 3306 kamailio_databasename = kamailio kamailio_username = root kamailio_password = ##NGCP MySQL connectivity - default db for distributed transactions (XA) to connect to: -xa_host = 192.168.0.178 +xa_host = 192.168.0.96 xa_port = 3306 xa_databasename = ngcp xa_username = root diff --git a/lib/NGCP/BulkProcessor/Projects/ETL/Customer/process.pl b/lib/NGCP/BulkProcessor/Projects/ETL/Customer/process.pl index a298f4d..75118a4 100644 --- a/lib/NGCP/BulkProcessor/Projects/ETL/Customer/process.pl +++ b/lib/NGCP/BulkProcessor/Projects/ETL/Customer/process.pl @@ -31,6 +31,8 @@ use NGCP::BulkProcessor::Projects::ETL::Customer::Settings qw( $dry $skip_errors $force + + $csv_header_line ); use NGCP::BulkProcessor::Logging qw( @@ -272,7 +274,7 @@ sub export_customers_tabular_task { &get_sqlite_db()->copydbfile($export_filename); } elsif ('csv' eq $export_format) { NGCP::BulkProcessor::Projects::ETL::Customer::Dao::Tabular::copy_table(\&get_csv_db); - &get_csv_db()->copytablefile(NGCP::BulkProcessor::Projects::ETL::Customer::Dao::Tabular::gettablename(),$export_filename); + &get_csv_db()->copytablefile(NGCP::BulkProcessor::Projects::ETL::Customer::Dao::Tabular::gettablename(),$export_filename,not $csv_header_line); } else { push(@$messages,'invalid extension for output filename $export_filename'); } diff --git a/lib/NGCP/BulkProcessor/Projects/ETL/Customer/settings.cfg b/lib/NGCP/BulkProcessor/Projects/ETL/Customer/settings.cfg index de798c4..382c44b 100644 --- a/lib/NGCP/BulkProcessor/Projects/ETL/Customer/settings.cfg +++ b/lib/NGCP/BulkProcessor/Projects/ETL/Customer/settings.cfg @@ -21,6 +21,8 @@ csv_sep_char = , csv_quote_char = " csv_escape_char = " +csv_header_line = 0; + diff --git a/lib/NGCP/BulkProcessor/SqlConnectors/CSVDB.pm b/lib/NGCP/BulkProcessor/SqlConnectors/CSVDB.pm index 918c2cc..e1bfcdf 100644 --- a/lib/NGCP/BulkProcessor/SqlConnectors/CSVDB.pm +++ b/lib/NGCP/BulkProcessor/SqlConnectors/CSVDB.pm @@ -440,9 +440,11 @@ sub copytablefile { my $self = shift; my $tablename = shift; my $target = shift; + my $drop_header_line = shift; my $tablefilename = $self->_gettablefilename($tablename); $self->db_disconnect(); if (File::Copy::copy($tablefilename,$target)) { + `sed -i '1d' $target` if $drop_header_line; dbinfo($self,"$tablefilename copied to $target",getlogger(__PACKAGE__)); } else { dberror($self,"copy from $tablefilename to $target failed: $!",getlogger(__PACKAGE__));