diff --git a/Build.PL b/Build.PL new file mode 100644 index 00000000..6839a6e5 --- /dev/null +++ b/Build.PL @@ -0,0 +1,74 @@ +use Module::Build qw(); +my $builder = Module::Build->new( + dist_name => 'NGCP::BulkProcessor', + dist_abstract => 'Framework for parallel/distributed processing of record blocks', + license => 'GPL_3', + dist_author => 'Rene Krenn ', + dist_version_from => 'lib/NGCP/BulkProcessor/Globals.pm', + perl_version_from => 'lib/NGCP/BulkProcessor/Globals.pm', + requires => { + 'Archive::Zip' => 0, + 'OLE::Storage_Lite' => 0, + 'XML::LibXML::Reader' => 0, + + 'Email::MIME' => 0, + 'Email::MIME::Attachment::Stripper' => 0, + + 'URI::Find' => 0, + 'LWP::UserAgent' => 0, + 'HTTP::Request' => 0, + + 'DateTime' => 0, + 'Time::HiRes' => 0, + 'Time::Warp' => 0, + 'DateTime::TimeZone' => 0, + 'DateTime::Format::Strptime' => 0, + 'DateTime::Format::ISO8601' => 0, + + 'Tie::IxHash' => 0, + + 'YAML::Tiny' => 0, + 'Log::Log4Perl' => 0, + 'MIME::Base64' => 0, + 'MIME::Lite' => 0, + 'Net::SMTP' => 0, + 'JSON::XS' => 0, + 'Data::Dump' => 0, + 'YAML::XS' => 0, + 'XML::Dumper' => '0.81', + 'PHP::Serialization' => 0, + #'Gearman::Worker' => 0, + 'Gearman::Client' => 0, + #'Gearman::Task' => 0, + 'Digest::MD5' => 0, + 'Data::UUID' => 0, + 'Net::Address::IP::Local' => 0, + 'Date::Manip' => 0, + 'Date::Calc' => 0, + #Sys::CpuAffinity + 'Marpa::R2' => 0, + 'Data::Dumper::Concise' => 0, + 'IO::Socket::SSL' => 0, + 'Mail::IMAPClient' => 0, + 'DBI' => '1.608', + 'DBD::CSV' => '0.26', + 'Locale::Recode' => 0, + 'Spreadsheet::ParseExcel' => 0, + 'Spreadsheet::ParseExcel::FmtUnicode' => 0, + 'Text::CSV_XS' => 0, + 'MIME::Parser' => 0, + 'HTML::Entities' => 0, + 'IO::Uncompress::Unzip' => 0, + 'DBD::mysql' => '4.014', + #'DBD::Oracle' => '1.21', + #'DBD::Pg' => '2.17.2', + #'DBD::ODBC' => '1.50', + 'DBD::SQLite' => '1.29', + }, + test_requires => { + 'Module::Runtime' => 0, + 'Test::Unit::Procedural' => 0, + }, + add_to_cleanup => ['NGCP-BulkProcessor-*', 'Excel-Reader-*'], +); +$builder->create_build_script; diff --git a/README b/README new file mode 100644 index 00000000..b0a1754e --- /dev/null +++ b/README @@ -0,0 +1,40 @@ +=encoding UTF-8 + +Bulk-Processor + +=head1 NAME + +README - basic information for users prior to downloading + +=head1 INSTALLATION + +See L. + +=head1 DEPENDENCIES + +See distribution meta file. + +=head1 LICENCE + +This software is Copyright © 2016 by Sipwise GmbH, Austria. + +This program is free software; you can redistribute it +and/or modify it under the terms of the GNU General Public +License as published by the Free Software Foundation; either +version 3 of the License, or (at your option) any later +version. + +This program is distributed in the hope that it will be +useful, but WITHOUT ANY WARRANTY; without even the implied +warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR +PURPOSE. See the GNU General Public License for more +details. + +You should have received a copy of the GNU General Public +License along with this package; if not, write to the Free +Software Foundation, Inc., 51 Franklin St, Fifth Floor, +Boston, MA 02110-1301 USA + +On Debian systems, the full text of the GNU General Public +License version 3 can be found in the file +F. diff --git a/debian/control b/debian/control index 824dfd52..aa4d77b6 100644 --- a/debian/control +++ b/debian/control @@ -11,6 +11,50 @@ Package: ngcp-bulk-processor-pro Architecture: all Depends: perl, + libarchive-zip-perl, + libole-storage-lite-perl, + libxml-libxml-perl, + libemail-mime-perl, + libemail-mime-attachment-stripper-perl, + liburi-find-perl, + libwww-perl, + libdatetime-perl, + libtime-warp-perl, + libdatetime-timezone-perl, + libdatetime-format-strptime-perl, + libdatetime-format-iso8601-perl, + libtie-ixhash-perl, + libyaml-tiny-perl, + liblog-log4perl-perl, + libmime-base64-perl, + libmime-lite-perl, + libnet-smtp-ssl-perl, + libjson-xs-perl, + libdata-dump-perl, + libyaml-libyaml-perl, + libxml-dumper-perl, + libphp-serialization-perl, + libgearman-client-perl, + libdigest-md5-perl, + libdata-uuid-perl, + libnet-address-ip-local-perl, + libdate-manip-perl, + libdate-calc-perl, + libmarpa-r2-perl, + libdata-dumper-concise-perl, + libio-socket-ssl-perl, + libmail-imapclient-perl, + libdbd-csv-perl, + libintl-perl, + libspreadsheet-parseexcel-perl, + libtext-csv-xs-perl, + libmime-tools-perl, + libemail-mime-perl, + libhtml-parser-perl, + libio-compress-perl, + libdbd-mysql-perl, + libdbd-sqlite3-perl, ${misc:Depends}, -Description: NGCP bulk-processor tool - Tool for bulk process for NGCP. + ${perl:Depends} +Description: NGCP bulk processor framework + Framework for parallel/distributed processing of record blocks. diff --git a/lib/NGCP/BulkProcessor/ConnectorPool.pm b/lib/NGCP/BulkProcessor/ConnectorPool.pm index 2b7fe2f0..64884154 100644 --- a/lib/NGCP/BulkProcessor/ConnectorPool.pm +++ b/lib/NGCP/BulkProcessor/ConnectorPool.pm @@ -37,7 +37,7 @@ use NGCP::BulkProcessor::SqlConnectors::CSVDB; #use NGCP::BulkProcessor::SqlConnectors::SQLServerDB; use NGCP::BulkProcessor::RestConnectors::NGCPRestApi; -use NGCP::BulkProcessor::SqlRecord qw(cleartableinfo); +use NGCP::BulkProcessor::SqlProcessor qw(cleartableinfo); use NGCP::BulkProcessor::Utils qw(threadid); diff --git a/lib/NGCP/BulkProcessor/Dao/Trunk/billing/billing_mappings.pm b/lib/NGCP/BulkProcessor/Dao/Trunk/billing/billing_mappings.pm new file mode 100644 index 00000000..fb64a39b --- /dev/null +++ b/lib/NGCP/BulkProcessor/Dao/Trunk/billing/billing_mappings.pm @@ -0,0 +1,110 @@ +package NGCP::BulkProcessor::Dao::Trunk::billing::billing_mappings; +use strict; + +## no critic + +#use File::Basename; +#use Cwd; +#use lib Cwd::abs_path(File::Basename::dirname(__FILE__) . '/../../../'); + +use NGCP::BulkProcessor::ConnectorPool qw( + get_billing_db + +); + +use NGCP::BulkProcessor::SqlProcessor qw( + checktableinfo + insert_record +); +use NGCP::BulkProcessor::SqlRecord qw(); + +require Exporter; +our @ISA = qw(Exporter NGCP::BulkProcessor::SqlRecord); +our @EXPORT_OK = qw( + gettablename + check_table + insert_row +); + +#my $logger = getlogger(__PACKAGE__); + +my $tablename = 'billing_mappings'; +my $get_db = \&get_billing_db; + +my $expected_fieldnames = [ + 'id', + 'start_date', + 'end_date', + 'billing_profile_id', + 'contract_id', + 'product_id', + 'network_id', +]; + +my $indexes = {}; +# 'balance_interval' => [ 'contract_id','start','end' ], +# 'invoice_idx' => [ 'invoice_id' ], +#}; + +my $insert_unique_fields = []; #[ 'contract_id','start','end' ]; + +sub new { + + my $class = shift; + my $self = NGCP::BulkProcessor::SqlRecord->new($get_db, + $tablename, + $expected_fieldnames,$indexes); + + bless($self,$class); + + copy_row($self,shift,$expected_fieldnames); + + return $self; + +} + +sub insert_row { + + my ($data,$insert_ignore) = @_; + check_table(); + return insert_record($get_db,$tablename,$data,$insert_ignore,$unique_fields) = @_; + +} + +sub buildrecords_fromrows { + + my ($rows,$load_recursive) = @_; + + my @records = (); + my $record; + + if (defined $rows and ref $rows eq 'ARRAY') { + foreach my $row (@$rows) { + $record = __PACKAGE__->new($row); + + # transformations go here ... + + push @records,$record; + } + } + + return \@records; + +} + +sub gettablename { + + return $tablename; + +} + +sub check_table { + + return checktableinfo($get_db, + $tablename, + $expected_fieldnames, + $indexes); + +} + +1; diff --git a/lib/NGCP/BulkProcessor/Dao/Trunk/billing/contract_balances.pm b/lib/NGCP/BulkProcessor/Dao/Trunk/billing/contract_balances.pm index 1bda5f12..f426f318 100644 --- a/lib/NGCP/BulkProcessor/Dao/Trunk/billing/contract_balances.pm +++ b/lib/NGCP/BulkProcessor/Dao/Trunk/billing/contract_balances.pm @@ -9,27 +9,27 @@ use strict; use NGCP::BulkProcessor::ConnectorPool qw( get_billing_db - billing_db_tableidentifier + ); -use NGCP::BulkProcessor::SqlRecord qw(checktableinfo); +use NGCP::BulkProcessor::SqlProcessor qw( + checktableinfo + insert_record +); +use NGCP::BulkProcessor::SqlRecord qw(); require Exporter; our @ISA = qw(Exporter NGCP::BulkProcessor::SqlRecord); our @EXPORT_OK = qw( -XX -backoffice_client_byboclientid - sync_table - drop_table - - check_local_table - check_source_table); + gettablename + check_table + insert_row +); #my $logger = getlogger(__PACKAGE__); my $tablename = 'contract_balances'; my $get_db = \&get_billing_db; -my $get_tablename = \&billing_db_tableidentifier; my $expected_fieldnames = [ 'id', @@ -47,14 +47,19 @@ my $expected_fieldnames = [ 'underrun_lock', ]; -#my $indexes = { $tablename . '_subscribernumber' => ['subscribernumber(11)'] }; +my $indexes = {}; +# 'balance_interval' => [ 'contract_id','start','end' ], +# 'invoice_idx' => [ 'invoice_id' ], +#}; + +my $insert_unique_fields = []; #[ 'contract_id','start','end' ]; sub new { my $class = shift; my $self = NGCP::BulkProcessor::SqlRecord->new($get_db, - gettablename(), - $expected_fieldnames); + $tablename, + $expected_fieldnames,$indexes); bless($self,$class); @@ -64,6 +69,13 @@ sub new { } +sub insert_row { + + my ($data,$insert_ignore) = @_; + check_table(); + return insert_record($get_db,$tablename,$data,$insert_ignore,$unique_fields) = @_; + +} sub buildrecords_fromrows { @@ -88,15 +100,16 @@ sub buildrecords_fromrows { sub gettablename { - return &$get_tablename($get_db,$tablename); + return $tablename; } sub check_table { return checktableinfo($get_db, - gettablename(), - $expected_fieldnames); + $tablename, + $expected_fieldnames, + $indexes); } diff --git a/lib/NGCP/BulkProcessor/Dao/Trunk/billing/contracts.pm b/lib/NGCP/BulkProcessor/Dao/Trunk/billing/contracts.pm new file mode 100644 index 00000000..ab53e7f0 --- /dev/null +++ b/lib/NGCP/BulkProcessor/Dao/Trunk/billing/contracts.pm @@ -0,0 +1,122 @@ +package NGCP::BulkProcessor::Dao::Trunk::billing::contracts; +use strict; + +## no critic + +#use File::Basename; +#use Cwd; +#use lib Cwd::abs_path(File::Basename::dirname(__FILE__) . '/../../../'); + +use NGCP::BulkProcessor::ConnectorPool qw( + get_billing_db + +); + +use NGCP::BulkProcessor::SqlProcessor qw( + checktableinfo + insert_record +); +use NGCP::BulkProcessor::SqlRecord qw(); + +require Exporter; +our @ISA = qw(Exporter NGCP::BulkProcessor::SqlRecord); +our @EXPORT_OK = qw( + gettablename + check_table + insert_row +); + +#my $logger = getlogger(__PACKAGE__); + +my $tablename = 'contracts'; +my $get_db = \&get_billing_db; + +my $expected_fieldnames = [ + 'id', + 'customer_id', + 'contact_id', + 'order_id', + 'profile_package_id', + 'status', + 'external_id', + 'modify_timestamp', + 'create_timestamp', + 'activate_timestamp', + 'terminate_timestamp', + 'max_subscribers', + 'send_invoice', + 'subscriber_email_template_id', + 'passreset_email_template_id', + 'invoice_email_template_id', + 'invoice_template_id', + 'vat_rate', + 'add_vat', +]; + +my $indexes = {}; +# 'balance_interval' => [ 'contract_id','start','end' ], +# 'invoice_idx' => [ 'invoice_id' ], +#}; + +my $insert_unique_fields = []; #[ 'contract_id','start','end' ]; + +sub new { + + my $class = shift; + my $self = NGCP::BulkProcessor::SqlRecord->new($get_db, + $tablename, + $expected_fieldnames,$indexes); + + bless($self,$class); + + copy_row($self,shift,$expected_fieldnames); + + return $self; + +} + +sub insert_row { + + my ($data,$insert_ignore) = @_; + check_table(); + return insert_record($get_db,$tablename,$data,$insert_ignore,$unique_fields) = @_; + +} + +sub buildrecords_fromrows { + + my ($rows,$load_recursive) = @_; + + my @records = (); + my $record; + + if (defined $rows and ref $rows eq 'ARRAY') { + foreach my $row (@$rows) { + $record = __PACKAGE__->new($row); + + # transformations go here ... + + push @records,$record; + } + } + + return \@records; + +} + +sub gettablename { + + return $tablename; + +} + +sub check_table { + + return checktableinfo($get_db, + $tablename, + $expected_fieldnames, + $indexes); + +} + +1; diff --git a/lib/NGCP/BulkProcessor/FakeTime.pm b/lib/NGCP/BulkProcessor/FakeTime.pm index 9e1a9c4a..1bfc15d4 100644 --- a/lib/NGCP/BulkProcessor/FakeTime.pm +++ b/lib/NGCP/BulkProcessor/FakeTime.pm @@ -3,6 +3,7 @@ use strict; ## no critic +use DateTime qw(); use Time::HiRes qw(); #prevent warning from Time::Warp use Time::Warp qw(); use DateTime::TimeZone qw(); diff --git a/lib/NGCP/BulkProcessor/FileProcessor.pm b/lib/NGCP/BulkProcessor/FileProcessor.pm index 1c8bf193..334495d3 100644 --- a/lib/NGCP/BulkProcessor/FileProcessor.pm +++ b/lib/NGCP/BulkProcessor/FileProcessor.pm @@ -88,7 +88,19 @@ sub process { my $self = shift; - my ($file,$process_code,$init_process_context_code,$uninit_process_context_code,$multithreading) = @_; + my %params = @_; + my ($file, + $process_code, + $init_process_context_code, + $uninit_process_context_code, + $multithreading) = @params{qw/ + file + process_code + init_process_context_code + uninit_process_context_code + multithreading + /}; + #my ($file,$process_code,$init_process_context_code,$uninit_process_context_code,$multithreading) = @_; if (ref $process_code eq 'CODE') { diff --git a/lib/NGCP/BulkProcessor/Globals.pm b/lib/NGCP/BulkProcessor/Globals.pm index 5dabdc57..c1ea6218 100644 --- a/lib/NGCP/BulkProcessor/Globals.pm +++ b/lib/NGCP/BulkProcessor/Globals.pm @@ -107,7 +107,8 @@ umask 0000; # general constants our $system_name = 'Sipwise Bulk Processing Framework'; -our $system_version = '0.0.1'; #keep this filename-save +our $VERSION = '0.0.1'; +our $system_version = $VERSION; #keep this filename-save our $system_abbreviation = 'sbpf'; #keep this filename-, dbname-save our $system_instance = 'initial'; #'test'; #'2014'; #dbname-save 0-9a-z_ our $system_instance_label = 'test'; diff --git a/lib/NGCP/BulkProcessor/Logging.pm b/lib/NGCP/BulkProcessor/Logging.pm index 39a3ba28..d2017073 100644 --- a/lib/NGCP/BulkProcessor/Logging.pm +++ b/lib/NGCP/BulkProcessor/Logging.pm @@ -509,9 +509,9 @@ sub rowinsertskipped { sub rowupdateskipped { - my ($db,$tablename,$logger) = @_; + my ($db,$tablename,$matched,$logger) = @_; if (defined $logger) { - $logger->info(_getsqlconnectorinstanceprefix($db) . 'row update skipped'); + $logger->info(_getsqlconnectorinstanceprefix($db) . "row update skipped, $matched matching rows"); } } diff --git a/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Dao/FeatureOption.pm b/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Dao/import/FeatureOption.pm similarity index 88% rename from lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Dao/FeatureOption.pm rename to lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Dao/import/FeatureOption.pm index 7868a774..7078535f 100644 --- a/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Dao/FeatureOption.pm +++ b/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Dao/import/FeatureOption.pm @@ -1,4 +1,4 @@ -package NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::FeatureOption; +package NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOption; use strict; ## no critic @@ -13,7 +13,7 @@ use NGCP::BulkProcessor::Projects::Migration::IPGallery::ProjectConnectorPool qw ); #import_db_tableidentifier -use NGCP::BulkProcessor::SqlRecord qw( +use NGCP::BulkProcessor::SqlProcessor qw( registertableinfo create_targettable checktableinfo @@ -21,8 +21,9 @@ use NGCP::BulkProcessor::SqlRecord qw( insert_stmt ); +use NGCP::BulkProcessor::SqlRecord qw(); -use NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::FeatureOptionSetItem qw(); +use NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOptionSetItem qw(); require Exporter; our @ISA = qw(Exporter NGCP::BulkProcessor::SqlRecord); @@ -41,14 +42,15 @@ my $get_db = \&get_import_db; #my $get_tablename = \&import_db_tableidentifier; -my $expected_fieldnames = [ 'subscribernumber', - 'option']; +my $expected_fieldnames = [ + 'subscribernumber', + 'option' +]; +# table creation: my $primarykey_fieldnames = [ 'subscribernumber', 'option' ]; - my $indexes = {}; - -my $fixtable_statements = []; +#my $fixtable_statements = []; sub new { @@ -127,7 +129,7 @@ sub buildrecords_fromrows { # transformations go here ... if ($load_recursive) { - $record->{_optionsetitems} = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::FeatureOptionSetItem::findby_subscribernumber_option( + $record->{_optionsetitems} = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOptionSetItem::findby_subscribernumber_option( $record->{subscribernumber}, $record->{option}, $load_recursive diff --git a/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Dao/FeatureOptionSetItem.pm b/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Dao/import/FeatureOptionSetItem.pm similarity index 88% rename from lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Dao/FeatureOptionSetItem.pm rename to lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Dao/import/FeatureOptionSetItem.pm index 34385b06..4ba73863 100644 --- a/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Dao/FeatureOptionSetItem.pm +++ b/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Dao/import/FeatureOptionSetItem.pm @@ -1,4 +1,4 @@ -package NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::FeatureOptionSetItem; +package NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOptionSetItem; use strict; ## no critic @@ -13,7 +13,7 @@ use NGCP::BulkProcessor::Projects::Migration::IPGallery::ProjectConnectorPool qw ); #import_db_tableidentifier -use NGCP::BulkProcessor::SqlRecord qw( +use NGCP::BulkProcessor::SqlProcessor qw( registertableinfo create_targettable checktableinfo @@ -21,6 +21,7 @@ use NGCP::BulkProcessor::SqlRecord qw( insert_stmt ); +use NGCP::BulkProcessor::SqlRecord qw(); require Exporter; our @ISA = qw(Exporter NGCP::BulkProcessor::SqlRecord); @@ -39,15 +40,18 @@ my $get_db = \&get_import_db; #my $get_tablename = \&import_db_tableidentifier; -my $expected_fieldnames = [ 'subscribernumber', - 'option', - 'optionsetitem' ]; +my $expected_fieldnames = [ + 'subscribernumber', + 'option', + 'optionsetitem' +]; +# table creation: my $primarykey_fieldnames = []; #[ 'subscribernumber', 'option', 'optionsetitem' ]; - -my $indexes = { $tablename . '_subscribernumber_option_optionsetitem' => ['subscribernumber(11)', 'option(32)', 'optionsetitem(32)'] }; #(25),(27) - -my $fixtable_statements = []; +my $indexes = { + $tablename . '_subscribernumber_option_optionsetitem' => ['subscribernumber(11)', 'option(32)', 'optionsetitem(32)'], #(25),(27) +}; +#my $fixtable_statements = []; sub new { diff --git a/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Dao/Lnp.pm b/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Dao/import/Lnp.pm similarity index 96% rename from lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Dao/Lnp.pm rename to lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Dao/import/Lnp.pm index ffdde5c0..2e2ac79b 100644 --- a/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Dao/Lnp.pm +++ b/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Dao/import/Lnp.pm @@ -1,4 +1,4 @@ -package NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::Lnp; +package NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Lnp; use strict; ## no critic @@ -13,7 +13,7 @@ use NGCP::BulkProcessor::Projects::Migration::IPGallery::ProjectConnectorPool qw ); #import_db_tableidentifier -use NGCP::BulkProcessor::SqlRecord qw( +use NGCP::BulkProcessor::SqlProcessor qw( registertableinfo create_targettable checktableinfo @@ -21,6 +21,7 @@ use NGCP::BulkProcessor::SqlRecord qw( insert_stmt ); +use NGCP::BulkProcessor::SqlRecord qw(); require Exporter; our @ISA = qw(Exporter NGCP::BulkProcessor::SqlRecord); @@ -46,11 +47,10 @@ my $expected_fieldnames = [ 'lrn_code', ]; +# table creation: my $primarykey_fieldnames = [ 'lrn_code', 'ported_number' ]; - my $indexes = {}; - -my $fixtable_statements = []; +#my $fixtable_statements = []; sub new { diff --git a/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Dao/Subscriber.pm b/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Dao/import/Subscriber.pm similarity index 58% rename from lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Dao/Subscriber.pm rename to lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Dao/import/Subscriber.pm index 346905bf..e6c5b100 100644 --- a/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Dao/Subscriber.pm +++ b/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Dao/import/Subscriber.pm @@ -1,4 +1,4 @@ -package NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::Subscriber; +package NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber; use strict; ## no critic @@ -13,7 +13,7 @@ use NGCP::BulkProcessor::Projects::Migration::IPGallery::ProjectConnectorPool qw ); #import_db_tableidentifier -use NGCP::BulkProcessor::SqlRecord qw( +use NGCP::BulkProcessor::SqlProcessor qw( registertableinfo create_targettable checktableinfo @@ -21,6 +21,7 @@ use NGCP::BulkProcessor::SqlRecord qw( insert_stmt ); +use NGCP::BulkProcessor::SqlRecord qw(); require Exporter; our @ISA = qw(Exporter NGCP::BulkProcessor::SqlRecord); @@ -29,9 +30,17 @@ our @EXPORT_OK = qw( gettablename check_table getinsertstatement + getupsertstatement findby_subscribernumber countby_subscribernumber + update_delta + findby_delta + countby_delta + + $deleted_delta + $updated_delta + $added_delta ); my $tablename = 'subscriber'; @@ -50,13 +59,17 @@ my $expected_fieldnames = [ 'time_zone_name', #malta 'lang_code', #eng 'barring_profile', #None + 'delta', ]; +# table creation: my $primarykey_fieldnames = [ 'country_code', 'area_code', 'dial_number' ]; +my $indexes = { $tablename . '_delta' => [ 'delta(7)' ]}; +#my $fixtable_statements = []; -my $indexes = {}; - -my $fixtable_statements = []; +our $deleted_delta = 'DELETED'; +our $updated_delta = 'UPDATED'; +our $added_delta = 'ADDED'; sub new { @@ -84,6 +97,27 @@ sub create_table { } +sub findby_delta { + + my ($delta,$load_recursive) = @_; + + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + + return [] unless defined $delta; + + my $rows = $db->db_get_all_arrayref( + 'SELECT * FROM ' . + $table . + ' WHERE ' . + $db->columnidentifier('delta') . ' = ?' + ,$delta); + + return buildrecords_fromrows($rows,$load_recursive); + +} + sub findby_subscribernumber { my ($subscribernumber,$load_recursive) = @_; @@ -129,6 +163,49 @@ sub countby_subscribernumber { } +sub countby_delta { + + my ($delta) = @_; + + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + + my $stmt = 'SELECT COUNT(*) FROM ' . $table; + my @params = (); + if (defined $delta) { + $stmt .= ' WHERE ' . + $db->columnidentifier('delta') . ' = ?'; + push(@params,$delta); + } + + return $db->db_get_value($stmt,@params); + +} + +sub update_delta { + + my ($subscribernumber,$delta) = @_; + + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + + my $stmt = 'UPDATE ' . $table . ' SET delta = ?'; + my @params = (); + push(@params,$delta); + if (defined $subscribernumber) { + $stmt .= ' WHERE ' . + $db->columnidentifier('country_code') . ' = ?' . + ' AND ' . $db->columnidentifier('area_code') . ' = ?' . + ' AND ' . $db->columnidentifier('dial_number') . ' = ?'; + push(@params,split_subscribernumber($subscribernumber)); + } + + return $db->db_do($stmt,@params); + +} + sub buildrecords_fromrows { my ($rows,$load_recursive) = @_; @@ -142,7 +219,7 @@ sub buildrecords_fromrows { # transformations go here ... if ($load_recursive) { - $record->{_features} = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::FeatureOption::findby_subscribernumber( + $record->{_features} = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOption::findby_subscribernumber( $record->subscribernumber(), $load_recursive ); @@ -164,6 +241,31 @@ sub getinsertstatement { } +sub getupsertstatement { + + my ($exists_delta,$new_delta) = @_; + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + my $upsert_stmt = 'INSERT OR REPLACE INTO ' . $table . ' (' . + join(', ',map { local $_ = $_; $_ = $db->columnidentifier($_); $_; } @$expected_fieldnames) . ')'; + my @values = (); + foreach my $fieldname (@$expected_fieldnames) { + if ('delta' eq $fieldname) { + my $stmt = 'SELECT \'' . $exists_delta . '\' FROM ' . $table . ' WHERE ' . + $db->columnidentifier('country_code') . ' = ?' . + ' AND ' . $db->columnidentifier('area_code') . ' = ?' . + ' AND ' . $db->columnidentifier('dial_number') . ' = ?'; + push(@values,'COALESCE((' . $stmt . '), \'' . $new_delta . '\')'); + } else { + push(@values,'?'); + } + } + $upsert_stmt .= ' VALUES (' . join(',',@values) . ')'; + return $upsert_stmt; + +} + sub gettablename { return $tablename; diff --git a/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/FeaturesDefineParser.pm b/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/FeaturesDefineParser.pm index b3e693e2..8da2e65a 100644 --- a/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/FeaturesDefineParser.pm +++ b/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/FeaturesDefineParser.pm @@ -31,7 +31,7 @@ OptionValues ::= OptionValue+ action => _build_setoptio SubscriberNumber ~ [0-9]+ OptionName ~ [-a-zA-Z_0-9]+ -OptionValue ~ [-a-zA-Z_0-9 \t]+ +OptionValue ~ [-a-zA-Z_0-9 \t#*]+ whitespace ~ [\s]+ :discard ~ whitespace __GRAMMAR__ diff --git a/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Import.pm b/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Import.pm index c16e8d4e..dea57dc9 100644 --- a/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Import.pm +++ b/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Import.pm @@ -40,15 +40,16 @@ use NGCP::BulkProcessor::Projects::Migration::IPGallery::FileProcessors::LnpDefi use NGCP::BulkProcessor::Projects::Migration::IPGallery::ProjectConnectorPool qw( get_import_db - destroy_dbs + destroy_all_dbs ); -use NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::FeatureOption qw(); -use NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::FeatureOptionSetItem qw(); -use NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::Subscriber qw(); -use NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::Lnp qw(); +use NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOption qw(); +use NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOptionSetItem qw(); +use NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber qw(); +use NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Lnp qw(); use NGCP::BulkProcessor::Array qw(removeduplicates); +use NGCP::BulkProcessor::Utils qw(threadid); require Exporter; our @ISA = qw(Exporter); @@ -62,8 +63,8 @@ sub import_features_define { my ($file) = @_; # create tables: - my $result = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::FeatureOption::create_table(1); - $result &= NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::FeatureOptionSetItem::create_table(1); + my $result = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOption::create_table(1); + $result &= NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOptionSetItem::create_table(1); # checks, e.g. other table must be present: # ..none.. @@ -73,8 +74,10 @@ sub import_features_define { $importer->stoponparseerrors(!$dry); # launch: - destroy_dbs(); #close all db connections before forking.. - return $result && $importer->process($file,sub { + destroy_all_dbs(); #close all db connections before forking.. + return $result && $importer->process( + file => $file, + process_code => sub { my ($context,$rows,$row_offset) = @_; my $rownum = $row_offset; my @featureoption_rows = (); @@ -126,7 +129,8 @@ sub import_features_define { } } return 1; - }, sub { + }, + init_process_context_code => sub { my ($context)= @_; if (not $importer->parselines()) { eval { @@ -137,19 +141,22 @@ sub import_features_define { } } $context->{db} = &get_import_db(); # keep ref count low.. - }, sub { + }, + uninit_process_context_code => sub { my ($context)= @_; undef $context->{db}; - destroy_dbs(); - },$import_multithreading); + destroy_all_dbs(); + }, + multithreading => $import_multithreading + ); } sub _insert_featureoption_rows { my ($context,$featureoption_rows) = @_; $context->{db}->db_do_begin( - NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::FeatureOption::getinsertstatement($ignore_options_unique), - #NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::FeatureOption::gettablename(), + NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOption::getinsertstatement($ignore_options_unique), + #NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOption::gettablename(), #lock - $import_multithreading ); $context->{db}->db_do_rowblock($featureoption_rows); @@ -159,8 +166,8 @@ sub _insert_featureoption_rows { sub _insert_featureoptionsetitem_rows { my ($context,$featureoptionsetitem_rows) = @_; $context->{db}->db_do_begin( - NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::FeatureOptionSetItem::getinsertstatement($ignore_setoptionitems_unique), - #NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::FeatureOptionSetItem::gettablename(), + NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOptionSetItem::getinsertstatement($ignore_setoptionitems_unique), + #NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOptionSetItem::gettablename(), #lock ); $context->{db}->db_do_rowblock($featureoptionsetitem_rows); @@ -171,57 +178,74 @@ sub import_subscriber_define { my ($file) = @_; - my $result = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::Subscriber::create_table(1); + my $result = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber::create_table(0); $result &= _import_subscriber_define_checks($file); my $importer = NGCP::BulkProcessor::Projects::Migration::IPGallery::FileProcessors::SubscriberDefineFile->new($subscriber_define_import_numofthreads); - destroy_dbs(); #close all db connections before forking.. - return $result && $importer->process($file,sub { + my $upsert = _import_subscriber_define_reset_delta(); + + destroy_all_dbs(); #close all db connections before forking.. + return $result && $importer->process( + file => $file, + process_code => sub { my ($context,$rows,$row_offset) = @_; my $rownum = $row_offset; my @subscriber_rows = (); foreach my $row (@$rows) { $rownum++; - my $record = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::Subscriber->new($row); + my $record = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber->new($row); next if 'None' eq $record->{rgw_fqdn}; if ($record->{dial_number} =~ $subscribernumer_exclude_pattern) { if ($record->{dial_number} =~ $subscribernumer_exclude_exception_pattern) { processing_info($context->{tid},'record ' . $rownum . ' - exclude exception pattern match: ' . $record->{dial_number},getlogger(__PACKAGE__)); - push(@subscriber_rows,$row) if _import_subscriber_define_referential_checks($context,$record,$rownum); + next unless _import_subscriber_define_referential_checks($context,$record,$rownum); } else { processing_info($context->{tid},'record ' . $rownum . ' - skipped, exclude pattern match: ' . $record->{dial_number},getlogger(__PACKAGE__)); + next; } } else { - push(@subscriber_rows,$row) if _import_subscriber_define_referential_checks($context,$record,$rownum); + next unless _import_subscriber_define_referential_checks($context,$record,$rownum); } + my @subscriber_row = @$row; + if ($context->{upsert}) { + push(@subscriber_row,$record->{country_code},$record->{area_code},$record->{dial_number}); + } else { + push(@subscriber_row,$NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber::added_delta); + } + push(@subscriber_rows,\@subscriber_row); } - if ((scalar @$rows) > 0) { + if ((scalar @subscriber_rows) > 0) { if ($dry) { - eval { _insert_subscriber_rows($context,$rows); }; + eval { _insert_subscriber_rows($context,\@subscriber_rows); }; } else { - _insert_subscriber_rows($context,$rows); + _insert_subscriber_rows($context,\@subscriber_rows); } } return 1; - }, sub { + }, + init_process_context_code => sub { my ($context)= @_; $context->{db} = &get_import_db(); # keep ref count low.. - }, sub { + $context->{upsert} = $upsert; + }, + uninit_process_context_code => sub { my ($context)= @_; undef $context->{db}; - destroy_dbs(); - }, $import_multithreading); + destroy_all_dbs(); + }, + multithreading => $import_multithreading + ); } sub _import_subscriber_define_referential_checks { my ($context,$record,$rownum) = @_; my $result = 0; - if (NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::FeatureOption::countby_subscribernumber($record->subscribernumber()) > 0) { + if (NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOption::countby_subscribernumber($record->subscribernumber()) > 0) { $result = 1; } else { if ($dry) { @@ -239,7 +263,7 @@ sub _import_subscriber_define_checks { my $result = 1; my $optioncount = 0; eval { - $optioncount = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::FeatureOption::countby_subscribernumber(); + $optioncount = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOption::countby_subscribernumber(); }; if ($@ or $optioncount == 0) { fileprocessingerror($file,'please import subscriber features first',getlogger(__PACKAGE__)); @@ -248,11 +272,27 @@ sub _import_subscriber_define_checks { return $result; } +sub _import_subscriber_define_reset_delta { + if (NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber::countby_subscribernumber() > 0) { + processing_info(threadid(),'resetting subscriber delta of ' . + NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber::update_delta(undef, + $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber::deleted_delta) . + ' records',getlogger(__PACKAGE__)); + return 1; + } + return 0; +} + sub _insert_subscriber_rows { my ($context,$subscriber_rows) = @_; $context->{db}->db_do_begin( - NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::Subscriber::getinsertstatement($ignore_subscriber_unique), - #NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::Subscriber::gettablename(), + ($context->{upsert} ? + NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber::getupsertstatement( + $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber::updated_delta, + $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber::added_delta + ) + : NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber::getinsertstatement($ignore_subscriber_unique)), + #NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber::gettablename(), #lock ); $context->{db}->db_do_rowblock($subscriber_rows); @@ -263,12 +303,14 @@ sub import_lnp_define { my ($file) = @_; - my $result = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::Lnp::create_table(1); + my $result = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Lnp::create_table(1); my $importer = NGCP::BulkProcessor::Projects::Migration::IPGallery::FileProcessors::LnpDefineFile->new($lnp_define_import_numofthreads); - destroy_dbs(); #close all db connections before forking.. - return $result && $importer->process($file,sub { + destroy_all_dbs(); #close all db connections before forking.. + return $result && $importer->process( + file => $file, + process_code => sub { my ($context,$rows,$row_offset) = @_; my $rownum = $row_offset; my @lnp_rows = (); @@ -289,22 +331,26 @@ sub import_lnp_define { } return 1; - }, sub { + }, + init_process_context_code => sub { my ($context)= @_; $context->{db} = &get_import_db(); # keep ref count low.. - }, sub { + }, + uninit_process_context_code => sub { my ($context)= @_; undef $context->{db}; - destroy_dbs(); - }, $import_multithreading); + destroy_all_dbs(); + }, + multithreading => $import_multithreading + ); } sub _insert_lnp_rows { my ($context,$lnp_rows) = @_; $context->{db}->db_do_begin( - NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::Lnp::getinsertstatement($ignore_lnp_unique), - #NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::Lnp::gettablename(), + NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Lnp::getinsertstatement($ignore_lnp_unique), + #NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Lnp::gettablename(), #lock ); $context->{db}->db_do_rowblock($lnp_rows); diff --git a/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/ProjectConnectorPool.pm b/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/ProjectConnectorPool.pm index bb92188b..012ca443 100644 --- a/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/ProjectConnectorPool.pm +++ b/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/ProjectConnectorPool.pm @@ -26,7 +26,7 @@ use NGCP::BulkProcessor::SqlConnectors::SQLiteDB qw( #use NGCP::BulkProcessor::SqlConnectors::SQLServerDB; #use NGCP::BulkProcessor::RestConnectors::NGCPRestApi; -use NGCP::BulkProcessor::SqlRecord qw(cleartableinfo); +use NGCP::BulkProcessor::SqlProcessor qw(cleartableinfo); require Exporter; our @ISA = qw(Exporter); @@ -35,6 +35,7 @@ our @EXPORT_OK = qw( import_db_tableidentifier destroy_dbs + destroy_all_dbs ); # thread connector pools: @@ -80,4 +81,9 @@ sub destroy_dbs { } +sub destroy_all_dbs() { + destroy_dbs(); + NGCP::BulkProcessor::ConnectorPool::destroy_dbs(); +} + 1; diff --git a/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Settings.pm b/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Settings.pm index 9cc1c52d..446db56b 100644 --- a/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Settings.pm +++ b/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Settings.pm @@ -67,6 +67,7 @@ our @EXPORT_OK = qw( $lnp_define_import_numofthreads $ignore_lnp_unique + $stats_record_list_limit ); our $defaultconfig = 'config.cfg'; @@ -98,6 +99,9 @@ our $lnp_define_filename = undef; our $lnp_define_import_numofthreads = $cpucount; our $ignore_lnp_unique = 1; + +our $stats_record_list_limit = 1000; + sub update_settings { my ($data,$configfile) = @_; diff --git a/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/process.pl b/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/process.pl index 137d3175..e83f6b61 100644 --- a/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/process.pl +++ b/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/process.pl @@ -23,6 +23,7 @@ use NGCP::BulkProcessor::Projects::Migration::IPGallery::Settings qw( $features_define_filename $subscriber_define_filename $lnp_define_filename + $stats_record_list_limit ); use NGCP::BulkProcessor::Logging qw( init_log @@ -58,8 +59,7 @@ use NGCP::BulkProcessor::Mail qw( use NGCP::BulkProcessor::SqlConnectors::CSVDB qw(cleanupcvsdirs); use NGCP::BulkProcessor::SqlConnectors::SQLiteDB qw(cleanupdbfiles); -use NGCP::BulkProcessor::ConnectorPool qw(); -use NGCP::BulkProcessor::Projects::Migration::IPGallery::ProjectConnectorPool qw(); +use NGCP::BulkProcessor::Projects::Migration::IPGallery::ProjectConnectorPool qw(destroy_all_dbs); use NGCP::BulkProcessor::Projects::Migration::IPGallery::Import qw( import_features_define @@ -67,10 +67,10 @@ use NGCP::BulkProcessor::Projects::Migration::IPGallery::Import qw( import_lnp_define ); -use NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::FeatureOption qw(); -use NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::FeatureOptionSetItem qw(); -use NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::Subscriber qw(); -use NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::Lnp qw(); +use NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOption qw(); +use NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOptionSetItem qw(); +use NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber qw(); +use NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Lnp qw(); scripterror(getscriptpath() . ' already running',getlogger(getscriptpath())) unless flock DATA, LOCK_EX | LOCK_NB; # not tested on windows yet @@ -85,6 +85,8 @@ my $import_features_define_task_opt = 'import_features'; push(@TASK_OPTS,$import_features_define_task_opt); my $import_subscriber_define_task_opt = 'import_subscriber'; push(@TASK_OPTS,$import_subscriber_define_task_opt); +my $import_truncate_subscriber_task_opt = 'truncate_subscriber'; +push(@TASK_OPTS,$import_truncate_subscriber_task_opt); my $import_lnp_define_task_opt = 'import_lnp'; push(@TASK_OPTS,$import_lnp_define_task_opt); @@ -137,6 +139,8 @@ sub main() { $result = import_features_define_task(\@messages) if taskinfo($import_features_define_task_opt,$result); } elsif (lc($import_subscriber_define_task_opt) eq lc($task)) { $result = import_subscriber_define_task(\@messages) if taskinfo($import_subscriber_define_task_opt,$result); + } elsif (lc($import_truncate_subscriber_task_opt) eq lc($task)) { + $result = import_truncate_subscriber_task(\@messages) if taskinfo($import_truncate_subscriber_task_opt,$result); } elsif (lc($import_lnp_define_task_opt) eq lc($task)) { $result = import_lnp_define_task(\@messages) if taskinfo($import_lnp_define_task_opt,$result); } elsif (lc('blah') eq lc($task)) { @@ -203,16 +207,19 @@ sub import_features_define_task { $result = import_features_define($features_define_filename); }; my $err = $@; - my $stats = ' feature option: ' . - NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::FeatureOption::countby_subscribernumber() . ' rows'; - $stats .= "\n feature set option items: " . - NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::FeatureOptionSetItem::countby_subscribernumber_option() . ' rows'; + my $stats = ''; + eval { + $stats .= "\n total feature option records: " . + NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOption::countby_subscribernumber() . ' rows'; + $stats .= "\n total feature set option item records: " . + NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOptionSetItem::countby_subscribernumber_option() . ' rows'; + }; if ($err or !$result) { - push(@$messages,"importing subscriber features incomplete\n$stats"); + push(@$messages,"importing subscriber features incomplete$stats"); } else { - push(@$messages,"importing subscriber features completed\n$stats"); + push(@$messages,"importing subscriber features completed$stats"); } - destroy_dbs(); #every task should leave with closed connections. + destroy_all_dbs(); #every task should leave with closed connections. return $result; } @@ -225,50 +232,106 @@ sub import_subscriber_define_task { $result = import_subscriber_define($subscriber_define_filename); }; my $err = $@; - my $stats = ' subscriber: ' . - NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::Subscriber::countby_subscribernumber() . ' rows'; + my $stats = ''; + eval { + $stats .= "\n total subscriber records: " . + NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber::countby_subscribernumber() . ' rows'; + my $added_count = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber::countby_delta( + $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber::added_delta + ); + $stats .= "\n new: $added_count rows"; + if ($added_count <= $stats_record_list_limit) { + foreach my $record (@{NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber::findby_delta( + $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber::added_delta + )}) { + $stats .= "\n " . $record->{dial_number}; + } + } + my $existing_count = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber::countby_delta( + $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber::updated_delta + ); + $stats .= "\n existing: $existing_count rows"; + if ($existing_count <= $stats_record_list_limit) { + foreach my $record (@{NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber::findby_delta( + $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber::updated_delta + )}) { + $stats .= "\n " . $record->{dial_number}; + } + } + my $deleted_count = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber::countby_delta( + $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber::deleted_delta + ); + $stats .= "\n removed: $deleted_count rows"; + if ($deleted_count <= $stats_record_list_limit) { + foreach my $record (@{NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber::findby_delta( + $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber::deleted_delta + )}) { + $stats .= "\n " . $record->{dial_number}; + } + } + }; if ($err or !$result) { - push(@$messages,"importing subscribers incomplete\n$stats"); + push(@$messages,"importing subscribers incomplete$stats"); } else { - push(@$messages,"importing subscribers completed\n$stats"); + push(@$messages,"importing subscribers completed$stats"); } - destroy_dbs(); #every task should leave with closed connections. + destroy_all_dbs(); #every task should leave with closed connections. return $result; } -sub import_lnp_define_task { +sub import_truncate_subscriber_task { my ($messages) = @_; my $result = 0; eval { - $result = import_lnp_define($lnp_define_filename); + $result = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber::create_table(1); }; my $err = $@; - my $stats = ' lnp numbers: ' . - NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::Lnp::countby_lrncode_portednumber() . ' rows'; - $stats .= "\n lrn codes: " . - NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::Lnp::count_lrncodes(); + my $stats = ''; + eval { + $stats .= "\n total subscriber records: " . + NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber::countby_subscribernumber() . ' rows'; + }; if ($err or !$result) { - push(@$messages,"importing lnp numbers incomplete\n$stats"); + push(@$messages,"truncating imported subscribers incomplete$stats"); } else { - push(@$messages,"importing lnp numbers\n$stats"); + push(@$messages,"truncating imported subscribers completed$stats"); } - destroy_dbs(); #every task should leave with closed connections. + destroy_all_dbs(); #every task should leave with closed connections. return $result; } -sub destroy_dbs() { - NGCP::BulkProcessor::Projects::Migration::IPGallery::ProjectConnectorPool::destroy_dbs(); - NGCP::BulkProcessor::ConnectorPool::destroy_dbs(); +sub import_lnp_define_task { + + my ($messages) = @_; + my $result = 0; + eval { + $result = import_lnp_define($lnp_define_filename); + }; + my $err = $@; + my $stats = ''; + eval { + $stats .= "\n total lnp number records: " . + NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Lnp::countby_lrncode_portednumber() . ' rows'; + $stats .= "\n total lrn codes: " . + NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Lnp::count_lrncodes(); + }; + if ($err or !$result) { + push(@$messages,"importing lnp numbers incomplete$stats"); + } else { + push(@$messages,"importing lnp numbers$stats"); + } + destroy_all_dbs(); #every task should leave with closed connections. + return $result; + } #END { # # this should not be required explicitly, but prevents Log4Perl's # # "rootlogger not initialized error upon exit.. -# NGCP::BulkProcessor::Projects::Migration::IPGallery::ProjectConnectorPool::destroy_dbs(); -# NGCP::BulkProcessor::ConnectorPool::destroy_dbs(); +# destroy_all_dbs #} __DATA__ diff --git a/lib/NGCP/BulkProcessor/SqlProcessor.pm b/lib/NGCP/BulkProcessor/SqlProcessor.pm new file mode 100644 index 00000000..0988c098 --- /dev/null +++ b/lib/NGCP/BulkProcessor/SqlProcessor.pm @@ -0,0 +1,1566 @@ +package NGCP::BulkProcessor::SqlProcessor; +use strict; + +## no critic + +use threads qw(yield); +use threads::shared; +use Thread::Queue; +#use Thread::Semaphore; + +use Time::HiRes qw(sleep); + +use NGCP::BulkProcessor::Globals qw( + $enablemultithreading + $cpucount + $cells_transfer_memory_limit + $transfer_defer_indexes +); + +use NGCP::BulkProcessor::Logging qw( + getlogger + fieldnamesaquired + primarykeycolsaquired + tableinfoscleared + + tablefixed + + tabletransferstarted + tableprocessingstarted + + rowtransferstarted + rowtransferred + rowskipped + rowinserted + rowupdated + rowsdeleted + totalrowsdeleted + rowinsertskipped + rowupdateskipped + tabletransferdone + tableprocessingdone + rowtransferdone + + fetching_rows + writing_rows + processing_rows + + tablethreadingdebug +); + +use NGCP::BulkProcessor::LogError qw( + fieldnamesdiffer + transferzerorowcount + processzerorowcount + deleterowserror + tabletransferfailed + tableprocessingfailed +); + +use NGCP::BulkProcessor::Table qw(); +use NGCP::BulkProcessor::Array qw(setcontains contains); +use NGCP::BulkProcessor::Utils qw(round threadid); + +require Exporter; +our @ISA = qw(Exporter); +our @EXPORT_OK = qw( + init_record + copy_row + transfer_table + process_table + cleartableinfo + checktableinfo + registertableinfo + create_targettable + insert_record + update_record + delete_records + + insert_stmt +); +#transfer_record +#transfer_records + +my $table_expected_fieldnames = {}; +my $table_fieldnames_cached = {}; +my $table_primarykeys = {}; +my $table_target_indexes = {}; + +#my $logger = getlogger(__PACKAGE__); + +my $tabletransfer_threadqueuelength = 5; #100; #30; #5; # ... >= 1 +my $minblocksize = 100; +my $maxblocksize = 100000; +my $minnumberofchunks = 10; + +my $tableprocessing_threadqueuelength = 10; +my $tableprocessing_threads = $cpucount; #3; + +my $reader_connection_name = 'reader'; +#my $writer_connection_name = 'writer'; + +my $thread_sleep_secs = 0.1; + +my $RUNNING = 1; +my $COMPLETED = 2; +my $ERROR = 4; + +sub init_record { + + my ($record,$get_db,$tablename,$expected_fieldnames,$target_indexes) = @_; + my $db = (ref $get_db eq 'CODE') ? &$get_db() : $get_db; + + my $connectidentifier = $db->connectidentifier(); + my $tid = threadid(); + + checktableinfo($db,$tablename,$expected_fieldnames,$target_indexes); + + if (defined $table_expected_fieldnames->{$tid}->{$connectidentifier}->{$tablename}) { # and ref $table_fieldnames_cached->{$connectidentifier}->{$tablename} eq 'ARRAY') { + # if there are fieldnames defined, we make a member variable for each and set it to undef + foreach my $fieldname (@{$table_expected_fieldnames->{$tid}->{$connectidentifier}->{$tablename}}) { + $record->{$fieldname} = undef; + } + } + + return $record; + +} + +sub copy_row { + my ($record,$row,$expected_fieldnames) = @_; + if (defined $record and defined $row) { + my $i; + if (ref $row eq 'ARRAY') { + $i = 0; + } elsif (ref $row eq 'HASH') { + $i = -1; + } elsif (ref $row eq ref $record) { + $i = -2; + } else { + $i = -3; + } + foreach my $fieldname (@$expected_fieldnames) { + if ($i >= 0) { + $record->{$fieldname} = $row->[$i]; + $i++; + } elsif ($i == -1 or $i == -2) { + if (exists $row->{$fieldname}) { + $record->{$fieldname} = $row->{$fieldname}; + } elsif (exists $row->{uc($fieldname)}) { + $record->{$fieldname} = $row->{uc($fieldname)}; + } else { + $record->{$fieldname} = undef; + } + } else { + last; + } + } + } + return $record; +} + +sub cleartableinfo { + + my $get_db = shift; + + my $db = (ref $get_db eq 'CODE') ? &$get_db() : $get_db; + my $tid = threadid(); + + my $connectidentifier = $db->connectidentifier(); + + my $found = 0; + + if (exists $table_expected_fieldnames->{$tid}) { + if (exists $table_expected_fieldnames->{$tid}->{$connectidentifier}) { + delete $table_expected_fieldnames->{$tid}->{$connectidentifier}; + $found = 1; + } + } + if (exists $table_fieldnames_cached->{$tid}){ + if (exists $table_fieldnames_cached->{$tid}->{$connectidentifier}) { + delete $table_fieldnames_cached->{$tid}->{$connectidentifier}; + $found = 1; + } + } + if (exists $table_primarykeys->{$tid}) { + if (exists $table_primarykeys->{$tid}->{$connectidentifier}) { + delete $table_primarykeys->{$tid}->{$connectidentifier}; + $found = 1; + } + } + if (exists $table_target_indexes->{$tid}) { + if (exists $table_target_indexes->{$tid}->{$connectidentifier}) { + delete $table_target_indexes->{$tid}->{$connectidentifier}; + $found = 1; + } + } + + if ((scalar keys %{$table_expected_fieldnames->{$tid}}) == 0) { + delete $table_expected_fieldnames->{$tid}; + $found = 1; + } + if ((scalar keys %{$table_fieldnames_cached->{$tid}}) == 0) { + delete $table_fieldnames_cached->{$tid}; + $found = 1; + } + if ((scalar keys %{$table_primarykeys->{$tid}}) == 0) { + delete $table_primarykeys->{$tid}; + $found = 1; + } + if ((scalar keys %{$table_target_indexes->{$tid}}) == 0) { + delete $table_target_indexes->{$tid}; + $found = 1; + } + + if ($found) { + tableinfoscleared($db,getlogger(__PACKAGE__)); + } + +} + +sub registertableinfo { # to prepare creation of non-existent tables.. + + my ($get_db,$tablename,$fieldnames,$indexes,$keycols) = @_; + + my $db = (ref $get_db eq 'CODE') ? &$get_db() : $get_db; + + my $connectidentifier = $db->connectidentifier(); + my $tid = threadid(); + + if (not exists $table_expected_fieldnames->{$tid}) { + $table_expected_fieldnames->{$tid} = {}; + } + if (not exists $table_expected_fieldnames->{$tid}->{$connectidentifier}) { + # create an empty category for the connection if none exists yet: + $table_expected_fieldnames->{$tid}->{$connectidentifier} = {}; + } + # we prefer to always update the expected fieldnames (that come from a derived class) + $table_expected_fieldnames->{$tid}->{$connectidentifier}->{$tablename} = $fieldnames // []; + + if (not exists $table_fieldnames_cached->{$tid}) { + $table_fieldnames_cached->{$tid} = {}; + } + if (not exists $table_fieldnames_cached->{$tid}->{$connectidentifier}) { + # create an empty fieldname cache for the connection if none exists yet: + $table_fieldnames_cached->{$tid}->{$connectidentifier} = {}; + } + + if (not exists $table_primarykeys->{$tid}) { + $table_primarykeys->{$tid} = {}; + } + if (not exists $table_primarykeys->{$tid}->{$connectidentifier}) { + # create an empty primary key column name cache for the connection if none exists yet: + $table_primarykeys->{$tid}->{$connectidentifier} = {}; + } + $table_primarykeys->{$tid}->{$connectidentifier}->{$tablename} = $keycols // []; + + if (not exists $table_target_indexes->{$tid}) { + $table_target_indexes->{$tid} = {}; + } + if (not exists $table_target_indexes->{$tid}->{$connectidentifier}) { + # create an empty index set list for target tables for the connection if none exists yet: + $table_target_indexes->{$tid}->{$connectidentifier} = {}; + } + # we prefer to always update the target table indexes (that come from a derived class) + $table_target_indexes->{$tid}->{$connectidentifier}->{$tablename} = $indexes // {}; + +} + +sub checktableinfo { + + my ($get_db,$tablename,$expected_fieldnames,$target_indexes) = @_; + + my $result = 1; + + my $db = (ref $get_db eq 'CODE') ? &$get_db() : $get_db; + + my $connectidentifier = $db->connectidentifier(); + my $tid = threadid(); + + if (not exists $table_expected_fieldnames->{$tid}) { + #$table_expected_fieldnames->{$tid} = shared_clone({}); + $table_expected_fieldnames->{$tid} = {}; + } + if (not exists $table_expected_fieldnames->{$tid}->{$connectidentifier}) { + # create an empty category for the connection if none exists yet: + #$table_expected_fieldnames->{$tid}->{$connectidentifier} = shared_clone({}); + $table_expected_fieldnames->{$tid}->{$connectidentifier} = {}; + } + # we prefer to always update the expected fieldnames (that come from a derived class) + #$table_expected_fieldnames->{$tid}->{$connectidentifier}->{$tablename} = shared_clone($expected_fieldnames); + $table_expected_fieldnames->{$tid}->{$connectidentifier}->{$tablename} = $expected_fieldnames // []; + + if (not exists $table_fieldnames_cached->{$tid}) { + #$table_fieldnames_cached->{$tid} = shared_clone({}); + $table_fieldnames_cached->{$tid} = {}; + } + if (not exists $table_fieldnames_cached->{$tid}->{$connectidentifier}) { + # create an empty fieldname cache for the connection if none exists yet: + #$table_fieldnames_cached->{$tid}->{$connectidentifier} = shared_clone({}); + $table_fieldnames_cached->{$tid}->{$connectidentifier} = {}; + } + + if (not exists $table_fieldnames_cached->{$tid}->{$connectidentifier}->{$tablename}) { + # query the database for fieldnames of the table if we don't have a cache entry yet: + #$table_fieldnames_cached->{$tid}->{$connectidentifier}->{$tablename} = shared_clone($db->getfieldnames($tablename)); + $table_fieldnames_cached->{$tid}->{$connectidentifier}->{$tablename} = $db->getfieldnames($tablename); + #my $fieldnames = $db->getfieldnames($tablename); + if (!defined $table_expected_fieldnames->{$tid}->{$connectidentifier}->{$tablename} + or (scalar @{$table_expected_fieldnames->{$tid}->{$connectidentifier}->{$tablename}}) == 0 + or setcontains($table_expected_fieldnames->{$tid}->{$connectidentifier}->{$tablename},$table_fieldnames_cached->{$tid}->{$connectidentifier}->{$tablename},1)) { + #fieldnames are case insensitive in general + # if not expected fieldnames are given or queried fieldnames match, we log this: + #$table_fieldnames_cached->{$connectidentifier}->{$tablename} = $table_expected_fieldnames->{$connectidentifier}->{$tablename}; + fieldnamesaquired($db,$tablename,getlogger(__PACKAGE__)); + } else { + # otherwise we log a failure (exit? - see Logging Module) + #$table_fieldnames_cached->{$connectidentifier}->{$tablename} = {}; #$fieldnames; + fieldnamesdiffer($db,$tablename,$table_expected_fieldnames->{$tid}->{$connectidentifier}->{$tablename},$table_fieldnames_cached->{$tid}->{$connectidentifier}->{$tablename},getlogger(__PACKAGE__)); + $result = 0; + } + } + + if (not exists $table_primarykeys->{$tid}) { + #$table_primarykeys->{$tid} = shared_clone({}); + $table_primarykeys->{$tid} = {}; + } + if (not exists $table_primarykeys->{$tid}->{$connectidentifier}) { + # create an empty primary key column name cache for the connection if none exists yet: + #$table_primarykeys->{$tid}->{$connectidentifier} = shared_clone({}); + $table_primarykeys->{$tid}->{$connectidentifier} = {}; + } + if (not exists $table_primarykeys->{$tid}->{$connectidentifier}->{$tablename}) { + # query the database for primary keys of the table if we don't have them cached yet: + #$table_primarykeys->{$tid}->{$connectidentifier}->{$tablename} = shared_clone($db->getprimarykeycols($tablename)); + $table_primarykeys->{$tid}->{$connectidentifier}->{$tablename} = $db->getprimarykeycols($tablename); + primarykeycolsaquired($db,$tablename,$table_primarykeys->{$tid}->{$connectidentifier}->{$tablename},getlogger(__PACKAGE__)); + } + + if (not exists $table_target_indexes->{$tid}) { + #$table_target_indexes->{$tid} = shared_clone({}); + $table_target_indexes->{$tid} = {}; + } + if (not exists $table_target_indexes->{$tid}->{$connectidentifier}) { + # create an empty index set list for target tables for the connection if none exists yet: + #$table_target_indexes->{$tid}->{$connectidentifier} = shared_clone({}); + $table_target_indexes->{$tid}->{$connectidentifier} = {}; + } + # we prefer to always update the target table indexes (that come from a derived class) + #$table_target_indexes->{$tid}->{$connectidentifier}->{$tablename} = shared_clone($target_indexes); + $table_target_indexes->{$tid}->{$connectidentifier}->{$tablename} = $target_indexes // {}; + + return $result; + +} + +sub create_targettable { + + my ($get_db,$tablename,$get_target_db,$targettablename,$truncate,$defer_indexes,$texttable_engine) = @_; + + my $db = (ref $get_db eq 'CODE') ? &$get_db() : $get_db; + my $target_db = (ref $get_target_db eq 'CODE') ? &$get_target_db() : $get_target_db; + + #my $targettablename = _gettargettablename($db,$tablename,$target_db); + my $connectidentifier = $db->connectidentifier(); + my $tid = threadid(); + + if ($truncate and $defer_indexes) { + $target_db->drop_table($targettablename); + } + + my $result = $target_db->create_texttable($targettablename, + $table_expected_fieldnames->{$tid}->{$connectidentifier}->{$tablename}, + $table_primarykeys->{$tid}->{$connectidentifier}->{$tablename}, + $table_target_indexes->{$tid}->{$connectidentifier}->{$tablename}, + # 'ifnotexists' is always true + $truncate, + $defer_indexes, + $texttable_engine); + + checktableinfo($target_db,$targettablename,$table_expected_fieldnames->{$tid}->{$connectidentifier}->{$tablename},$defer_indexes ? undef : $table_target_indexes->{$tid}->{$connectidentifier}->{$tablename}); + return $result; + +} + +sub delete_records { + + my ($get_db,$tablename,$keyfields,$equal,$vals_table) = @_; + + my $db = (ref $get_db eq 'CODE') ? &$get_db() : $get_db; + + my $connectidentifier = $db->connectidentifier(); + my $tid = threadid(); + + my $expected_fieldnames = $table_expected_fieldnames->{$tid}->{$connectidentifier}->{$tablename}; + my $primarykeys = $table_primarykeys->{$tid}->{$connectidentifier}->{$tablename}; + + if (defined $expected_fieldnames and + (defined $keyfields and + ref $keyfields eq 'ARRAY') and + (defined $vals_table and + ref $vals_table eq 'NGCP::BulkProcessor::Table')) { + + my @fields = @$keyfields; + my $field_cnt = scalar @fields; + + my $total_rowcount = 0; + + my $initial_rowcount = $db->db_get_value('SELECT COUNT(*) FROM ' . $db->tableidentifier($tablename)); + + if ($field_cnt > 0) { + my $where_clause; + if ($equal) { + $where_clause = ' WHERE ' . join(' = ? AND ',map { local $_ = $_; $_ = $db->columnidentifier($_); $_; } @fields) . ' = ?'; + + my $count_stmt = 'SELECT COUNT(*) FROM ' . $db->tableidentifier($tablename) . $where_clause; + my $delete_stmt = 'DELETE FROM ' . $db->tableidentifier($tablename) . $where_clause; + + for (my $i = 0; $i < $vals_table->rowcount(); $i++) { + my @vals = $vals_table->getrow($i); + my $new_initial_rowcount = $db->db_get_value('SELECT COUNT(*) FROM ' . $db->tableidentifier($tablename)); + my $rowcount = $db->db_get_value($count_stmt,@vals); + $db->db_do($delete_stmt,@vals); + rowsdeleted($db,$tablename,$rowcount,$new_initial_rowcount,getlogger(__PACKAGE__)); + $total_rowcount += $rowcount; + } + + } elsif ($field_cnt == 1) { + my @ne_vals = $vals_table->getcol(0); + $where_clause = ' WHERE ' . $db->columnidentifier($fields[0]) . ' NOT IN (' . substr(',?' x scalar @ne_vals,1) . ')'; + my $count_stmt = 'SELECT COUNT(*) FROM ' . $db->tableidentifier($tablename) . $where_clause; + my $delete_stmt = 'DELETE FROM ' . $db->tableidentifier($tablename) . $where_clause; + my $rowcount = $db->db_get_value($count_stmt,@ne_vals); + $db->db_do($delete_stmt,@ne_vals); + rowsdeleted($db,$tablename,$rowcount,$initial_rowcount,getlogger(__PACKAGE__)); + $total_rowcount += $rowcount; + } else { + + deleterowserror($db,$tablename,'deletings rows by complementary identifier values works with a single identifier column only',getlogger(__PACKAGE__)); + return; + + } + } else { + my $delete_stmt = 'DELETE FROM ' . $db->tableidentifier($tablename); + my $count_stmt = 'SELECT COUNT(*) FROM ' . $db->tableidentifier($tablename); + my $rowcount = $db->db_get_value($count_stmt); + $db->db_do($delete_stmt); + rowsdeleted($db,$tablename,$rowcount,$initial_rowcount,getlogger(__PACKAGE__)); + $total_rowcount += $rowcount; + } + + $db->vacuum($tablename); + + #if ($total_rowcount > 0) { + totalrowsdeleted($db,$tablename,$total_rowcount,$initial_rowcount,getlogger(__PACKAGE__)); + #} + + return $total_rowcount; + + } + +} + +sub insert_record { + + my ($get_db,$tablename,$row,$insert_ignore,$unique_count_fields) = @_; + + my $db = (ref $get_db eq 'CODE') ? &$get_db() : $get_db; + + #my $targettablename = _gettargettablename($db,$tablename,$target_db); #$target_db->getsafetablename($db->tableidentifier($tablename)); + my $connectidentifier = $db->connectidentifier(); + my $tid = threadid(); + + my $expected_fieldnames = $table_expected_fieldnames->{$tid}->{$connectidentifier}->{$tablename}; + #my $primarykeys = $table_primarykeys->{$tid}->{$connectidentifier}->{$tablename}; + + if (defined $expected_fieldnames and defined $row) { + + my @fields = (); + my @vals = (); + + foreach my $fieldname (@$expected_fieldnames) { + if (exists $row->{$fieldname}) { + push @fields,$fieldname; + push @vals,$row->{$fieldname}; + } + } + + my @unique_fields = (); + my @unique_vals = (); + if ('ARRAY' eq ref $unique_count_fields) { + foreach my $fieldname (@$unique_count_fields) { + push(@unique_fields,$fieldname); + if (exists $row->{$fieldname}) { + push @unique_vals,$row->{$fieldname}; + } else { + push @unique_vals,undef; + } + } + } + + my $stmt = 'INSERT ' . ($insert_ignore ? $db->insert_ignore_phrase() . ' ' : '') . 'INTO ' . $db->tableidentifier($tablename) . ' (' . join(', ',map { local $_ = $_; $_ = $db->columnidentifier($_); $_; } @fields) . ') VALUES (' . substr(',?' x scalar @fields,1) . ')'; + if ((scalar @unique_fields) > 0) { + if ($db->db_get_value('SELECT COUNT(*) FROM ' . $db->tableidentifier($tablename) . ' WHERE ' . join(' = ? AND ',map { local $_ = $_; $_ = $db->columnidentifier($_); $_; } @unique_fields) . ' = ?',@unique_vals) == 0) { + if ($db->db_do($stmt,@vals)) { + rowinserted($db,$tablename,getlogger(__PACKAGE__)); + return 1; + } else { + rowinsertskipped($db,$tablename,getlogger(__PACKAGE__)); + return 0; + } + } else { + rowinsertskipped($db,$tablename,getlogger(__PACKAGE__)); + return 0; + } + } else { + if ($db->db_do($stmt,@vals)) { + rowinserted($db,$tablename,getlogger(__PACKAGE__)); + return 1; + } else { + rowinsertskipped($db,$tablename,getlogger(__PACKAGE__)); + return 0; + } + } + + } + +} + +sub update_record { + + my ($get_db,$tablename,$row) = @_; + + my $db = (ref $get_db eq 'CODE') ? &$get_db() : $get_db; + + #my $targettablename = _gettargettablename($db,$tablename,$target_db); #$target_db->getsafetablename($db->tableidentifier($tablename)); + my $connectidentifier = $db->connectidentifier(); + my $tid = threadid(); + + my $expected_fieldnames = $table_expected_fieldnames->{$tid}->{$connectidentifier}->{$tablename}; + my $primarykeys = $table_primarykeys->{$tid}->{$connectidentifier}->{$tablename}; + + if (defined $expected_fieldnames and defined $row) { + + my @fields = (); + my @vals = (); + + foreach my $fieldname (@$expected_fieldnames) { + if (exists $row->{$fieldname}) { + push @fields,$fieldname; + push @vals,$row->{$fieldname}; + } + } + + my @pk_fields = (); + my @pk_vals = (); + + if (defined $primarykeys) { + if (scalar @$primarykeys > 0) { + foreach my $fieldname (@$primarykeys) { + if (exists $row->{$fieldname}) { + push @pk_fields,$fieldname; + push @pk_vals,$row->{$fieldname}; + #} else { + # 'insert error: pk field not foun din row'; + # push @pk_vals,undef; + } + } + } else { + @pk_fields = @fields; + @pk_vals = @vals; + } + } else { + @pk_fields = @fields; + @pk_vals = @vals; + } + + my $selectpk_fieldnames = join(' = ? AND ',map { local $_ = $_; $_ = $db->columnidentifier($_); $_; } @pk_fields); + + my $count = $db->db_get_value('SELECT COUNT(*) FROM ' . $db->tableidentifier($tablename) . ' WHERE ' . $selectpk_fieldnames . ' = ?',@pk_vals); + if ($count == 1) { + $db->db_do('UPDATE ' . $db->tableidentifier($tablename) . ' SET ' . join(' = ?, ',map { local $_ = $_; $_ = $db->columnidentifier($_); $_; } @fields) . ' = ? WHERE ' . $selectpk_fieldnames . ' = ?',@vals,@pk_vals); + rowupdated($db,$tablename,getlogger(__PACKAGE__)); + return 1; + } else { + rowupdateskipped($db,$tablename,$count,getlogger(__PACKAGE__)); + return 0; + } + + } + +} + +#sub transfer_record { +# +# #my $self = shift +# my ($get_db,$tablename,$get_target_db,$targettablename,$allowdupes,$row) = @_; +# +# my $db = (ref $get_db eq 'CODE') ? &$get_db() : $get_db; +# my $target_db = (ref $get_target_db eq 'CODE') ? &$get_target_db() : $get_target_db; +# +# #my $targettablename = _gettargettablename($db,$tablename,$target_db); #$target_db->getsafetablename($db->tableidentifier($tablename)); +# my $connectidentifier = $db->connectidentifier(); +# my $tid = threadid(); +# +# my $expected_fieldnames = $table_expected_fieldnames->{$tid}->{$connectidentifier}->{$tablename}; +# my $primarykeys = $table_primarykeys->{$tid}->{$connectidentifier}->{$tablename}; +# +# if (defined $expected_fieldnames and defined $row) { +# +# my @vals = (); +# +# foreach my $fieldname (@$expected_fieldnames) { +# push @vals,$row->{$fieldname}; +# } +# +# my @pk_fieldnames; +# my @pk_vals = (); +# +# if (not $allowdupes) { +# if (defined $primarykeys) { +# if (scalar @$primarykeys > 0) { +# foreach my $fieldname (@$primarykeys) { +# push @pk_vals,$row->{$fieldname}; +# } +# } else { +# @pk_fieldnames = @fieldnames; +# @pk_vals = @vals; +# } +# } else { +# @pk_fieldnames = @fieldnames; +# @pk_vals = @vals; +# } +# } +# +# if ($allowdupes or $target_db->db_get_value('SELECT COUNT(*) FROM ' . $target_db->tableidentifier($targettablename) . ' WHERE ' . join(' = ? AND ',map { local $_ = $_; $_ = $target_db->columnidentifier($_); $_; } @pk_fieldnames) . ' = ?',@pk_vals) == 0) { +# $target_db->db_do('INSERT INTO ' . $target_db->tableidentifier($targettablename) . ' (' . join(', ',map { local $_ = $_; $_ = $target_db->columnidentifier($_); $_; } @fieldnames) . ') VALUES (' . substr(',?' x scalar @fieldnames,1) . ')',@vals); +# rowtransferred($db,$tablename,$target_db,$targettablename,1,1,getlogger(__PACKAGE__)); +# return 1; +# } else { +# rowskipped($db,$tablename,$target_db,$targettablename,1,1,getlogger(__PACKAGE__)); +# return 0; +# } +# +# } +# +#} + +#sub transfer_records { +# +# my ($get_db,$tablename,$get_target_db,$targettablename,$allowdupes,$rows) = @_; +# +# my $db = (ref $get_db eq 'CODE') ? &$get_db() : $get_db; +# my $target_db = (ref $get_target_db eq 'CODE') ? &$get_target_db() : $get_target_db; +# +# #my $targettablename = _gettargettablename($db,$tablename,$target_db); #$target_db->getsafetablename($db->tableidentifier($tablename)); +# my $connectidentifier = $db->connectidentifier(); +# my $tid = threadid(); +# +# my $expected_fieldnames = $table_expected_fieldnames->{$tid}->{$connectidentifier}->{$tablename}; +# my $primarykeys = $table_primarykeys->{$tid}->{$connectidentifier}->{$tablename}; +# +# if (defined $expected_fieldnames and defined $rows and ref $rows eq 'ARRAY') { # and defined $getrecords_code and ref $getrecords_code eq 'CODE') { +# #get_local_db(); +# +# my $numofrows = scalar @$rows; +# rowtransferstarted($db,$tablename,$target_db,$targettablename,$numofrows,getlogger(__PACKAGE__)); +# +# my @fieldnames = @$expected_fieldnames; +# +# my $setfieldnames = join(', ',map { local $_ = $_; $_ = $target_db->columnidentifier($_); $_; } @fieldnames); +# my $valueplaceholders = substr(',?' x scalar @fieldnames,1); +# +# my $rowstransferred = 0; +# +# if ($allowdupes) { +# +# my @rows_array = (); +# $target_db->db_do_begin('INSERT INTO ' . $target_db->tableidentifier($targettablename) . ' (' . $setfieldnames . ') VALUES (' . $valueplaceholders . ')'); +# foreach my $row (@$rows) { +# my @vals = (); +# foreach my $fieldname (@fieldnames) { +# push @vals,$row->{$fieldname}; +# } +# push @rows_array,\@vals; +# } +# $target_db->db_do_rowblock(\@rows_array); +# $target_db->db_finish(); +# $rowstransferred = scalar @rows_array; +# +# } else { +# +# my $i = 1; +# +# my @pk_fieldnames; +# +# if (defined $primarykeys) { +# @pk_fieldnames = @$primarykeys; +# if (scalar @pk_fieldnames == 0) { +# @pk_fieldnames = @fieldnames; +# } +# } else { +# @pk_fieldnames = @fieldnames; +# } +# +# my $selectpk_fieldnames = join(' = ? AND ',map { local $_ = $_; $_ = $target_db->columnidentifier($_); $_; } @pk_fieldnames) . ' = ?'; +# +# foreach my $row (@$rows) { +# +# my @vals = (); +# +# foreach my $fieldname (@fieldnames) { +# push @vals,$row->{$fieldname}; +# } +# +# my @pk_vals; +# +# foreach my $fieldname (@pk_fieldnames) { +# push @pk_vals,$row->{$fieldname}; +# } +# +# if ($target_db->db_get_value('SELECT COUNT(*) FROM ' . $target_db->tableidentifier($targettablename) . ' WHERE ' . $selectpk_fieldnames,@pk_vals) == 0) { +# $target_db->db_do('INSERT INTO ' . $db->target_tableidentifier($targettablename) . ' (' . $setfieldnames . ') VALUES (' . $valueplaceholders . ')',@vals); +# rowtransferred($db,$tablename,$target_db,$targettablename,$i,$numofrows,getlogger(__PACKAGE__)); +# $rowstransferred += 1; +# } else { +# rowskipped($db,$tablename,$target_db,$targettablename,$i,$numofrows,getlogger(__PACKAGE__)); +# } +# $i++; +# } +# } +# rowtransferdone($db,$tablename,$target_db,$targettablename,$numofrows,getlogger(__PACKAGE__)); +# return $rowstransferred; +# } +# +#} + +sub insert_stmt { + + my ($get_db,$tablename,$insert_ignore) = @_; + my $db = (ref $get_db eq 'CODE') ? &$get_db() : $get_db; + my $connectidentifier = $db->connectidentifier(); + my $tid = threadid(); + my $expected_fieldnames = $table_expected_fieldnames->{$tid}->{$connectidentifier}->{$tablename}; + return 'INSERT ' . ($insert_ignore ? $db->insert_ignore_phrase() . ' ' : '') . 'INTO ' . $db->tableidentifier($tablename) . ' (' . + join(', ',map { local $_ = $_; $_ = $db->columnidentifier($_); $_; } @$expected_fieldnames) . + ') VALUES (' . substr(',?' x scalar @$expected_fieldnames,1) . ')'; + +} + +sub transfer_table { + + my %params = @_; + my ($get_db, + $tablename, + $get_target_db, + $targettablename, + $truncate_targettable, + $create_indexes, + $texttable_engine, + $fixtable_statements, + $multithreading, + $selectcount, + $select, + $values) = @params{qw/ + get_db + tablename + get_target_db + targettablename + truncate_targettable + create_indexes + texttable_engine + fixtable_statements + multithreading + selectcount + select + values + /}; + + #my ($get_db,$tablename,$get_target_db,$targettablename,$truncate_targettable,$create_indexes,$texttable_engine,$fixtable_statements,$selectcount,$select,@values) = @_; + + if (ref $get_db eq 'CODE' and ref $get_target_db eq 'CODE') { + + my $db = &$get_db($reader_connection_name,1); + my $target_db = &$get_target_db(); #$writer_connection_name); + + my $countstatement; + if (defined $selectcount) { + $countstatement = $selectcount; + } else { + $countstatement = 'SELECT COUNT(*) FROM ' . $db->tableidentifier($tablename); + } + + my $rowcount = $db->db_get_value($countstatement,@$values); + + #my $targettablename = _gettargettablename($db,$tablename,$target_db); #$target_db->getsafetablename($db->tableidentifier($tablename)); + + if ($rowcount > 0) { + tabletransferstarted($db,$tablename,$target_db,$targettablename,$rowcount,getlogger(__PACKAGE__)); + } else { + transferzerorowcount($db,$tablename,$target_db,$targettablename,$rowcount,getlogger(__PACKAGE__)); + return; + } + + my $errorstate = $RUNNING; # 1; + + $create_indexes = ((defined $create_indexes) ? $create_indexes : $transfer_defer_indexes); + + if (create_targettable($db,$tablename,$target_db,$targettablename,$truncate_targettable,$create_indexes,$texttable_engine)) { + + my $connectidentifier = $db->connectidentifier(); + my $tid = threadid(); + my $expected_fieldnames = $table_expected_fieldnames->{$tid}->{$connectidentifier}->{$tablename}; + + my @fieldnames = @$expected_fieldnames; + + #my $setfieldnames = join(', ',map { local $_ = $_; $_ = $db->columnidentifier($_); $_; } @fieldnames); + my $valueplaceholders = substr(',?' x scalar @fieldnames,1); + + my $selectstatement; + if (length($select) > 0) { + $selectstatement = $select; + } else { + $selectstatement = 'SELECT ' . join(', ',map { local $_ = $_; $_ = $db->columnidentifier($_); $_; } @fieldnames) . ' FROM ' . $db->tableidentifier($tablename) + } + + my $insertstatement = 'INSERT INTO ' . $target_db->tableidentifier($targettablename) . ' (' . join(', ',map { local $_ = $_; $_ = $target_db->columnidentifier($_); $_; } @fieldnames) . ') VALUES (' . $valueplaceholders . ')'; + + my $blocksize; + + if ($enablemultithreading and $multithreading and $db->multithreading_supported() and $target_db->multithreading_supported() and $cpucount > 1) { # and $multithreaded) { # definitely no multithreading when CSVDB is involved + + $blocksize = _calc_blocksize($rowcount,scalar @fieldnames,1,$tabletransfer_threadqueuelength); + + my $reader; + my $writer; + + my %errorstates :shared = (); + #$errorstates{$tid} = $errorstate; + + #my $readererrorstate :shared = 1; + #my $writererrorstate :shared = 1; + + my $queue = Thread::Queue->new(); + + tablethreadingdebug('shutting down db connections ...',getlogger(__PACKAGE__)); + + $db->db_disconnect(); + #undef $db; + $target_db->db_disconnect(); + #undef $target_db; + my $default_connection = &$get_db(undef,0); + my $default_connection_reconnect = $default_connection->is_connected(); + $default_connection->db_disconnect(); + + tablethreadingdebug('starting reader thread',getlogger(__PACKAGE__)); + + $reader = threads->create(\&_reader, + { queue => $queue, + errorstates => \%errorstates, + #readererrorstate_ref => \$readererrorstate, + #writererrorstate_ref => \$writererrorstate, + threadqueuelength => $tabletransfer_threadqueuelength, + get_db => $get_db, + tablename => $tablename, + selectstatement => $selectstatement, + blocksize => $blocksize, + rowcount => $rowcount, + #logger => $logger, + values_ref => $values, + }); + + tablethreadingdebug('starting writer thread',getlogger(__PACKAGE__)); + + $writer = threads->create(\&_writer, + { queue => $queue, + errorstates => \%errorstates, + readertid => $reader->tid(), + #readererrorstate_ref => \$readererrorstate, + #writererrorstate_ref => \$writererrorstate, + get_target_db => $get_target_db, + targettablename => $targettablename, + insertstatement => $insertstatement, + blocksize => $blocksize, + rowcount => $rowcount, + #logger => $logger, + }); + + $reader->join(); + tablethreadingdebug('reader thread joined',getlogger(__PACKAGE__)); + $writer->join(); + tablethreadingdebug('writer thread joined',getlogger(__PACKAGE__)); + + #$errorstate = $readererrorstate | $writererrorstate; + $errorstate = _get_other_threads_state(\%errorstates,$tid); + + tablethreadingdebug('restoring db connections ...',getlogger(__PACKAGE__)); + + #$db = &$get_db($reader_connection_name,1); + $target_db = &$get_target_db(undef,1); + if ($default_connection_reconnect) { + $default_connection = &$get_db(undef,1); + } + + } else { + + $blocksize = _calc_blocksize($rowcount,scalar @fieldnames,0,undef); + + #$db->db_disconnect(); + #undef $db; + #$db = &$get_db($reader_connection_name); + #$target_db->db_disconnect(); + #undef $target_db; + #$target_db = &$get_target_db($writer_connection_name); + + eval { + $db->db_get_begin($selectstatement,@$values); #$tablename + + my $i = 0; + while (1) { + fetching_rows($db,$tablename,$i,$blocksize,$rowcount,getlogger(__PACKAGE__)); + my $rowblock = $db->db_get_rowblock($blocksize); + my $realblocksize = scalar @$rowblock; + if ($realblocksize > 0) { + writing_rows($target_db,$targettablename,$i,$realblocksize,$rowcount,getlogger(__PACKAGE__)); + $target_db->db_do_begin($insertstatement); #,$targettablename); + $target_db->db_do_rowblock($rowblock); + $target_db->db_finish(); + $i += $realblocksize; + + #foreach my $row (@$rowblock) { + # undef $row; + #} + #undef $rowblock; + + if ($realblocksize < $blocksize) { + last; + } + } else { + last; + } + } + $db->db_finish(); + + }; + + if ($@) { + $errorstate = $ERROR; + } else { + $errorstate = $COMPLETED; + } + + $db->db_disconnect(); + #undef $db; + #$target_db->db_disconnect(); + #undef $target_db; + + } + + #$db = &$get_db($controller_name,1); + #$target_db = &$get_target_db($controller_name,1); + + if ($errorstate == $COMPLETED and ref $fixtable_statements eq 'ARRAY' and (scalar @$fixtable_statements) > 0) { + eval { + foreach my $fixtable_statement (@$fixtable_statements) { + if (ref $fixtable_statement eq '') { + $target_db->db_do($fixtable_statement); + tablefixed($target_db,$targettablename,$fixtable_statement,getlogger(__PACKAGE__)); + } else { + $fixtable_statement = &$fixtable_statement($target_db->tableidentifier($targettablename)); + $target_db->db_do($fixtable_statement); + tablefixed($target_db,$targettablename,$fixtable_statement,getlogger(__PACKAGE__)); + } + + } + }; + if ($@) { + $errorstate = $ERROR; + #} else { + # $errorstate = $COMPLETED; + } + } + + if ($errorstate == $COMPLETED and $create_indexes) { + + eval { + $target_db->create_primarykey($targettablename, + $table_primarykeys->{$tid}->{$connectidentifier}->{$tablename}, + $table_expected_fieldnames->{$tid}->{$connectidentifier}->{$tablename}); + + $target_db->create_indexes($targettablename, + $table_target_indexes->{$tid}->{$connectidentifier}->{$tablename}, + $table_primarykeys->{$tid}->{$connectidentifier}->{$tablename}); + + + delete $table_primarykeys->{$tid}->{$target_db->connectidentifier()}->{$targettablename}; + checktableinfo($target_db,$targettablename,$table_expected_fieldnames->{$tid}->{$connectidentifier}->{$tablename},$table_target_indexes->{$tid}->{$connectidentifier}->{$tablename}); + + $target_db->vacuum($targettablename); + + }; + + if ($@) { + $errorstate = $ERROR; + #} else { + # $errorstate = $COMPLETED; + } + } + + } + + if ($errorstate == $COMPLETED) { + tabletransferdone($db,$tablename,$target_db,$targettablename,$rowcount,getlogger(__PACKAGE__)); + #$db->db_disconnect(); + #$target_db->db_disconnect(); + return 1; + } else { + tabletransferfailed($db,$tablename,$target_db,$targettablename,$rowcount,getlogger(__PACKAGE__)); + #$db->db_disconnect(); + #$target_db->db_disconnect(); + } + + } + + return 0; + +} + +sub process_table { + + my %params = @_; + my ($get_db, + $tablename, + $process_code, + $init_process_context_code, + $uninit_process_context_code, + $multithreading, + $selectcount, + $select, + $values) = @params{qw/ + get_db + tablename + process_code + init_process_context_code + uninit_process_context_code + multithreading + selectcount + select + values + /}; + + #my ($get_db,$tablename,$process_code,$init_process_context_code,$uninit_process_context_code,$multithreading,$selectcount,$select,@values) = @_; + + if (ref $get_db eq 'CODE') { + + my $db = &$get_db($reader_connection_name,1); + + my $countstatement; + if (defined $selectcount) { + $countstatement = $selectcount; + } else { + $countstatement = 'SELECT COUNT(*) FROM ' . $db->tableidentifier($tablename); + } + + my $rowcount = $db->db_get_value($countstatement,@$values); + + if ($rowcount > 0) { + tableprocessingstarted($db,$tablename,$rowcount,getlogger(__PACKAGE__)); + } else { + processzerorowcount($db,$tablename,$rowcount,getlogger(__PACKAGE__)); + return; + } + + my $errorstate = $RUNNING; + + my $connectidentifier = $db->connectidentifier(); + my $tid = threadid(); + my $expected_fieldnames = $table_expected_fieldnames->{$tid}->{$connectidentifier}->{$tablename}; + + my @fieldnames = @$expected_fieldnames; + + #my $setfieldnames = join(', ',@fieldnames); + #my $valueplaceholders = substr(',?' x scalar @fieldnames,1); + + my $selectstatement; + if (length($select) > 0) { + $selectstatement = $select; + } else { + $selectstatement = 'SELECT ' . join(', ',map { local $_ = $_; $_ = $db->columnidentifier($_); $_; } @fieldnames) . ' FROM ' . $db->tableidentifier($tablename); + } + + my $blocksize; + + if ($enablemultithreading and $multithreading and $db->multithreading_supported() and $cpucount > 1) { # and $multithreaded) { # definitely no multithreading when CSVDB is involved + + $blocksize = _calc_blocksize($rowcount,scalar @fieldnames,1,$tableprocessing_threadqueuelength); + + my $reader; + #my $processor; + my %processors = (); + + my %errorstates :shared = (); + #$errorstates{$tid} = $errorstate; + + #my $readererrorstate :shared = 1; + #my $processorerrorstate :shared = 1; + + my $queue = Thread::Queue->new(); + + tablethreadingdebug('shutting down db connections ...',getlogger(__PACKAGE__)); + + $db->db_disconnect(); + #undef $db; + my $default_connection = &$get_db(undef,0); + my $default_connection_reconnect = $default_connection->is_connected(); + $default_connection->db_disconnect(); + + tablethreadingdebug('starting reader thread',getlogger(__PACKAGE__)); + + $reader = threads->create(\&_reader, + { queue => $queue, + errorstates => \%errorstates, + #readererrorstate_ref => \$readererrorstate, + #writererrorstate_ref => \$processorerrorstate, + threadqueuelength => $tableprocessing_threadqueuelength, + get_db => $get_db, + tablename => $tablename, + selectstatement => $selectstatement, + blocksize => $blocksize, + rowcount => $rowcount, + #logger => $logger, + values_ref => $values, + }); + + for (my $i = 0; $i < $tableprocessing_threads; $i++) { + tablethreadingdebug('starting processor thread ' . ($i + 1) . ' of ' . $tableprocessing_threads,getlogger(__PACKAGE__)); + my $processor = threads->create(\&_process, + { queue => $queue, + errorstates => \%errorstates, + readertid => $reader->tid(), + #readererrorstate_ref => \$readererrorstate, + #processorerrorstate_ref => \$processorerrorstate, + process_code => $process_code, + init_process_context_code => $init_process_context_code, + uninit_process_context_code => $uninit_process_context_code, + blocksize => $blocksize, + rowcount => $rowcount, + #logger => $logger, + }); + if (!defined $processor) { + tablethreadingdebug('processor thread ' . ($i + 1) . ' of ' . $tableprocessing_threads . ' NOT started',getlogger(__PACKAGE__)); + } + $processors{$processor->tid()} = $processor; + #push (@processors,$processor); + } + + #$reader->join(); + #tablethreadingdebug('reader thread joined',getlogger(__PACKAGE__)); + #for (my $i = 0; $i < $tableprocessing_threads; $i++) { + # my $processor = $processors[$i]; + # if (defined $processor) { + # $processor->join(); + # tablethreadingdebug('processor thread ' . ($i + 1) . ' of ' . $tableprocessing_threads . ' joinded',getlogger(__PACKAGE__)); + # } else { + # tablethreadingdebug('processor thread ' . ($i + 1) . ' of ' . $tableprocessing_threads . ' NOT joinded',getlogger(__PACKAGE__)); + # } + #} + + $reader->join(); + tablethreadingdebug('reader thread joined',getlogger(__PACKAGE__)); + #print 'threads running: ' . (scalar threads->list(threads::running)); + #while ((scalar threads->list(threads::running)) > 1 or (scalar threads->list(threads::joinable)) > 0) { + while ((scalar keys %processors) > 0) { + #for (my $i = 0; $i < $tableprocessing_threads; $i++) { + foreach my $processor (values %processors) { + #my $processor = $processors[$i]; + if (defined $processor and $processor->is_joinable()) { + $processor->join(); + delete $processors{$processor->tid()}; + #tablethreadingdebug('processor thread ' . ($i + 1) . ' of ' . $tableprocessing_threads . ' joinded',getlogger(__PACKAGE__)); + tablethreadingdebug('processor thread tid ' . $processor->tid() . ' joined',getlogger(__PACKAGE__)); + } + #} else { + # tablethreadingdebug('processor thread ' . ($i + 1) . ' of ' . $tableprocessing_threads . ' NOT joinded',getlogger(__PACKAGE__)); + #} + } + sleep($thread_sleep_secs); + } + + #$errorstate = $readererrorstate | $processorerrorstate; + $errorstate = (_get_other_threads_state(\%errorstates,$tid) & ~$RUNNING); + + tablethreadingdebug('restoring db connections ...',getlogger(__PACKAGE__)); + + #$db = &$get_db($reader_connection_name,1); + if ($default_connection_reconnect) { + $default_connection = &$get_db(undef,1); + } + + } else { + + $blocksize = _calc_blocksize($rowcount,scalar @fieldnames,0,undef); + #$db->db_disconnect(); + #undef $db; + #$db = &$get_db($reader_connection_name); + my $context = { tid => $tid }; + my $rowblock_result = 1; + eval { + if ('CODE' eq ref $init_process_context_code) { + &$init_process_context_code($context); + } + + $db->db_get_begin($selectstatement,@$values); #$tablename + + my $i = 0; + while (1) { + fetching_rows($db,$tablename,$i,$blocksize,$rowcount,getlogger(__PACKAGE__)); + my $rowblock = $db->db_get_rowblock($blocksize); + my $realblocksize = scalar @$rowblock; + if ($realblocksize > 0) { + processing_rows($tid,$i,$realblocksize,$rowcount,getlogger(__PACKAGE__)); + + $rowblock_result = &$process_code($context,$rowblock,$i); + + #$target_db->db_do_begin($insertstatement); #,$targettablename); + #$target_db->db_do_rowblock($rowblock); + #$target_db->db_finish(); + $i += $realblocksize; + + if ($realblocksize < $blocksize || not $rowblock_result) { + last; + } + } else { + last; + } + } + $db->db_finish(); + + }; + + if ($@) { + $errorstate = $ERROR; + } else { + $errorstate = (not $rowblock_result) ? $ERROR : $COMPLETED; + } + + eval { + if ('CODE' eq ref $uninit_process_context_code) { + &$uninit_process_context_code($context); + } + }; + $db->db_disconnect(); + #undef $db; + + } + + #$db = &$get_db($controller_name,1); + + if ($errorstate == $COMPLETED) { + tableprocessingdone($db,$tablename,$rowcount,getlogger(__PACKAGE__)); + #$db->db_disconnect(); + return 1; + } else { + tableprocessingfailed($db,$tablename,$rowcount,getlogger(__PACKAGE__)); + #$db->db_disconnect(); + } + + } + + return 0; + +} + +sub _calc_blocksize { + + my ($rowcount,$columncount,$multithreaded,$threadqueuelength) = @_; + + if ($rowcount > $minblocksize) { + + my $exp = int ( log ($rowcount) / log(10.0) ); + my $blocksize = int ( 10 ** $exp ); + my $cellcount_in_memory = $columncount * $blocksize; + if ($multithreaded) { + $cellcount_in_memory *= $threadqueuelength; + } + + while ( $cellcount_in_memory > $cells_transfer_memory_limit or + $rowcount / $blocksize < $minnumberofchunks) { + $exp -= 1.0; + $blocksize = int ( 10 ** $exp ); + $cellcount_in_memory = $columncount * $blocksize; + if ($multithreaded) { + $cellcount_in_memory *= $threadqueuelength; + } + } + + if ($blocksize < $minblocksize) { + return $minblocksize; + } elsif ($blocksize > $maxblocksize) { + return $maxblocksize; + } else { + return $blocksize; + } + + } else { + + return $minblocksize; + + } + +} + +sub _get_other_threads_state { + my ($errorstates,$tid) = @_; + my $result = 0; + if (!defined $tid) { + $tid = threadid(); + } + if (defined $errorstates and ref $errorstates eq 'HASH') { + lock $errorstates; + foreach my $threadid (keys %$errorstates) { + if ($threadid != $tid) { + $result |= $errorstates->{$threadid}; + } + } + } + return $result; +} + +sub _get_stop_consumer_thread { + my ($context,$tid) = @_; + my $result = 1; + my $other_threads_state; + my $reader_state; + my $queuesize; + { + my $errorstates = $context->{errorstates}; + lock $errorstates; + $other_threads_state = _get_other_threads_state($errorstates,$tid); + $reader_state = $errorstates->{$context->{readertid}}; + } + $queuesize = $context->{queue}->pending(); + if (($other_threads_state & $ERROR) == 0 and ($queuesize > 0 or $reader_state == $RUNNING)) { + $result = 0; + #keep the consumer thread running if there is no defunct thread and queue is not empty or reader is still running + } + + if ($result) { + tablethreadingdebug('[' . $tid . '] consumer thread is shutting down (' . + (($other_threads_state & $ERROR) == 0 ? 'no defunct thread(s)' : 'defunct thread(s)') . ', ' . + ($queuesize > 0 ? 'blocks pending' : 'no blocks pending') . ', ' . + ($reader_state == $RUNNING ? 'reader thread running' : 'reader thread not running') . ') ...' + ,getlogger(__PACKAGE__)); + } + + return $result; + +} + +sub _reader { + + #my ($queue,$readererrorstate_ref,$writererrorstate_ref,$get_db,$tablename,$selectstatement,$blocksize,$rowcount,$logger,@values) = @_; + my $context = shift; + + my $reader_db; + my $tid = threadid(); + $context->{tid} = $tid; + { + lock $context->{errorstates}; + $context->{errorstates}->{$tid} = $RUNNING; + } + + tablethreadingdebug('[' . $tid . '] reader thread tid ' . $tid . ' started',getlogger(__PACKAGE__)); + + my $blockcount = 0; + eval { + $reader_db = &{$context->{get_db}}(); #$reader_connection_name); + $reader_db->db_get_begin($context->{selectstatement},@{$context->{values_ref}}); #$context->{tablename} + tablethreadingdebug('[' . $tid . '] reader thread waiting for consumer threads',getlogger(__PACKAGE__)); + while ((_get_other_threads_state($context->{errorstates},$tid) & $RUNNING) == 0) { #wait on cosumers to come up + #yield(); + sleep($thread_sleep_secs); + } + my $i = 0; + my $state = $RUNNING; #start at first + while (($state & $RUNNING) == $RUNNING and ($state & $ERROR) == 0) { #as long there is one running consumer and no defunct consumer + fetching_rows($reader_db,$context->{tablename},$i,$context->{blocksize},$context->{rowcount},getlogger(__PACKAGE__)); + my $rowblock = $reader_db->db_get_rowblock($context->{blocksize}); + my $realblocksize = scalar @$rowblock; + #my $packet = {rows => $rowblock, + # size => $realblocksize, + # #block => $i, + # row_offset => $i}; + my %packet :shared = (); + $packet{rows} = $rowblock; + $packet{size} = $realblocksize; + $packet{row_offset} = $i; + if ($realblocksize > 0) { + $context->{queue}->enqueue(\%packet); #$packet); + $blockcount++; + #wait if thequeue is full and there there is one running consumer + while (((($state = _get_other_threads_state($context->{errorstates},$tid)) & $RUNNING) == $RUNNING) and $context->{queue}->pending() >= $context->{threadqueuelength}) { + #yield(); + sleep($thread_sleep_secs); + } + $i += $realblocksize; + if ($realblocksize < $context->{blocksize}) { + tablethreadingdebug('[' . $tid . '] reader thread is shutting down (end of data) ...',getlogger(__PACKAGE__)); + last; + } + } else { + $context->{queue}->enqueue(\%packet); #$packet); + tablethreadingdebug('[' . $tid . '] reader thread is shutting down (end of data - empty block) ...',getlogger(__PACKAGE__)); + last; + } + } + if (not (($state & $RUNNING) == $RUNNING and ($state & $ERROR) == 0)) { + tablethreadingdebug('[' . $tid . '] reader thread is shutting down (' . + (($state & $RUNNING) == $RUNNING ? 'still running consumer threads' : 'no running consumer threads') . ', ' . + (($state & $ERROR) == 0 ? 'no defunct thread(s)' : 'defunct thread(s)') . ') ...' + ,getlogger(__PACKAGE__)); + } + $reader_db->db_finish(); + }; + # stop the consumer: + # $context->{queue}->enqueue(undef); + if (defined $reader_db) { + # if thread cleanup has a problem... + $reader_db->db_disconnect(); + } + tablethreadingdebug($@ ? '[' . $tid . '] reader thread error: ' . $@ : '[' . $tid . '] reader thread finished (' . $blockcount . ' blocks)',getlogger(__PACKAGE__)); + lock $context->{errorstates}; + if ($@) { + $context->{errorstates}->{$tid} = $ERROR; + } else { + $context->{errorstates}->{$tid} = $COMPLETED; + } + return $context->{errorstates}->{$tid}; +} + +sub _writer { + + my $context = shift; + + #get_target_db + my $writer_db; + my $tid = threadid(); + $context->{tid} = $tid; + { + lock $context->{errorstates}; + $context->{errorstates}->{$tid} = $RUNNING; + } + tablethreadingdebug('[' . $tid . '] writer thread tid ' . $tid . ' started',getlogger(__PACKAGE__)); + + my $blockcount = 0; + eval { + $writer_db = &{$context->{get_target_db}}(); #$writer_connection_name); + while (not _get_stop_consumer_thread($context,$tid)) { + my $packet = $context->{queue}->dequeue_nb(); + if (defined $packet) { + if ($packet->{size} > 0) { + writing_rows($writer_db,$context->{targettablename},$packet->{row_offset},$packet->{size},$context->{rowcount},getlogger(__PACKAGE__)); + + $writer_db->db_do_begin($context->{insertstatement}); #,$context->{targettablename}); + $writer_db->db_do_rowblock($packet->{rows}); + $writer_db->db_finish(); + $blockcount++; + + } else { #empty packet received + tablethreadingdebug('[' . $tid . '] shutting down writer thread (end of data - empty block) ...',getlogger(__PACKAGE__)); + last; + } + } else { + #yield(); + sleep($thread_sleep_secs); #2015-01 + } + } + }; + if (defined $writer_db) { + # if thread cleanup has a problem... + $writer_db->db_disconnect(); + } + tablethreadingdebug($@ ? '[' . $tid . '] writer thread error: ' . $@ : '[' . $tid . '] writer thread finished (' . $blockcount . ' blocks)',getlogger(__PACKAGE__)); + lock $context->{errorstates}; + if ($@) { + $context->{errorstates}->{$tid} = $ERROR; + } else { + $context->{errorstates}->{$tid} = $COMPLETED; + } + return $context->{errorstates}->{$tid}; +} + +sub _process { + + my $context = shift; + + #my $writer_db; + my $rowblock_result = 1; + my $tid = threadid(); + $context->{tid} = $tid; + { + lock $context->{errorstates}; + $context->{errorstates}->{$tid} = $RUNNING; + } + + tablethreadingdebug('[' . $tid . '] processor thread tid ' . $tid . ' started',getlogger(__PACKAGE__)); + + my $blockcount = 0; + eval { + if ('CODE' eq ref $context->{init_process_context_code}) { + &{$context->{init_process_context_code}}($context); + } + #$writer_db = &{$context->{get_target_db}}($writer_connection_name); + while (not _get_stop_consumer_thread($context,$tid)) { + my $packet = $context->{queue}->dequeue_nb(); + if (defined $packet) { + if ($packet->{size} > 0) { + + #writing_rows($writer_db,$context->{targettablename},$i,$realblocksize,$context->{rowcount},getlogger(__PACKAGE__)); + + #$writer_db->db_do_begin($context->{insertstatement}); #,$context->{targettablename}); + #$writer_db->db_do_rowblock($rowblock); + #$writer_db->db_finish(); + + #$i += $realblocksize; + + processing_rows($tid,$packet->{row_offset},$packet->{size},$context->{rowcount},getlogger(__PACKAGE__)); + + $rowblock_result = &{$context->{process_code}}($context,$packet->{rows},$packet->{row_offset}); + + $blockcount++; + + #$i += $realblocksize; + + if (not $rowblock_result) { + tablethreadingdebug('[' . $tid . '] shutting down processor thread (processing block NOK) ...',getlogger(__PACKAGE__)); + last; + } + + } else { + tablethreadingdebug('[' . $tid . '] shutting down processor thread (end of data - empty block) ...',getlogger(__PACKAGE__)); + last; + } + } else { + #yield(); + sleep($thread_sleep_secs); #2015-01 + } + } + + }; + my $err = $@; + tablethreadingdebug($err ? '[' . $tid . '] processor thread error: ' . $err : '[' . $tid . '] processor thread finished (' . $blockcount . ' blocks)',getlogger(__PACKAGE__)); + eval { + if ('CODE' eq ref $context->{uninit_process_context_code}) { + &{$context->{uninit_process_context_code}}($context); + } + }; + lock $context->{errorstates}; + if ($err) { + $context->{errorstates}->{$tid} = $ERROR; + } else { + $context->{errorstates}->{$tid} = (not $rowblock_result) ? $ERROR : $COMPLETED; + } + return $context->{errorstates}->{$tid}; +} + +1; diff --git a/lib/NGCP/BulkProcessor/SqlRecord.pm b/lib/NGCP/BulkProcessor/SqlRecord.pm index cb50cdb8..46a4ad34 100644 --- a/lib/NGCP/BulkProcessor/SqlRecord.pm +++ b/lib/NGCP/BulkProcessor/SqlRecord.pm @@ -3,129 +3,19 @@ use strict; ## no critic -use threads qw(yield); -use threads::shared; -use Thread::Queue; -#use Thread::Semaphore; - -use Time::HiRes qw(sleep); - -use NGCP::BulkProcessor::Globals qw( - $enablemultithreading - $cpucount - $cells_transfer_memory_limit - $transfer_defer_indexes -); - -use NGCP::BulkProcessor::Logging qw( - getlogger - fieldnamesaquired - primarykeycolsaquired - tableinfoscleared - - tablefixed - - tabletransferstarted - tableprocessingstarted - - rowtransferstarted - rowtransferred - rowskipped - rowinserted - rowupdated - rowsdeleted - totalrowsdeleted - rowinsertskipped - rowupdateskipped - tabletransferdone - tableprocessingdone - rowtransferdone - - fetching_rows - writing_rows - processing_rows - - tablethreadingdebug -); - -use NGCP::BulkProcessor::LogError qw( - fieldnamesdiffer - transferzerorowcount - processzerorowcount - deleterowserror - tabletransferfailed - tableprocessingfailed -); - use NGCP::BulkProcessor::Table qw(get_rowhash); -use NGCP::BulkProcessor::Array qw(setcontains contains); -use NGCP::BulkProcessor::Utils qw(round threadid); + +use NGCP::BulkProcessor::SqlProcessor qw(init_record); require Exporter; our @ISA = qw(Exporter); -our @EXPORT_OK = qw( - transfer_table - copy_row - process_table - cleartableinfo - checktableinfo - registertableinfo - create_targettable - transfer_record - transfer_records - insert_record - update_record - delete_records - - insert_stmt -); - -my $table_expected_fieldnames = {}; -my $table_fieldnames_cached = {}; -my $table_primarykeys = {}; -my $table_target_indexes = {}; - -#my $logger = getlogger(__PACKAGE__); - -my $tabletransfer_threadqueuelength = 5; #100; #30; #5; # ... >= 1 -my $minblocksize = 100; -my $maxblocksize = 100000; -my $minnumberofchunks = 10; - -my $tableprocessing_threadqueuelength = 10; -my $tableprocessing_threads = $cpucount; #3; - -my $reader_connection_name = 'reader'; -#my $writer_connection_name = 'writer'; - -my $thread_sleep_secs = 0.1; - -my $RUNNING = 1; -my $COMPLETED = 2; -my $ERROR = 4; +our @EXPORT_OK = qw(); sub new { my $class = shift; my $self = bless {}, $class; - my ($get_db,$tablename,$expected_fieldnames,$target_indexes) = @_; - - my $db = (ref $get_db eq 'CODE') ? &$get_db() : $get_db; - - my $connectidentifier = $db->connectidentifier(); - my $tid = threadid(); - - checktableinfo($db,$tablename,$expected_fieldnames,$target_indexes); - - - if (defined $table_expected_fieldnames->{$tid}->{$connectidentifier}->{$tablename}) { # and ref $table_fieldnames_cached->{$connectidentifier}->{$tablename} eq 'ARRAY') { - # if there are fieldnames defined, we make a member variable for each and set it to undef - foreach my $fieldname (@{$table_expected_fieldnames->{$tid}->{$connectidentifier}->{$tablename}}) { - $self->{$fieldname} = undef; - } - } - - return $self; + return init_record($self,@_); } @@ -141,1394 +31,4 @@ sub gethash { return get_rowhash(\@fieldvalues); } -sub cleartableinfo { - - my $get_db = shift; - - my $db = (ref $get_db eq 'CODE') ? &$get_db() : $get_db; - my $tid = threadid(); - - my $connectidentifier = $db->connectidentifier(); - - my $found = 0; - - if (exists $table_expected_fieldnames->{$tid}) { - if (exists $table_expected_fieldnames->{$tid}->{$connectidentifier}) { - delete $table_expected_fieldnames->{$tid}->{$connectidentifier}; - $found = 1; - } - } - if (exists $table_fieldnames_cached->{$tid}){ - if (exists $table_fieldnames_cached->{$tid}->{$connectidentifier}) { - delete $table_fieldnames_cached->{$tid}->{$connectidentifier}; - $found = 1; - } - } - if (exists $table_primarykeys->{$tid}) { - if (exists $table_primarykeys->{$tid}->{$connectidentifier}) { - delete $table_primarykeys->{$tid}->{$connectidentifier}; - $found = 1; - } - } - if (exists $table_target_indexes->{$tid}) { - if (exists $table_target_indexes->{$tid}->{$connectidentifier}) { - delete $table_target_indexes->{$tid}->{$connectidentifier}; - $found = 1; - } - } - - if ((scalar keys %{$table_expected_fieldnames->{$tid}}) == 0) { - delete $table_expected_fieldnames->{$tid}; - $found = 1; - } - if ((scalar keys %{$table_fieldnames_cached->{$tid}}) == 0) { - delete $table_fieldnames_cached->{$tid}; - $found = 1; - } - if ((scalar keys %{$table_primarykeys->{$tid}}) == 0) { - delete $table_primarykeys->{$tid}; - $found = 1; - } - if ((scalar keys %{$table_target_indexes->{$tid}}) == 0) { - delete $table_target_indexes->{$tid}; - $found = 1; - } - - if ($found) { - tableinfoscleared($db,getlogger(__PACKAGE__)); - } - -} - -sub registertableinfo { - - my ($get_db,$tablename,$fieldnames,$indexes,$keycols) = @_; - - my $db = (ref $get_db eq 'CODE') ? &$get_db() : $get_db; - - my $connectidentifier = $db->connectidentifier(); - my $tid = threadid(); - - if (not exists $table_expected_fieldnames->{$tid}) { - $table_expected_fieldnames->{$tid} = {}; - } - if (not exists $table_expected_fieldnames->{$tid}->{$connectidentifier}) { - # create an empty category for the connection if none exists yet: - $table_expected_fieldnames->{$tid}->{$connectidentifier} = {}; - } - # we prefer to always update the expected fieldnames (that come from a derived class) - $table_expected_fieldnames->{$tid}->{$connectidentifier}->{$tablename} = $fieldnames; - - if (not exists $table_fieldnames_cached->{$tid}) { - $table_fieldnames_cached->{$tid} = {}; - } - if (not exists $table_fieldnames_cached->{$tid}->{$connectidentifier}) { - # create an empty fieldname cache for the connection if none exists yet: - $table_fieldnames_cached->{$tid}->{$connectidentifier} = {}; - } - - if (not exists $table_primarykeys->{$tid}) { - $table_primarykeys->{$tid} = {}; - } - if (not exists $table_primarykeys->{$tid}->{$connectidentifier}) { - # create an empty primary key column name cache for the connection if none exists yet: - $table_primarykeys->{$tid}->{$connectidentifier} = {}; - } - $table_primarykeys->{$tid}->{$connectidentifier}->{$tablename} = $keycols; - - if (not exists $table_target_indexes->{$tid}) { - $table_target_indexes->{$tid} = {}; - } - if (not exists $table_target_indexes->{$tid}->{$connectidentifier}) { - # create an empty index set list for target tables for the connection if none exists yet: - $table_target_indexes->{$tid}->{$connectidentifier} = {}; - } - # we prefer to always update the target table indexes (that come from a derived class) - $table_target_indexes->{$tid}->{$connectidentifier}->{$tablename} = $indexes; - -} - -sub checktableinfo { - - my ($get_db,$tablename,$expected_fieldnames,$target_indexes) = @_; - - my $result = 1; - - my $db = (ref $get_db eq 'CODE') ? &$get_db() : $get_db; - - my $connectidentifier = $db->connectidentifier(); - my $tid = threadid(); - - if (not exists $table_expected_fieldnames->{$tid}) { - #$table_expected_fieldnames->{$tid} = shared_clone({}); - $table_expected_fieldnames->{$tid} = {}; - } - if (not exists $table_expected_fieldnames->{$tid}->{$connectidentifier}) { - # create an empty category for the connection if none exists yet: - #$table_expected_fieldnames->{$tid}->{$connectidentifier} = shared_clone({}); - $table_expected_fieldnames->{$tid}->{$connectidentifier} = {}; - } - # we prefer to always update the expected fieldnames (that come from a derived class) - #$table_expected_fieldnames->{$tid}->{$connectidentifier}->{$tablename} = shared_clone($expected_fieldnames); - $table_expected_fieldnames->{$tid}->{$connectidentifier}->{$tablename} = $expected_fieldnames; - - if (not exists $table_fieldnames_cached->{$tid}) { - #$table_fieldnames_cached->{$tid} = shared_clone({}); - $table_fieldnames_cached->{$tid} = {}; - } - if (not exists $table_fieldnames_cached->{$tid}->{$connectidentifier}) { - # create an empty fieldname cache for the connection if none exists yet: - #$table_fieldnames_cached->{$tid}->{$connectidentifier} = shared_clone({}); - $table_fieldnames_cached->{$tid}->{$connectidentifier} = {}; - } - - if (not exists $table_fieldnames_cached->{$tid}->{$connectidentifier}->{$tablename}) { - # query the database for fieldnames of the table if we don't have a cache entry yet: - #$table_fieldnames_cached->{$tid}->{$connectidentifier}->{$tablename} = shared_clone($db->getfieldnames($tablename)); - $table_fieldnames_cached->{$tid}->{$connectidentifier}->{$tablename} = $db->getfieldnames($tablename); - #my $fieldnames = $db->getfieldnames($tablename); - if (!defined $table_expected_fieldnames->{$tid}->{$connectidentifier}->{$tablename} or setcontains($table_expected_fieldnames->{$tid}->{$connectidentifier}->{$tablename},$table_fieldnames_cached->{$tid}->{$connectidentifier}->{$tablename},1)) { #fieldnames are case insensitive in general - # if not expected fieldnames are given or queried fieldnames match, we log this: - #$table_fieldnames_cached->{$connectidentifier}->{$tablename} = $table_expected_fieldnames->{$connectidentifier}->{$tablename}; - fieldnamesaquired($db,$tablename,getlogger(__PACKAGE__)); - } else { - # otherwise we log a failure (exit? - see Logging Module) - #$table_fieldnames_cached->{$connectidentifier}->{$tablename} = {}; #$fieldnames; - fieldnamesdiffer($db,$tablename,$table_expected_fieldnames->{$tid}->{$connectidentifier}->{$tablename},$table_fieldnames_cached->{$tid}->{$connectidentifier}->{$tablename},getlogger(__PACKAGE__)); - $result = 0; - } - } - - if (not exists $table_primarykeys->{$tid}) { - #$table_primarykeys->{$tid} = shared_clone({}); - $table_primarykeys->{$tid} = {}; - } - if (not exists $table_primarykeys->{$tid}->{$connectidentifier}) { - # create an empty primary key column name cache for the connection if none exists yet: - #$table_primarykeys->{$tid}->{$connectidentifier} = shared_clone({}); - $table_primarykeys->{$tid}->{$connectidentifier} = {}; - } - if (not exists $table_primarykeys->{$tid}->{$connectidentifier}->{$tablename}) { - # query the database for primary keys of the table if we don't have them cached yet: - #$table_primarykeys->{$tid}->{$connectidentifier}->{$tablename} = shared_clone($db->getprimarykeycols($tablename)); - $table_primarykeys->{$tid}->{$connectidentifier}->{$tablename} = $db->getprimarykeycols($tablename); - primarykeycolsaquired($db,$tablename,$table_primarykeys->{$tid}->{$connectidentifier}->{$tablename},getlogger(__PACKAGE__)); - } - - if (not exists $table_target_indexes->{$tid}) { - #$table_target_indexes->{$tid} = shared_clone({}); - $table_target_indexes->{$tid} = {}; - } - if (not exists $table_target_indexes->{$tid}->{$connectidentifier}) { - # create an empty index set list for target tables for the connection if none exists yet: - #$table_target_indexes->{$tid}->{$connectidentifier} = shared_clone({}); - $table_target_indexes->{$tid}->{$connectidentifier} = {}; - } - # we prefer to always update the target table indexes (that come from a derived class) - #$table_target_indexes->{$tid}->{$connectidentifier}->{$tablename} = shared_clone($target_indexes); - $table_target_indexes->{$tid}->{$connectidentifier}->{$tablename} = $target_indexes; - - return $result; - -} - -sub create_targettable { - - my ($get_db,$tablename,$get_target_db,$targettablename,$truncate,$defer_indexes,$texttable_engine) = @_; - - my $db = (ref $get_db eq 'CODE') ? &$get_db() : $get_db; - my $target_db = (ref $get_target_db eq 'CODE') ? &$get_target_db() : $get_target_db; - - #my $targettablename = _gettargettablename($db,$tablename,$target_db); - my $connectidentifier = $db->connectidentifier(); - my $tid = threadid(); - - if ($truncate and $defer_indexes) { - $target_db->drop_table($targettablename); - } - - my $result = $target_db->create_texttable($targettablename, - $table_expected_fieldnames->{$tid}->{$connectidentifier}->{$tablename}, - $table_primarykeys->{$tid}->{$connectidentifier}->{$tablename}, - $table_target_indexes->{$tid}->{$connectidentifier}->{$tablename}, - # 'ifnotexists' is always true - $truncate, - $defer_indexes, - $texttable_engine); - - checktableinfo($target_db,$targettablename,$table_expected_fieldnames->{$tid}->{$connectidentifier}->{$tablename},$defer_indexes ? undef : $table_target_indexes->{$tid}->{$connectidentifier}->{$tablename}); - return $result; - -} - -sub delete_records { - - my ($get_db,$tablename,$keyfields,$equal,$vals_table) = @_; - - my $db = (ref $get_db eq 'CODE') ? &$get_db() : $get_db; - - my $connectidentifier = $db->connectidentifier(); - my $tid = threadid(); - - my $expected_fieldnames = $table_expected_fieldnames->{$tid}->{$connectidentifier}->{$tablename}; - my $primarykeys = $table_primarykeys->{$tid}->{$connectidentifier}->{$tablename}; - - if (defined $expected_fieldnames and - (defined $keyfields and - ref $keyfields eq 'ARRAY') and - (defined $vals_table and - ref $vals_table eq 'NGCP::BulkProcessor::Table')) { - - my @fields = @$keyfields; - my $field_cnt = scalar @fields; - - my $total_rowcount = 0; - - my $initial_rowcount = $db->db_get_value('SELECT COUNT(*) FROM ' . $db->tableidentifier($tablename)); - - if ($field_cnt > 0) { - my $where_clause; - if ($equal) { - $where_clause = ' WHERE ' . join(' = ? AND ',map { local $_ = $_; $_ = $db->columnidentifier($_); $_; } @fields) . ' = ?'; - - my $count_stmt = 'SELECT COUNT(*) FROM ' . $db->tableidentifier($tablename) . $where_clause; - my $delete_stmt = 'DELETE FROM ' . $db->tableidentifier($tablename) . $where_clause; - - for (my $i = 0; $i < $vals_table->rowcount(); $i++) { - my @vals = $vals_table->getrow($i); - my $new_initial_rowcount = $db->db_get_value('SELECT COUNT(*) FROM ' . $db->tableidentifier($tablename)); - my $rowcount = $db->db_get_value($count_stmt,@vals); - $db->db_do($delete_stmt,@vals); - rowsdeleted($db,$tablename,$rowcount,$new_initial_rowcount,getlogger(__PACKAGE__)); - $total_rowcount += $rowcount; - } - - } elsif ($field_cnt == 1) { - my @ne_vals = $vals_table->getcol(0); - $where_clause = ' WHERE ' . $db->columnidentifier($fields[0]) . ' NOT IN (' . substr(',?' x scalar @ne_vals,1) . ')'; - my $count_stmt = 'SELECT COUNT(*) FROM ' . $db->tableidentifier($tablename) . $where_clause; - my $delete_stmt = 'DELETE FROM ' . $db->tableidentifier($tablename) . $where_clause; - my $rowcount = $db->db_get_value($count_stmt,@ne_vals); - $db->db_do($delete_stmt,@ne_vals); - rowsdeleted($db,$tablename,$rowcount,$initial_rowcount,getlogger(__PACKAGE__)); - $total_rowcount += $rowcount; - } else { - - deleterowserror($db,$tablename,'deletings rows by complementary identifier values works with a single identifier column only',getlogger(__PACKAGE__)); - return; - - } - } else { - my $delete_stmt = 'DELETE FROM ' . $db->tableidentifier($tablename); - my $count_stmt = 'SELECT COUNT(*) FROM ' . $db->tableidentifier($tablename); - my $rowcount = $db->db_get_value($count_stmt); - $db->db_do($delete_stmt); - rowsdeleted($db,$tablename,$rowcount,$initial_rowcount,getlogger(__PACKAGE__)); - $total_rowcount += $rowcount; - } - - $db->vacuum($tablename); - - #if ($total_rowcount > 0) { - totalrowsdeleted($db,$tablename,$total_rowcount,$initial_rowcount,getlogger(__PACKAGE__)); - #} - - return $total_rowcount; - - } - -} - -sub insert_record { - - my ($get_db,$tablename,$allowdupes,$row) = @_; - - my $db = (ref $get_db eq 'CODE') ? &$get_db() : $get_db; - - #my $targettablename = _gettargettablename($db,$tablename,$target_db); #$target_db->getsafetablename($db->tableidentifier($tablename)); - my $connectidentifier = $db->connectidentifier(); - my $tid = threadid(); - - my $expected_fieldnames = $table_expected_fieldnames->{$tid}->{$connectidentifier}->{$tablename}; - my $primarykeys = $table_primarykeys->{$tid}->{$connectidentifier}->{$tablename}; - - if (defined $expected_fieldnames and defined $row) { - - my @fieldnames = @$expected_fieldnames; - my @fields = (); - my @vals = (); - - foreach my $fieldname (@fieldnames) { - if (exists $row->{$fieldname}) { - push @fields,$fieldname; - push @vals,$row->{$fieldname}; - } - } - - my @pk_fieldnames; - my @pk_fields = (); - my @pk_vals = (); - - if (not $allowdupes) { - if (defined $primarykeys) { - @pk_fieldnames = @$primarykeys; - if (scalar @pk_fieldnames > 0) { - foreach my $fieldname (@pk_fieldnames) { - if (exists $row->{$fieldname}) { - push @pk_fields,$fieldname; - push @pk_vals,$row->{$fieldname}; - #} else { - # 'insert error: pk field not foun din row'; - # push @pk_vals,undef; - } - } - } else { - @pk_fields = @fields; - @pk_vals = @vals; - } - } else { - @pk_fields = @fields; - @pk_vals = @vals; - } - } - - if ($allowdupes or $db->db_get_value('SELECT COUNT(*) FROM ' . $db->tableidentifier($tablename) . ' WHERE ' . join(' = ? AND ',map { local $_ = $_; $_ = $db->columnidentifier($_); $_; } @pk_fields) . ' = ?',@pk_vals) == 0) { - $db->db_do('INSERT INTO ' . $db->tableidentifier($tablename) . ' (' . join(', ',map { local $_ = $_; $_ = $db->columnidentifier($_); $_; } @fields) . ') VALUES (' . substr(',?' x scalar @fields,1) . ')',@vals); - rowinserted($db,$tablename,getlogger(__PACKAGE__)); - return 1; - } else { - rowinsertskipped($db,$tablename,getlogger(__PACKAGE__)); - return 0; - } - - } - -} - -sub update_record { - - my ($get_db,$tablename,$row) = @_; - - my $db = (ref $get_db eq 'CODE') ? &$get_db() : $get_db; - - #my $targettablename = _gettargettablename($db,$tablename,$target_db); #$target_db->getsafetablename($db->tableidentifier($tablename)); - my $connectidentifier = $db->connectidentifier(); - my $tid = threadid(); - - my $expected_fieldnames = $table_expected_fieldnames->{$tid}->{$connectidentifier}->{$tablename}; - my $primarykeys = $table_primarykeys->{$tid}->{$connectidentifier}->{$tablename}; - - if (defined $expected_fieldnames and defined $row) { - - my @fieldnames = @$expected_fieldnames; - my @fields = (); - my @vals = (); - - foreach my $fieldname (@fieldnames) { - if (exists $row->{$fieldname}) { - push @fields,$fieldname; - push @vals,$row->{$fieldname}; - } - } - - my @pk_fieldnames; - my @pk_fields = (); - my @pk_vals = (); - - if (defined $primarykeys) { - @pk_fieldnames = @$primarykeys; - if (scalar @pk_fieldnames > 0) { - foreach my $fieldname (@pk_fieldnames) { - if (exists $row->{$fieldname}) { - push @pk_fields,$fieldname; - push @pk_vals,$row->{$fieldname}; - #} else { - # 'insert error: pk field not foun din row'; - # push @pk_vals,undef; - } - } - } else { - @pk_fields = @fields; - @pk_vals = @vals; - } - } else { - @pk_fields = @fields; - @pk_vals = @vals; - } - - my $selectpk_fieldnames = join(' = ? AND ',map { local $_ = $_; $_ = $db->columnidentifier($_); $_; } @pk_fields); - - if ($db->db_get_value('SELECT COUNT(*) FROM ' . $db->tableidentifier($tablename) . ' WHERE ' . $selectpk_fieldnames . ' = ?',@pk_vals) == 1) { - $db->db_do('UPDATE ' . $db->tableidentifier($tablename) . ' SET ' . join(' = ?, ',map { local $_ = $_; $_ = $db->columnidentifier($_); $_; } @fields) . ' = ? WHERE ' . $selectpk_fieldnames . ' = ?',@vals,@pk_vals); - rowupdated($db,$tablename,getlogger(__PACKAGE__)); - return 1; - } else { - rowupdateskipped($db,$tablename,getlogger(__PACKAGE__)); - return 0; - } - - } - -} - -sub transfer_record { - - #my $self = shift - my ($get_db,$tablename,$get_target_db,$targettablename,$allowdupes,$row) = @_; - - my $db = (ref $get_db eq 'CODE') ? &$get_db() : $get_db; - my $target_db = (ref $get_target_db eq 'CODE') ? &$get_target_db() : $get_target_db; - - #my $targettablename = _gettargettablename($db,$tablename,$target_db); #$target_db->getsafetablename($db->tableidentifier($tablename)); - my $connectidentifier = $db->connectidentifier(); - my $tid = threadid(); - - my $expected_fieldnames = $table_expected_fieldnames->{$tid}->{$connectidentifier}->{$tablename}; - my $primarykeys = $table_primarykeys->{$tid}->{$connectidentifier}->{$tablename}; - - if (defined $expected_fieldnames and defined $row) { - - my @fieldnames = @$expected_fieldnames; - my @vals = (); - - foreach my $fieldname (@fieldnames) { - push @vals,$row->{$fieldname}; - } - - my @pk_fieldnames; - my @pk_vals = (); - - if (not $allowdupes) { - if (defined $primarykeys) { - @pk_fieldnames = @$primarykeys; - if (scalar @pk_fieldnames > 0) { - foreach my $fieldname (@pk_fieldnames) { - push @pk_vals,$row->{$fieldname}; - } - } else { - @pk_fieldnames = @fieldnames; - @pk_vals = @vals; - } - } else { - @pk_fieldnames = @fieldnames; - @pk_vals = @vals; - } - } - - if ($allowdupes or $target_db->db_get_value('SELECT COUNT(*) FROM ' . $target_db->tableidentifier($targettablename) . ' WHERE ' . join(' = ? AND ',map { local $_ = $_; $_ = $target_db->columnidentifier($_); $_; } @pk_fieldnames) . ' = ?',@pk_vals) == 0) { - $target_db->db_do('INSERT INTO ' . $target_db->tableidentifier($targettablename) . ' (' . join(', ',map { local $_ = $_; $_ = $target_db->columnidentifier($_); $_; } @fieldnames) . ') VALUES (' . substr(',?' x scalar @fieldnames,1) . ')',@vals); - rowtransferred($db,$tablename,$target_db,$targettablename,1,1,getlogger(__PACKAGE__)); - return 1; - } else { - rowskipped($db,$tablename,$target_db,$targettablename,1,1,getlogger(__PACKAGE__)); - return 0; - } - - } - -} - -sub transfer_records { - - my ($get_db,$tablename,$get_target_db,$targettablename,$allowdupes,$rows) = @_; - - my $db = (ref $get_db eq 'CODE') ? &$get_db() : $get_db; - my $target_db = (ref $get_target_db eq 'CODE') ? &$get_target_db() : $get_target_db; - - #my $targettablename = _gettargettablename($db,$tablename,$target_db); #$target_db->getsafetablename($db->tableidentifier($tablename)); - my $connectidentifier = $db->connectidentifier(); - my $tid = threadid(); - - my $expected_fieldnames = $table_expected_fieldnames->{$tid}->{$connectidentifier}->{$tablename}; - my $primarykeys = $table_primarykeys->{$tid}->{$connectidentifier}->{$tablename}; - - if (defined $expected_fieldnames and defined $rows and ref $rows eq 'ARRAY') { # and defined $getrecords_code and ref $getrecords_code eq 'CODE') { - #get_local_db(); - - my $numofrows = scalar @$rows; - rowtransferstarted($db,$tablename,$target_db,$targettablename,$numofrows,getlogger(__PACKAGE__)); - - my @fieldnames = @$expected_fieldnames; - - my $setfieldnames = join(', ',map { local $_ = $_; $_ = $target_db->columnidentifier($_); $_; } @fieldnames); - my $valueplaceholders = substr(',?' x scalar @fieldnames,1); - - my $rowstransferred = 0; - - if ($allowdupes) { - - my @rows_array = (); - $target_db->db_do_begin('INSERT INTO ' . $target_db->tableidentifier($targettablename) . ' (' . $setfieldnames . ') VALUES (' . $valueplaceholders . ')'); - foreach my $row (@$rows) { - my @vals = (); - foreach my $fieldname (@fieldnames) { - push @vals,$row->{$fieldname}; - } - push @rows_array,\@vals; - } - $target_db->db_do_rowblock(\@rows_array); - $target_db->db_finish(); - $rowstransferred = scalar @rows_array; - - } else { - - my $i = 1; - - my @pk_fieldnames; - - if (defined $primarykeys) { - @pk_fieldnames = @$primarykeys; - if (scalar @pk_fieldnames == 0) { - @pk_fieldnames = @fieldnames; - } - } else { - @pk_fieldnames = @fieldnames; - } - - my $selectpk_fieldnames = join(' = ? AND ',map { local $_ = $_; $_ = $target_db->columnidentifier($_); $_; } @pk_fieldnames) . ' = ?'; - - foreach my $row (@$rows) { - - my @vals = (); - - foreach my $fieldname (@fieldnames) { - push @vals,$row->{$fieldname}; - } - - my @pk_vals; - - foreach my $fieldname (@pk_fieldnames) { - push @pk_vals,$row->{$fieldname}; - } - - if ($target_db->db_get_value('SELECT COUNT(*) FROM ' . $target_db->tableidentifier($targettablename) . ' WHERE ' . $selectpk_fieldnames,@pk_vals) == 0) { - $target_db->db_do('INSERT INTO ' . $db->target_tableidentifier($targettablename) . ' (' . $setfieldnames . ') VALUES (' . $valueplaceholders . ')',@vals); - rowtransferred($db,$tablename,$target_db,$targettablename,$i,$numofrows,getlogger(__PACKAGE__)); - $rowstransferred += 1; - } else { - rowskipped($db,$tablename,$target_db,$targettablename,$i,$numofrows,getlogger(__PACKAGE__)); - } - $i++; - } - } - rowtransferdone($db,$tablename,$target_db,$targettablename,$numofrows,getlogger(__PACKAGE__)); - return $rowstransferred; - } - -} - -sub insert_stmt { - - my ($get_db,$tablename,$insert_ignore) = @_; - my $db = (ref $get_db eq 'CODE') ? &$get_db() : $get_db; - my $connectidentifier = $db->connectidentifier(); - my $tid = threadid(); - my $expected_fieldnames = $table_expected_fieldnames->{$tid}->{$connectidentifier}->{$tablename}; - return 'INSERT ' . ($insert_ignore ? $db->insert_ignore_phrase() . ' ' : '') . 'INTO ' . $db->tableidentifier($tablename) . ' (' . - join(', ',map { local $_ = $_; $_ = $db->columnidentifier($_); $_; } @$expected_fieldnames) . - ') VALUES (' . substr(',?' x scalar @$expected_fieldnames,1) . ')'; - -} - -sub transfer_table { - - my ($get_db,$tablename,$get_target_db,$targettablename,$truncate_targettable,$create_indexes,$texttable_engine,$fixtable_statements,$selectcount,$select,@values) = @_; - - if (ref $get_db eq 'CODE' and ref $get_target_db eq 'CODE') { - - my $db = &$get_db($reader_connection_name,1); - my $target_db = &$get_target_db(); #$writer_connection_name); - - my $countstatement; - if (defined $selectcount) { - $countstatement = $selectcount; - } else { - $countstatement = 'SELECT COUNT(*) FROM ' . $db->tableidentifier($tablename); - } - - my $rowcount = $db->db_get_value($countstatement,@values); - - #my $targettablename = _gettargettablename($db,$tablename,$target_db); #$target_db->getsafetablename($db->tableidentifier($tablename)); - - if ($rowcount > 0) { - tabletransferstarted($db,$tablename,$target_db,$targettablename,$rowcount,getlogger(__PACKAGE__)); - } else { - transferzerorowcount($db,$tablename,$target_db,$targettablename,$rowcount,getlogger(__PACKAGE__)); - return; - } - - my $errorstate = $RUNNING; # 1; - - $create_indexes = ((defined $create_indexes) ? $create_indexes : $transfer_defer_indexes); - - if (create_targettable($db,$tablename,$target_db,$targettablename,$truncate_targettable,$create_indexes,$texttable_engine)) { - - my $connectidentifier = $db->connectidentifier(); - my $tid = threadid(); - my $expected_fieldnames = $table_expected_fieldnames->{$tid}->{$connectidentifier}->{$tablename}; - - my @fieldnames = @$expected_fieldnames; - - #my $setfieldnames = join(', ',map { local $_ = $_; $_ = $db->columnidentifier($_); $_; } @fieldnames); - my $valueplaceholders = substr(',?' x scalar @fieldnames,1); - - my $selectstatement; - if (length($select) > 0) { - $selectstatement = $select; - } else { - $selectstatement = 'SELECT ' . join(', ',map { local $_ = $_; $_ = $db->columnidentifier($_); $_; } @fieldnames) . ' FROM ' . $db->tableidentifier($tablename) - } - - my $insertstatement = 'INSERT INTO ' . $target_db->tableidentifier($targettablename) . ' (' . join(', ',map { local $_ = $_; $_ = $target_db->columnidentifier($_); $_; } @fieldnames) . ') VALUES (' . $valueplaceholders . ')'; - - my $blocksize; - - if ($enablemultithreading and $db->multithreading_supported() and $target_db->multithreading_supported() and $cpucount > 1) { # and $multithreaded) { # definitely no multithreading when CSVDB is involved - - $blocksize = _calc_blocksize($rowcount,scalar @fieldnames,1,$tabletransfer_threadqueuelength); - - my $reader; - my $writer; - - my %errorstates :shared = (); - #$errorstates{$tid} = $errorstate; - - #my $readererrorstate :shared = 1; - #my $writererrorstate :shared = 1; - - my $queue = Thread::Queue->new(); - - tablethreadingdebug('shutting down db connections ...',getlogger(__PACKAGE__)); - - $db->db_disconnect(); - #undef $db; - $target_db->db_disconnect(); - #undef $target_db; - my $default_connection = &$get_db(undef,0); - my $default_connection_reconnect = $default_connection->is_connected(); - $default_connection->db_disconnect(); - - tablethreadingdebug('starting reader thread',getlogger(__PACKAGE__)); - - $reader = threads->create(\&_reader, - { queue => $queue, - errorstates => \%errorstates, - #readererrorstate_ref => \$readererrorstate, - #writererrorstate_ref => \$writererrorstate, - threadqueuelength => $tabletransfer_threadqueuelength, - get_db => $get_db, - tablename => $tablename, - selectstatement => $selectstatement, - blocksize => $blocksize, - rowcount => $rowcount, - #logger => $logger, - values_ref => \@values, - }); - - tablethreadingdebug('starting writer thread',getlogger(__PACKAGE__)); - - $writer = threads->create(\&_writer, - { queue => $queue, - errorstates => \%errorstates, - readertid => $reader->tid(), - #readererrorstate_ref => \$readererrorstate, - #writererrorstate_ref => \$writererrorstate, - get_target_db => $get_target_db, - targettablename => $targettablename, - insertstatement => $insertstatement, - blocksize => $blocksize, - rowcount => $rowcount, - #logger => $logger, - }); - - $reader->join(); - tablethreadingdebug('reader thread joined',getlogger(__PACKAGE__)); - $writer->join(); - tablethreadingdebug('writer thread joined',getlogger(__PACKAGE__)); - - #$errorstate = $readererrorstate | $writererrorstate; - $errorstate = _get_other_threads_state(\%errorstates,$tid); - - tablethreadingdebug('restoring db connections ...',getlogger(__PACKAGE__)); - - #$db = &$get_db($reader_connection_name,1); - $target_db = &$get_target_db(undef,1); - if ($default_connection_reconnect) { - $default_connection = &$get_db(undef,1); - } - - } else { - - $blocksize = _calc_blocksize($rowcount,scalar @fieldnames,0,undef); - - #$db->db_disconnect(); - #undef $db; - #$db = &$get_db($reader_connection_name); - #$target_db->db_disconnect(); - #undef $target_db; - #$target_db = &$get_target_db($writer_connection_name); - - eval { - $db->db_get_begin($selectstatement,@values); #$tablename - - my $i = 0; - while (1) { - fetching_rows($db,$tablename,$i,$blocksize,$rowcount,getlogger(__PACKAGE__)); - my $rowblock = $db->db_get_rowblock($blocksize); - my $realblocksize = scalar @$rowblock; - if ($realblocksize > 0) { - writing_rows($target_db,$targettablename,$i,$realblocksize,$rowcount,getlogger(__PACKAGE__)); - $target_db->db_do_begin($insertstatement); #,$targettablename); - $target_db->db_do_rowblock($rowblock); - $target_db->db_finish(); - $i += $realblocksize; - - #foreach my $row (@$rowblock) { - # undef $row; - #} - #undef $rowblock; - - if ($realblocksize < $blocksize) { - last; - } - } else { - last; - } - } - $db->db_finish(); - - }; - - if ($@) { - $errorstate = $ERROR; - } else { - $errorstate = $COMPLETED; - } - - $db->db_disconnect(); - #undef $db; - #$target_db->db_disconnect(); - #undef $target_db; - - } - - #$db = &$get_db($controller_name,1); - #$target_db = &$get_target_db($controller_name,1); - - if ($errorstate == $COMPLETED and ref $fixtable_statements eq 'ARRAY' and (scalar @$fixtable_statements) > 0) { - eval { - foreach my $fixtable_statement (@$fixtable_statements) { - if (ref $fixtable_statement eq '') { - $target_db->db_do($fixtable_statement); - tablefixed($target_db,$targettablename,$fixtable_statement,getlogger(__PACKAGE__)); - } else { - $fixtable_statement = &$fixtable_statement($target_db->tableidentifier($targettablename)); - $target_db->db_do($fixtable_statement); - tablefixed($target_db,$targettablename,$fixtable_statement,getlogger(__PACKAGE__)); - } - - } - }; - if ($@) { - $errorstate = $ERROR; - #} else { - # $errorstate = $COMPLETED; - } - } - - if ($errorstate == $COMPLETED and $create_indexes) { - - eval { - $target_db->create_primarykey($targettablename, - $table_primarykeys->{$tid}->{$connectidentifier}->{$tablename}, - $table_expected_fieldnames->{$tid}->{$connectidentifier}->{$tablename}); - - $target_db->create_indexes($targettablename, - $table_target_indexes->{$tid}->{$connectidentifier}->{$tablename}, - $table_primarykeys->{$tid}->{$connectidentifier}->{$tablename}); - - - delete $table_primarykeys->{$tid}->{$target_db->connectidentifier()}->{$targettablename}; - checktableinfo($target_db,$targettablename,$table_expected_fieldnames->{$tid}->{$connectidentifier}->{$tablename},$table_target_indexes->{$tid}->{$connectidentifier}->{$tablename}); - - $target_db->vacuum($targettablename); - - }; - - if ($@) { - $errorstate = $ERROR; - #} else { - # $errorstate = $COMPLETED; - } - } - - } - - if ($errorstate == $COMPLETED) { - tabletransferdone($db,$tablename,$target_db,$targettablename,$rowcount,getlogger(__PACKAGE__)); - #$db->db_disconnect(); - #$target_db->db_disconnect(); - return 1; - } else { - tabletransferfailed($db,$tablename,$target_db,$targettablename,$rowcount,getlogger(__PACKAGE__)); - #$db->db_disconnect(); - #$target_db->db_disconnect(); - } - - } - - return 0; - -} - -sub process_table { - - my ($get_db,$tablename,$process_code,$init_process_context_code,$uninit_process_context_code,$multithreading,$selectcount,$select,@values) = @_; - - if (ref $get_db eq 'CODE') { - - my $db = &$get_db($reader_connection_name,1); - - my $countstatement; - if (defined $selectcount) { - $countstatement = $selectcount; - } else { - $countstatement = 'SELECT COUNT(*) FROM ' . $db->tableidentifier($tablename); - } - - my $rowcount = $db->db_get_value($countstatement,@values); - - if ($rowcount > 0) { - tableprocessingstarted($db,$tablename,$rowcount,getlogger(__PACKAGE__)); - } else { - processzerorowcount($db,$tablename,$rowcount,getlogger(__PACKAGE__)); - return; - } - - my $errorstate = $RUNNING; - - my $connectidentifier = $db->connectidentifier(); - my $tid = threadid(); - my $expected_fieldnames = $table_expected_fieldnames->{$tid}->{$connectidentifier}->{$tablename}; - - my @fieldnames = @$expected_fieldnames; - - #my $setfieldnames = join(', ',@fieldnames); - #my $valueplaceholders = substr(',?' x scalar @fieldnames,1); - - my $selectstatement; - if (length($select) > 0) { - $selectstatement = $select; - } else { - $selectstatement = 'SELECT ' . join(', ',map { local $_ = $_; $_ = $db->columnidentifier($_); $_; } @fieldnames) . ' FROM ' . $db->tableidentifier($tablename); - } - - my $blocksize; - - if ($enablemultithreading and $multithreading and $db->multithreading_supported() and $cpucount > 1) { # and $multithreaded) { # definitely no multithreading when CSVDB is involved - - $blocksize = _calc_blocksize($rowcount,scalar @fieldnames,1,$tableprocessing_threadqueuelength); - - my $reader; - #my $processor; - my %processors = (); - - my %errorstates :shared = (); - #$errorstates{$tid} = $errorstate; - - #my $readererrorstate :shared = 1; - #my $processorerrorstate :shared = 1; - - my $queue = Thread::Queue->new(); - - tablethreadingdebug('shutting down db connections ...',getlogger(__PACKAGE__)); - - $db->db_disconnect(); - #undef $db; - my $default_connection = &$get_db(undef,0); - my $default_connection_reconnect = $default_connection->is_connected(); - $default_connection->db_disconnect(); - - tablethreadingdebug('starting reader thread',getlogger(__PACKAGE__)); - - $reader = threads->create(\&_reader, - { queue => $queue, - errorstates => \%errorstates, - #readererrorstate_ref => \$readererrorstate, - #writererrorstate_ref => \$processorerrorstate, - threadqueuelength => $tableprocessing_threadqueuelength, - get_db => $get_db, - tablename => $tablename, - selectstatement => $selectstatement, - blocksize => $blocksize, - rowcount => $rowcount, - #logger => $logger, - values_ref => \@values, - }); - - for (my $i = 0; $i < $tableprocessing_threads; $i++) { - tablethreadingdebug('starting processor thread ' . ($i + 1) . ' of ' . $tableprocessing_threads,getlogger(__PACKAGE__)); - my $processor = threads->create(\&_process, - { queue => $queue, - errorstates => \%errorstates, - readertid => $reader->tid(), - #readererrorstate_ref => \$readererrorstate, - #processorerrorstate_ref => \$processorerrorstate, - process_code => $process_code, - init_process_context_code => $init_process_context_code, - uninit_process_context_code => $uninit_process_context_code, - blocksize => $blocksize, - rowcount => $rowcount, - #logger => $logger, - }); - if (!defined $processor) { - tablethreadingdebug('processor thread ' . ($i + 1) . ' of ' . $tableprocessing_threads . ' NOT started',getlogger(__PACKAGE__)); - } - $processors{$processor->tid()} = $processor; - #push (@processors,$processor); - } - - #$reader->join(); - #tablethreadingdebug('reader thread joined',getlogger(__PACKAGE__)); - #for (my $i = 0; $i < $tableprocessing_threads; $i++) { - # my $processor = $processors[$i]; - # if (defined $processor) { - # $processor->join(); - # tablethreadingdebug('processor thread ' . ($i + 1) . ' of ' . $tableprocessing_threads . ' joinded',getlogger(__PACKAGE__)); - # } else { - # tablethreadingdebug('processor thread ' . ($i + 1) . ' of ' . $tableprocessing_threads . ' NOT joinded',getlogger(__PACKAGE__)); - # } - #} - - $reader->join(); - tablethreadingdebug('reader thread joined',getlogger(__PACKAGE__)); - #print 'threads running: ' . (scalar threads->list(threads::running)); - #while ((scalar threads->list(threads::running)) > 1 or (scalar threads->list(threads::joinable)) > 0) { - while ((scalar keys %processors) > 0) { - #for (my $i = 0; $i < $tableprocessing_threads; $i++) { - foreach my $processor (values %processors) { - #my $processor = $processors[$i]; - if (defined $processor and $processor->is_joinable()) { - $processor->join(); - delete $processors{$processor->tid()}; - #tablethreadingdebug('processor thread ' . ($i + 1) . ' of ' . $tableprocessing_threads . ' joinded',getlogger(__PACKAGE__)); - tablethreadingdebug('processor thread tid ' . $processor->tid() . ' joined',getlogger(__PACKAGE__)); - } - #} else { - # tablethreadingdebug('processor thread ' . ($i + 1) . ' of ' . $tableprocessing_threads . ' NOT joinded',getlogger(__PACKAGE__)); - #} - } - sleep($thread_sleep_secs); - } - - #$errorstate = $readererrorstate | $processorerrorstate; - $errorstate = (_get_other_threads_state(\%errorstates,$tid) & ~$RUNNING); - - tablethreadingdebug('restoring db connections ...',getlogger(__PACKAGE__)); - - #$db = &$get_db($reader_connection_name,1); - if ($default_connection_reconnect) { - $default_connection = &$get_db(undef,1); - } - - } else { - - $blocksize = _calc_blocksize($rowcount,scalar @fieldnames,0,undef); - #$db->db_disconnect(); - #undef $db; - #$db = &$get_db($reader_connection_name); - my $context = { tid => $tid }; - my $rowblock_result = 1; - eval { - if ('CODE' eq ref $init_process_context_code) { - &$init_process_context_code($context); - } - - $db->db_get_begin($selectstatement,@values); #$tablename - - my $i = 0; - while (1) { - fetching_rows($db,$tablename,$i,$blocksize,$rowcount,getlogger(__PACKAGE__)); - my $rowblock = $db->db_get_rowblock($blocksize); - my $realblocksize = scalar @$rowblock; - if ($realblocksize > 0) { - processing_rows($tid,$i,$realblocksize,$rowcount,getlogger(__PACKAGE__)); - - $rowblock_result = &$process_code($context,$rowblock,$i); - - #$target_db->db_do_begin($insertstatement); #,$targettablename); - #$target_db->db_do_rowblock($rowblock); - #$target_db->db_finish(); - $i += $realblocksize; - - if ($realblocksize < $blocksize || not $rowblock_result) { - last; - } - } else { - last; - } - } - $db->db_finish(); - - }; - - if ($@) { - $errorstate = $ERROR; - } else { - $errorstate = (not $rowblock_result) ? $ERROR : $COMPLETED; - } - - eval { - if ('CODE' eq ref $uninit_process_context_code) { - &$uninit_process_context_code($context); - } - }; - $db->db_disconnect(); - #undef $db; - - } - - #$db = &$get_db($controller_name,1); - - if ($errorstate == $COMPLETED) { - tableprocessingdone($db,$tablename,$rowcount,getlogger(__PACKAGE__)); - #$db->db_disconnect(); - return 1; - } else { - tableprocessingfailed($db,$tablename,$rowcount,getlogger(__PACKAGE__)); - #$db->db_disconnect(); - } - - } - - return 0; - -} - -sub _calc_blocksize { - - my ($rowcount,$columncount,$multithreaded,$threadqueuelength) = @_; - - if ($rowcount > $minblocksize) { - - my $exp = int ( log ($rowcount) / log(10.0) ); - my $blocksize = int ( 10 ** $exp ); - my $cellcount_in_memory = $columncount * $blocksize; - if ($multithreaded) { - $cellcount_in_memory *= $threadqueuelength; - } - - while ( $cellcount_in_memory > $cells_transfer_memory_limit or - $rowcount / $blocksize < $minnumberofchunks) { - $exp -= 1.0; - $blocksize = int ( 10 ** $exp ); - $cellcount_in_memory = $columncount * $blocksize; - if ($multithreaded) { - $cellcount_in_memory *= $threadqueuelength; - } - } - - if ($blocksize < $minblocksize) { - return $minblocksize; - } elsif ($blocksize > $maxblocksize) { - return $maxblocksize; - } else { - return $blocksize; - } - - } else { - - return $minblocksize; - - } - -} - -sub _get_other_threads_state { - my ($errorstates,$tid) = @_; - my $result = 0; - if (!defined $tid) { - $tid = threadid(); - } - if (defined $errorstates and ref $errorstates eq 'HASH') { - lock $errorstates; - foreach my $threadid (keys %$errorstates) { - if ($threadid != $tid) { - $result |= $errorstates->{$threadid}; - } - } - } - return $result; -} - -sub _get_stop_consumer_thread { - my ($context,$tid) = @_; - my $result = 1; - my $other_threads_state; - my $reader_state; - my $queuesize; - { - my $errorstates = $context->{errorstates}; - lock $errorstates; - $other_threads_state = _get_other_threads_state($errorstates,$tid); - $reader_state = $errorstates->{$context->{readertid}}; - } - $queuesize = $context->{queue}->pending(); - if (($other_threads_state & $ERROR) == 0 and ($queuesize > 0 or $reader_state == $RUNNING)) { - $result = 0; - #keep the consumer thread running if there is no defunct thread and queue is not empty or reader is still running - } - - if ($result) { - tablethreadingdebug('[' . $tid . '] consumer thread is shutting down (' . - (($other_threads_state & $ERROR) == 0 ? 'no defunct thread(s)' : 'defunct thread(s)') . ', ' . - ($queuesize > 0 ? 'blocks pending' : 'no blocks pending') . ', ' . - ($reader_state == $RUNNING ? 'reader thread running' : 'reader thread not running') . ') ...' - ,getlogger(__PACKAGE__)); - } - - return $result; - -} - -sub _reader { - - #my ($queue,$readererrorstate_ref,$writererrorstate_ref,$get_db,$tablename,$selectstatement,$blocksize,$rowcount,$logger,@values) = @_; - my $context = shift; - - my $reader_db; - my $tid = threadid(); - $context->{tid} = $tid; - { - lock $context->{errorstates}; - $context->{errorstates}->{$tid} = $RUNNING; - } - - tablethreadingdebug('[' . $tid . '] reader thread tid ' . $tid . ' started',getlogger(__PACKAGE__)); - - my $blockcount = 0; - eval { - $reader_db = &{$context->{get_db}}(); #$reader_connection_name); - $reader_db->db_get_begin($context->{selectstatement},@{$context->{values_ref}}); #$context->{tablename} - tablethreadingdebug('[' . $tid . '] reader thread waiting for consumer threads',getlogger(__PACKAGE__)); - while ((_get_other_threads_state($context->{errorstates},$tid) & $RUNNING) == 0) { #wait on cosumers to come up - #yield(); - sleep($thread_sleep_secs); - } - my $i = 0; - my $state = $RUNNING; #start at first - while (($state & $RUNNING) == $RUNNING and ($state & $ERROR) == 0) { #as long there is one running consumer and no defunct consumer - fetching_rows($reader_db,$context->{tablename},$i,$context->{blocksize},$context->{rowcount},getlogger(__PACKAGE__)); - my $rowblock = $reader_db->db_get_rowblock($context->{blocksize}); - my $realblocksize = scalar @$rowblock; - #my $packet = {rows => $rowblock, - # size => $realblocksize, - # #block => $i, - # row_offset => $i}; - my %packet :shared = (); - $packet{rows} = $rowblock; - $packet{size} = $realblocksize; - $packet{row_offset} = $i; - if ($realblocksize > 0) { - $context->{queue}->enqueue(\%packet); #$packet); - $blockcount++; - #wait if thequeue is full and there there is one running consumer - while (((($state = _get_other_threads_state($context->{errorstates},$tid)) & $RUNNING) == $RUNNING) and $context->{queue}->pending() >= $context->{threadqueuelength}) { - #yield(); - sleep($thread_sleep_secs); - } - $i += $realblocksize; - if ($realblocksize < $context->{blocksize}) { - tablethreadingdebug('[' . $tid . '] reader thread is shutting down (end of data) ...',getlogger(__PACKAGE__)); - last; - } - } else { - $context->{queue}->enqueue(\%packet); #$packet); - tablethreadingdebug('[' . $tid . '] reader thread is shutting down (end of data - empty block) ...',getlogger(__PACKAGE__)); - last; - } - } - if (not (($state & $RUNNING) == $RUNNING and ($state & $ERROR) == 0)) { - tablethreadingdebug('[' . $tid . '] reader thread is shutting down (' . - (($state & $RUNNING) == $RUNNING ? 'still running consumer threads' : 'no running consumer threads') . ', ' . - (($state & $ERROR) == 0 ? 'no defunct thread(s)' : 'defunct thread(s)') . ') ...' - ,getlogger(__PACKAGE__)); - } - $reader_db->db_finish(); - }; - # stop the consumer: - # $context->{queue}->enqueue(undef); - if (defined $reader_db) { - # if thread cleanup has a problem... - $reader_db->db_disconnect(); - } - tablethreadingdebug($@ ? '[' . $tid . '] reader thread error: ' . $@ : '[' . $tid . '] reader thread finished (' . $blockcount . ' blocks)',getlogger(__PACKAGE__)); - lock $context->{errorstates}; - if ($@) { - $context->{errorstates}->{$tid} = $ERROR; - } else { - $context->{errorstates}->{$tid} = $COMPLETED; - } - return $context->{errorstates}->{$tid}; -} - -sub _writer { - - my $context = shift; - - #get_target_db - my $writer_db; - my $tid = threadid(); - $context->{tid} = $tid; - { - lock $context->{errorstates}; - $context->{errorstates}->{$tid} = $RUNNING; - } - tablethreadingdebug('[' . $tid . '] writer thread tid ' . $tid . ' started',getlogger(__PACKAGE__)); - - my $blockcount = 0; - eval { - $writer_db = &{$context->{get_target_db}}(); #$writer_connection_name); - while (not _get_stop_consumer_thread($context,$tid)) { - my $packet = $context->{queue}->dequeue_nb(); - if (defined $packet) { - if ($packet->{size} > 0) { - writing_rows($writer_db,$context->{targettablename},$packet->{row_offset},$packet->{size},$context->{rowcount},getlogger(__PACKAGE__)); - - $writer_db->db_do_begin($context->{insertstatement}); #,$context->{targettablename}); - $writer_db->db_do_rowblock($packet->{rows}); - $writer_db->db_finish(); - $blockcount++; - - } else { #empty packet received - tablethreadingdebug('[' . $tid . '] shutting down writer thread (end of data - empty block) ...',getlogger(__PACKAGE__)); - last; - } - } else { - #yield(); - sleep($thread_sleep_secs); #2015-01 - } - } - }; - if (defined $writer_db) { - # if thread cleanup has a problem... - $writer_db->db_disconnect(); - } - tablethreadingdebug($@ ? '[' . $tid . '] writer thread error: ' . $@ : '[' . $tid . '] writer thread finished (' . $blockcount . ' blocks)',getlogger(__PACKAGE__)); - lock $context->{errorstates}; - if ($@) { - $context->{errorstates}->{$tid} = $ERROR; - } else { - $context->{errorstates}->{$tid} = $COMPLETED; - } - return $context->{errorstates}->{$tid}; -} - -sub _process { - - my $context = shift; - - #my $writer_db; - my $rowblock_result = 1; - my $tid = threadid(); - $context->{tid} = $tid; - { - lock $context->{errorstates}; - $context->{errorstates}->{$tid} = $RUNNING; - } - - tablethreadingdebug('[' . $tid . '] processor thread tid ' . $tid . ' started',getlogger(__PACKAGE__)); - - my $blockcount = 0; - eval { - if ('CODE' eq ref $context->{init_process_context_code}) { - &{$context->{init_process_context_code}}($context); - } - #$writer_db = &{$context->{get_target_db}}($writer_connection_name); - while (not _get_stop_consumer_thread($context,$tid)) { - my $packet = $context->{queue}->dequeue_nb(); - if (defined $packet) { - if ($packet->{size} > 0) { - - #writing_rows($writer_db,$context->{targettablename},$i,$realblocksize,$context->{rowcount},getlogger(__PACKAGE__)); - - #$writer_db->db_do_begin($context->{insertstatement}); #,$context->{targettablename}); - #$writer_db->db_do_rowblock($rowblock); - #$writer_db->db_finish(); - - #$i += $realblocksize; - - processing_rows($tid,$packet->{row_offset},$packet->{size},$context->{rowcount},getlogger(__PACKAGE__)); - - $rowblock_result = &{$context->{process_code}}($context,$packet->{rows},$packet->{row_offset}); - - $blockcount++; - - #$i += $realblocksize; - - if (not $rowblock_result) { - tablethreadingdebug('[' . $tid . '] shutting down processor thread (processing block NOK) ...',getlogger(__PACKAGE__)); - last; - } - - } else { - tablethreadingdebug('[' . $tid . '] shutting down processor thread (end of data - empty block) ...',getlogger(__PACKAGE__)); - last; - } - } else { - #yield(); - sleep($thread_sleep_secs); #2015-01 - } - } - - }; - my $err = $@; - tablethreadingdebug($err ? '[' . $tid . '] processor thread error: ' . $err : '[' . $tid . '] processor thread finished (' . $blockcount . ' blocks)',getlogger(__PACKAGE__)); - eval { - if ('CODE' eq ref $context->{uninit_process_context_code}) { - &{$context->{uninit_process_context_code}}($context); - } - }; - lock $context->{errorstates}; - if ($err) { - $context->{errorstates}->{$tid} = $ERROR; - } else { - $context->{errorstates}->{$tid} = (not $rowblock_result) ? $ERROR : $COMPLETED; - } - return $context->{errorstates}->{$tid}; -} - -sub copy_row { - my ($record,$row,$expected_fieldnames) = @_; - if (defined $record and defined $row) { - my $i; - if (ref $row eq 'ARRAY') { - $i = 0; - } elsif (ref $row eq 'HASH') { - $i = -1; - } elsif (ref $row eq ref $record) { - $i = -2; - } else { - $i = -3; - } - foreach my $fieldname (@$expected_fieldnames) { - if ($i >= 0) { - $record->{$fieldname} = $row->[$i]; - $i++; - } elsif ($i == -1 or $i == -2) { - if (exists $row->{$fieldname}) { - $record->{$fieldname} = $row->{$fieldname}; - } elsif (exists $row->{uc($fieldname)}) { - $record->{$fieldname} = $row->{uc($fieldname)}; - } else { - $record->{$fieldname} = undef; - } - } else { - last; - } - } - } - return $record; -} - 1;