diff --git a/ConnectorPool.pm b/ConnectorPool.pm index 03d1fa78..47d9291c 100644 --- a/ConnectorPool.pm +++ b/ConnectorPool.pm @@ -4,8 +4,6 @@ use strict; ## no critic use Globals qw( - $system_abbreviation - $system_instance $accounting_databasename $accounting_username @@ -18,7 +16,7 @@ use Globals qw( $billing_password $billing_host $billing_port - + $ngcprestapi_uri $ngcprestapi_username $ngcprestapi_password @@ -60,6 +58,8 @@ our @EXPORT_OK = qw( billing_db_tableidentifier destroy_dbs + get_connectorinstancename + get_cluster_db ); my $connectorinstancenameseparator = '_'; @@ -75,7 +75,7 @@ my $ngcp_restapis = {}; sub get_accounting_db { my ($instance_name,$reconnect) = @_; - my $name = _get_connectorinstancename($instance_name); + my $name = get_connectorinstancename($instance_name); if (!defined $accounting_dbs->{$name}) { $accounting_dbs->{$name} = SqlConnectors::MySQLDB->new($instance_name); if (!defined $reconnect) { @@ -101,7 +101,7 @@ sub accounting_db_tableidentifier { sub get_billing_db { my ($instance_name,$reconnect) = @_; - my $name = _get_connectorinstancename($instance_name); + my $name = get_connectorinstancename($instance_name); if (!defined $billing_dbs->{$name}) { $billing_dbs->{$name} = SqlConnectors::MySQLDB->new($instance_name); if (!defined $reconnect) { @@ -126,7 +126,7 @@ sub billing_db_tableidentifier { sub get_ngcp_restapi { my ($instance_name) = @_; - my $name = _get_connectorinstancename($instance_name); + my $name = get_connectorinstancename($instance_name); if (!defined $ngcp_restapis->{$name}) { $ngcp_restapis->{$name} = RestConnectors::NGCPRestApi->new($instance_name,$ngcprestapi_uri,$ngcprestapi_username,$ngcprestapi_password,$ngcprestapi_realm); } @@ -135,7 +135,7 @@ sub get_ngcp_restapi { } -sub _get_connectorinstancename { +sub get_connectorinstancename { my ($name) = @_; my $instance_name = threadid(); if (length($name) > 0) { @@ -161,8 +161,7 @@ sub destroy_dbs { } - -sub _get_cluster_db { # oracle RAC and the like ... +sub get_cluster_db { # oracle RAC and the like ... my ($cluster,$instance_name,$reconnect) = @_; #if ((defined $cluster) and ref $cluster ne 'HASH') { @@ -211,7 +210,7 @@ sub _get_cluster_db { # oracle RAC and the like ... if ($@) { dbclusterwarn($cluster->{name},'node ' . $node->{label} . ' inactive',getlogger(__PACKAGE__)); delete $nodes->{$node->{label}}; - return _get_cluster_db($cluster,$instance_name,$reconnect); + return get_cluster_db($cluster,$instance_name,$reconnect); } else { #$db->cluster($cluster); return $db; @@ -219,7 +218,7 @@ sub _get_cluster_db { # oracle RAC and the like ... } else { dbclustererror($cluster->{name},'node ' . $node->{label} . ' configuration error',getlogger(__PACKAGE__)); delete $nodes->{$node->{label}}; - return _get_cluster_db($cluster,$instance_name,$reconnect); + return get_cluster_db($cluster,$instance_name,$reconnect); } } } else { @@ -233,4 +232,4 @@ sub _get_cluster_db { # oracle RAC and the like ... } -1; \ No newline at end of file +1; diff --git a/Dao/Trunk/billing/contract_balances.pm b/Dao/Trunk/billing/contract_balances.pm index 7288d32f..2d77f582 100644 --- a/Dao/Trunk/billing/contract_balances.pm +++ b/Dao/Trunk/billing/contract_balances.pm @@ -8,7 +8,7 @@ use File::Basename; use Cwd; use lib Cwd::abs_path(File::Basename::dirname(__FILE__) . '/../../../'); -use Logging qw(getlogger); +#use Logging qw(getlogger); use ConnectorPool qw(get_billing_db billing_db_tableidentifier); @@ -48,6 +48,8 @@ my $expected_fieldnames = [ 'underrun_lock', ]; +#my $indexes = { $tablename . '_subscribernumber' => ['subscribernumber(11)'] }; + sub new { my $class = shift; @@ -87,19 +89,16 @@ sub buildrecords_fromrows { sub gettablename { - my $db = &$get_local_db(); - return &$get_tablename($db,$tablename); + return &$get_tablename($get_db,$tablename); } sub check_table { - my $db = &$get_local_db(); - return checktableinfo($get_db, gettablename(), $expected_fieldnames); } -1; \ No newline at end of file +1; diff --git a/FileProcessors/CSVFile.pm b/FileProcessors/CSVFile.pm index 7771ad77..07140442 100644 --- a/FileProcessors/CSVFile.pm +++ b/FileProcessors/CSVFile.pm @@ -7,9 +7,9 @@ use File::Basename; use Cwd; use lib Cwd::abs_path(File::Basename::dirname(__FILE__) . '/../'); -use Globals qw( - $cpucount -); +#use Globals qw( +# $cpucount +#); use Logging qw( getlogger @@ -27,7 +27,7 @@ my $default_encoding = 'UTF-8'; my $buffersize = 100 * 1024; my $threadqueuelength = 10; -my $numofthreads = $cpucount; #3; +my $default_numofthreads = 3; #my $multithreading = 0; my $blocksize = 100; @@ -37,12 +37,12 @@ sub new { my $self = FileProcessor->new(@_); + $self->{numofthreads} = shift // $default_numofthreads; $self->{line_separator} = shift // $default_lineseparator; $self->{field_separator} = shift // $default_fieldseparator; $self->{encoding} = shift // $default_encoding; $self->{buffersize} = $buffersize; $self->{threadqueuelength} = $threadqueuelength; - $self->{numofthreads} = $numofthreads; #$self->{multithreading} = $multithreading; $self->{blocksize} = $blocksize; diff --git a/Globals.pm b/Globals.pm index 51b737b7..53f001be 100644 --- a/Globals.pm +++ b/Globals.pm @@ -46,7 +46,7 @@ our @EXPORT_OK = qw( $cells_transfer_memory_limit $LongReadLen_limit - $defer_indexes + $transfer_defer_indexes $accounting_databasename $accounting_username @@ -136,7 +136,7 @@ our $cpucount = get_cpucount(); our $root_threadid = 0; #threadid() . ''; #0 our $cells_transfer_memory_limit = 10000000; #db fields -our $defer_indexes = 1; +our $transfer_defer_indexes = 1; #http://docstore.mik.ua/orelly/linux/dbi/ch06_01.htm our $LongReadLen_limit = 128*1024; #longest LOB field size in bytes @@ -264,9 +264,10 @@ sub update_mainconfig { $ngcprestapi_password = $data->{ngcprestapi_password} if exists $data->{ngcprestapi_password}; $ngcprestapi_realm = $data->{ngcprestapi_realm} if exists $data->{ngcprestapi_realm}; + $cpucount = $data->{cpucount} if exists $data->{cpucount}; $enablemultithreading = $data->{enablemultithreading} if exists $data->{enablemultithreading}; $cells_transfer_memory_limit = $data->{cells_transfer_memory_limit} if exists $data->{cells_transfer_memory_limit}; - $defer_indexes = $data->{defer_indexes} if exists $data->{defer_indexes}; + $transfer_defer_indexes = $data->{transfer_defer_indexes} if exists $data->{transfer_defer_indexes}; if (defined $split_tuplecode and ref $split_tuplecode eq 'CODE') { diff --git a/LogError.pm b/LogError.pm index 688d7864..32e6ab21 100644 --- a/LogError.pm +++ b/LogError.pm @@ -90,6 +90,9 @@ our @EXPORT_OK = qw( success completion + + scripterror + scriptwarn ); my $erroremailsubject = 'error: module '; @@ -521,7 +524,7 @@ sub fileprocessingwarn { my ($file,$message,$logger) = @_; my $message = basename($file) . ': ' . $message; if (defined $logger) { - $logger->error($message); + $logger->warn($message); } warning($message, $logger); @@ -689,6 +692,30 @@ sub _getsqlconnectorinstanceprefix { return ''; } +sub scripterror { + + my ($message, $logger) = @_; + if (defined $logger) { + $logger->error($message); + } + + terminate($message, $logger); + #terminatethreads(); + #die(); + +} + +sub scriptwarn { + + my ($message, $logger) = @_; + if (defined $logger) { + $logger->warn($message); + } + + warning($message, $logger); + +} + sub _getsqlconnectidentifiermessage { my ($db,$message) = @_; my $result = $db->connectidentifier(); diff --git a/Logging.pm b/Logging.pm index caa813c6..1c4c2d7c 100644 --- a/Logging.pm +++ b/Logging.pm @@ -71,7 +71,7 @@ our @EXPORT_OK = qw( init_log $currentlogfile $attachmentlogfile - cleanupinfo + scriptinfo xls2csvinfo tablethreadingdebug @@ -649,7 +649,9 @@ sub configurationinfo { } -sub cleanupinfo { + + +sub scriptinfo { my ($message,$logger) = @_; if (defined $logger) { diff --git a/Projects/Migration/IPGallery/Dao/FeatureOption.pm b/Projects/Migration/IPGallery/Dao/FeatureOption.pm new file mode 100644 index 00000000..d981be98 --- /dev/null +++ b/Projects/Migration/IPGallery/Dao/FeatureOption.pm @@ -0,0 +1,125 @@ +package Projects::Migration::IPGallery::Dao::FeatureOption; +use strict; + +## no critic + +use File::Basename; +use Cwd; +use lib Cwd::abs_path(File::Basename::dirname(__FILE__) . '/../../../../'); + +#use Logging qw(getlogger); + +use Projects::Migration::IPGallery::ProjectConnectorPool qw( + get_import_db + +); +#import_db_tableidentifier + +use SqlRecord qw( + registertableinfo + create_targettable + checktableinfo + copy_row + + insert_stmt +); + +require Exporter; +our @ISA = qw(Exporter SqlRecord); +our @EXPORT_OK = qw( + create_table + gettablename + check_table + getinsertstatement + + test_table_bycolumn1 + test_table_local_select + test_table_source_select + test_table_source_select_temptable +); + +my $tablename = 'feature_option'; +my $get_db = \&get_import_db; +#my $get_tablename = \&import_db_tableidentifier; + + +my $expected_fieldnames = [ 'subscribernumber', + 'option']; + +my $primarykey_fieldnames = [ 'subscribernumber', 'option' ]; + +my $indexes = {}; + +my $fixtable_statements = []; + +sub new { + + my $class = shift; + my $self = SqlRecord->new($get_db, + $tablename, + $expected_fieldnames,$indexes); + + bless($self,$class); + + copy_row($self,shift,$expected_fieldnames); + + return $self; + +} + +sub create_table { + + my ($truncate) = @_; + + my $db = &$get_db(); + + registertableinfo($db,$tablename,$expected_fieldnames,$indexes,$primarykey_fieldnames); + return create_targettable($db,$tablename,$db,$tablename,$truncate,0,undef); + +} + + +sub buildrecords_fromrows { + + my ($rows,$load_recursive) = @_; + + my @records = (); + my $record; + + if (defined $rows and ref $rows eq 'ARRAY') { + foreach my $row (@$rows) { + $record = __PACKAGE__->new($row); + + # transformations go here ... + + push @records,$record; + } + } + + return \@records; + +} + +sub getinsertstatement { + + check_table(); + return insert_stmt($get_db,$tablename); + +} + +sub gettablename { + + return $tablename; + +} + +sub check_table { + + return checktableinfo($get_db, + $tablename, + $expected_fieldnames, + $indexes); + +} + +1; diff --git a/Projects/Migration/IPGallery/Dao/FeatureOptionSet.pm b/Projects/Migration/IPGallery/Dao/FeatureOptionSet.pm new file mode 100644 index 00000000..94c1bc28 --- /dev/null +++ b/Projects/Migration/IPGallery/Dao/FeatureOptionSet.pm @@ -0,0 +1,126 @@ +package Projects::Migration::IPGallery::Dao::FeatureOptionSet; +use strict; + +## no critic + +use File::Basename; +use Cwd; +use lib Cwd::abs_path(File::Basename::dirname(__FILE__) . '/../../../../'); + +#use Logging qw(getlogger); + +use Projects::Migration::IPGallery::ProjectConnectorPool qw( + get_import_db + +); +#import_db_tableidentifier + +use SqlRecord qw( + registertableinfo + create_targettable + checktableinfo + copy_row + + insert_stmt +); + +require Exporter; +our @ISA = qw(Exporter SqlRecord); +our @EXPORT_OK = qw( + create_table + gettablename + check_table + getinsertstatement + + test_table_bycolumn1 + test_table_local_select + test_table_source_select + test_table_source_select_temptable +); + +my $tablename = 'feature_option_set'; +my $get_db = \&get_import_db; +#my $get_tablename = \&import_db_tableidentifier; + + +my $expected_fieldnames = [ 'subscribernumber', + 'option', + 'optionsetitem' ]; + +my $primarykey_fieldnames = [ 'subscribernumber', 'option', 'optionsetitem' ]; + +my $indexes = {}; + +my $fixtable_statements = []; + +sub new { + + my $class = shift; + my $self = SqlRecord->new($get_db, + $tablename, + $expected_fieldnames,$indexes); + + bless($self,$class); + + copy_row($self,shift,$expected_fieldnames); + + return $self; + +} + +sub create_table { + + my ($truncate) = @_; + + my $db = &$get_db(); + + registertableinfo($db,$tablename,$expected_fieldnames,$indexes,$primarykey_fieldnames); + return create_targettable($db,$tablename,$db,$tablename,$truncate,0,undef); + +} + + +sub buildrecords_fromrows { + + my ($rows,$load_recursive) = @_; + + my @records = (); + my $record; + + if (defined $rows and ref $rows eq 'ARRAY') { + foreach my $row (@$rows) { + $record = __PACKAGE__->new($row); + + # transformations go here ... + + push @records,$record; + } + } + + return \@records; + +} + +sub getinsertstatement { + + check_table(); + return insert_stmt($get_db,$tablename); + +} + +sub gettablename { + + return $tablename; + +} + +sub check_table { + + return checktableinfo($get_db, + $tablename, + $expected_fieldnames, + $indexes); + +} + +1; diff --git a/Projects/Migration/IPGallery/FeaturesDefineParser.pm b/Projects/Migration/IPGallery/FeaturesDefineParser.pm index 3e13025b..41edaab7 100644 --- a/Projects/Migration/IPGallery/FeaturesDefineParser.pm +++ b/Projects/Migration/IPGallery/FeaturesDefineParser.pm @@ -31,7 +31,7 @@ OptionValues ::= OptionValue+ action => _build_setoptio SubscriberNumber ~ [0-9]+ OptionName ~ [-a-zA-Z_0-9]+ -OptionValue ~ [-a-zA-Z_0-9]+ +OptionValue ~ [-a-zA-Z_0-9 \t]+ whitespace ~ [\s]+ :discard ~ whitespace __GRAMMAR__ diff --git a/Projects/Migration/IPGallery/FileProcessors/FeaturesDefineFile.pm b/Projects/Migration/IPGallery/FileProcessors/FeaturesDefineFile.pm index dddf951c..359a2a29 100644 --- a/Projects/Migration/IPGallery/FileProcessors/FeaturesDefineFile.pm +++ b/Projects/Migration/IPGallery/FileProcessors/FeaturesDefineFile.pm @@ -7,9 +7,9 @@ use File::Basename; use Cwd; use lib Cwd::abs_path(File::Basename::dirname(__FILE__) . '/../../../../'); -use Globals qw( - $cpucount -); +#use Globals qw( +# $cpucount +#); use Logging qw( getlogger @@ -34,11 +34,11 @@ my $encoding = 'UTF-8'; my $buffersize = 1400; # 512 * 1024; my $threadqueuelength = 10; -my $numofthreads = $cpucount; #3; +my $default_numofthreads = undef; #3; #my $multithreading = 0; -my $blocksize = 2000; +my $blocksize = 200; #2000; -my $stoponparseerrors = 0; #1; +my $stoponparseerrors = 1; #1; my $parselines = 0; sub new { @@ -47,11 +47,11 @@ sub new { my $self = FileProcessor->new(@_); + $self->{numofthreads} = shift // $default_numofthreads; $self->{line_separator} = $lineseparator; $self->{encoding} = $encoding; $self->{buffersize} = $buffersize; $self->{threadqueuelength} = $threadqueuelength; - $self->{numofthreads} = $numofthreads; #$self->{multithreading} = $multithreading; $self->{blocksize} = $blocksize; $self->{parselines} = $parselines; diff --git a/Projects/Migration/IPGallery/Import.pm b/Projects/Migration/IPGallery/Import.pm index ac8d4f8e..45727401 100644 --- a/Projects/Migration/IPGallery/Import.pm +++ b/Projects/Migration/IPGallery/Import.pm @@ -7,9 +7,13 @@ use File::Basename; use Cwd; use lib Cwd::abs_path(File::Basename::dirname(__FILE__) . '/../../../'); +use Globals qw( + $cpucount +); use Projects::Migration::IPGallery::Settings qw( - $defaultsettings - update_settings + $import_multithreading + $feature_define_import_numofthreads + $dry ); use Logging qw ( getlogger @@ -20,12 +24,18 @@ use LogError qw( ); #use FileProcessors::CSVFile; -use Projects::Migration::IPGallery::FileProcessors::FeaturesDefineFile; -use Projects::Migration::IPGallery::FeaturesDefineParser qw( - create_grammar - parse +use Projects::Migration::IPGallery::FileProcessors::FeaturesDefineFile qw(); +use Projects::Migration::IPGallery::FeaturesDefineParser qw(); + +use Projects::Migration::IPGallery::ProjectConnectorPool qw( + get_import_db ); +use Projects::Migration::IPGallery::Dao::FeatureOption qw(); +use Projects::Migration::IPGallery::Dao::FeatureOptionSet qw(); + +use Array qw(removeduplicates); + require Exporter; our @ISA = qw(Exporter); our @EXPORT_OK = qw( @@ -36,16 +46,20 @@ our @EXPORT_OK = qw( sub import_features_define { my ($file) = @_; - my $multithreading = 1; - my $importer = Projects::Migration::IPGallery::FileProcessors::FeaturesDefineFile->new(); - return $importer->process($file,sub { + my $result = Projects::Migration::IPGallery::Dao::FeatureOption::create_table(1); + $result &= Projects::Migration::IPGallery::Dao::FeatureOptionSet::create_table(1); + my $importer = Projects::Migration::IPGallery::FileProcessors::FeaturesDefineFile->new($feature_define_import_numofthreads); + $importer->{stoponparseerrors} = !$dry; + return $result && $importer->process($file,sub { my ($context,$rows,$row_offset) = @_; my $rownum = $row_offset; + my @featureoption_rows = (); + my @featureoptionset_rows = (); foreach my $line (@$rows) { my $row = undef; if (not $importer->{parselines}) { eval { - $row = parse(\$line,$context->{grammar}); + $row = Projects::Migration::IPGallery::FeaturesDefineParser::parse(\$line,$context->{grammar}); }; if ($@) { if ($importer->{stoponparseerrors}) { @@ -57,20 +71,53 @@ sub import_features_define { } next unless defined $row; $rownum++; - # continue to write to sqlite ... + foreach my $subscriber_number (keys %$row) { + foreach my $option (@{$row->{$subscriber_number}}) { + if ('HASH' eq ref $option) { + foreach my $setoption (keys %$option) { + foreach my $setoptionitem (@{removeduplicates($option->{$setoption})}) { + push(@featureoptionset_rows,[ $subscriber_number, $setoption, $setoptionitem ]); + } + push(@featureoption_rows,[ $subscriber_number, $setoption ]); + } + } else { + push(@featureoption_rows,[ $subscriber_number, $option ]); + } + } + } + } + + my $import_db = &get_import_db(); + if ((scalar @featureoption_rows) > 0) { + $import_db->db_do_begin( + Projects::Migration::IPGallery::Dao::FeatureOption::getinsertstatement(), + Projects::Migration::IPGallery::Dao::FeatureOption::gettablename(), + #lock - $import_multithreading + ); + $import_db->db_do_rowblock(\@featureoption_rows); + $import_db->db_finish(); + } + if ((scalar @featureoptionset_rows) > 0) { + $import_db->db_do_begin( + Projects::Migration::IPGallery::Dao::FeatureOptionSet::getinsertstatement(), + Projects::Migration::IPGallery::Dao::FeatureOptionSet::gettablename(), + #lock + ); + $import_db->db_do_rowblock(\@featureoptionset_rows); + $import_db->db_finish(); } return 1; }, sub { my ($context)= @_; if (not $importer->{parselines}) { eval { - $context->{grammar} = create_grammar(); + $context->{grammar} = Projects::Migration::IPGallery::FeaturesDefineParser::create_grammar(); }; if ($@) { fileprocessingerror($context->{filename},$@,getlogger(__PACKAGE__)); } } - },$multithreading); + },$import_multithreading); } diff --git a/Projects/Migration/IPGallery/ProjectConnectorPool.pm b/Projects/Migration/IPGallery/ProjectConnectorPool.pm new file mode 100644 index 00000000..2f000fe0 --- /dev/null +++ b/Projects/Migration/IPGallery/ProjectConnectorPool.pm @@ -0,0 +1,85 @@ +package Projects::Migration::IPGallery::ProjectConnectorPool; +use strict; + +## no critic + +use File::Basename; +use Cwd; +use lib Cwd::abs_path(File::Basename::dirname(__FILE__) . '/../../../'); + +use Projects::Migration::IPGallery::Settings qw( + $import_db_file +); + +use ConnectorPool qw( + get_connectorinstancename +); + +#use SqlConnectors::MySQLDB; +#use SqlConnectors::OracleDB; +#use SqlConnectors::PostgreSQLDB; +use SqlConnectors::SQLiteDB qw( + $staticdbfilemode + cleanupdbfiles +); +#use SqlConnectors::CSVDB; +#use SqlConnectors::SQLServerDB; +#use RestConnectors::NGCPRestApi; + +use SqlRecord qw(cleartableinfo); + +#use Utils qw(threadid); + +require Exporter; +our @ISA = qw(Exporter); +our @EXPORT_OK = qw( + get_import_db + import_db_tableidentifier + + destroy_dbs +); + +# thread connector pools: +my $import_dbs = {}; + + +sub get_import_db { + + my ($instance_name,$reconnect) = @_; + my $name = get_connectorinstancename($instance_name); #threadid(); #shift; + + if (not defined $import_dbs->{$name}) { + $import_dbs->{$name} = SqlConnectors::SQLiteDB->new($instance_name); #$name); + if (not defined $reconnect) { + $reconnect = 1; + } + } + if ($reconnect) { + $import_dbs->{$name}->db_connect($staticdbfilemode,$import_db_file); + } + + return $import_dbs->{$name}; + +} + +sub import_db_tableidentifier { + + my ($get_target_db,$tablename) = @_; + my $target_db = (ref $get_target_db eq 'CODE') ? &$get_target_db() : $get_target_db; + return $target_db->getsafetablename(SqlConnectors::SQLiteDB::get_tableidentifier($tablename,$staticdbfilemode,$import_db_file)); + +} + + +sub destroy_dbs { + + + foreach my $name (keys %$import_dbs) { + cleartableinfo($import_dbs->{$name}); + undef $import_dbs->{$name}; + delete $import_dbs->{$name}; + } + +} + +1; diff --git a/Projects/Migration/IPGallery/Settings.pm b/Projects/Migration/IPGallery/Settings.pm index 9c51e9ca..cfc91176 100644 --- a/Projects/Migration/IPGallery/Settings.pm +++ b/Projects/Migration/IPGallery/Settings.pm @@ -7,22 +7,51 @@ use File::Basename; use Cwd; use lib Cwd::abs_path(File::Basename::dirname(__FILE__) . '/../../../'); +use Globals qw( + update_working_path + $input_path + $enablemultithreading + $cpucount +); + +use Logging qw( + getlogger + scriptinfo +); + require Exporter; our @ISA = qw(Exporter); our @EXPORT_OK = qw( update_settings $defaultsettings + $defaultconfig + $features_define_filename + $feature_define_import_numofthreads + + $import_multithreading + $run_id + $dry + $force + $import_db_file + check_dry ); +our $defaultconfig = 'config.cfg'; our $defaultsettings = 'settings.cfg'; +our $features_define_filename = undef; +our $import_multithreading = $enablemultithreading; +our $feature_define_import_numofthreads = $cpucount; +our $force = 0; +our $dry = 1; +our $run_id = ''; - +our $import_db_file = ((defined $run_id and length($run_id) > 0) ? '_' : '') . 'import'; sub update_settings { - + my ($data,$configfile, $split_tuplecode, $format_number, @@ -31,20 +60,55 @@ sub update_settings { $configurationerrorcode, $fileerrorcode, $configlogger) = @_; - + if (defined $data) { -print "$configlogger narf"; - &$configurationinfocode("testinfomessage",$configlogger); - # databases - dsp - #$accounting_host = $config->{accounting_host} if exists $config->{accounting_host}; + #print "$configlogger narf"; + #&$configurationinfocode("testinfomessage",$configlogger); + + $features_define_filename = $data->{features_define_filename} if exists $data->{features_define_filename}; + if (defined $features_define_filename and length($features_define_filename) > 0) { + $features_define_filename = $input_path . $features_define_filename unless -e $features_define_filename; + } + + $import_multithreading = $data->{import_multithreading} if exists $data->{import_multithreading}; + #my $new_working_path = (exists $data->{working_path} ? $data->{working_path} : $working_path); + + $feature_define_import_numofthreads = $cpucount; +$feature_define_import_numofthreads = $data->{feature_define_import_numofthreads} if exists $data->{feature_define_import_numofthreads}; + $feature_define_import_numofthreads = $cpucount if $feature_define_import_numofthreads > $cpucount; + #return update_working_path($new_working_path,1,$fileerrorcode,$configlogger); + + $import_db_file = ((defined $run_id and length($run_id) > 0) ? '_' : '') . 'import'; + + $dry = $data->{dry} if exists $data->{dry}; return 1; - + } return 0; - + } +sub check_dry { + + if ($dry) { + scriptinfo('running in dry mode - NGCP databases will not be modified',getlogger(__PACKAGE__)); + return 1; + } else { + scriptinfo('NO DRY MODE - NGCP DATABASES WILL BE MODIFIED!',getlogger(__PACKAGE__)); + if (!$force) { + if ('yes' eq lc(prompt("Type 'yes' to proceed: "))) { + return 1; + } else { + return 0; + } + } else { + scriptinfo('force option applied',getlogger(__PACKAGE__)); + return 1; + } + } + +} -1; \ No newline at end of file +1; diff --git a/Projects/Migration/IPGallery/config.cfg b/Projects/Migration/IPGallery/config.cfg new file mode 100644 index 00000000..c2582e96 --- /dev/null +++ b/Projects/Migration/IPGallery/config.cfg @@ -0,0 +1,39 @@ +##general settings: +working_path = /var/sipwise/Migration/IPGallery +cpucount = 4 +enablemultithreading = 1 + +##gearman/service listener config: +jobservers = 127.0.0.1:4730 + +##NGCP MySQL connectivity - "accounting" db: +accounting_host = 127.0.0.1 +accounting_port = 3306 +accounting_databasename = accounting +accounting_username = root +accounting_password = + +##NGCP MySQL connectivity - "billing" db: +billing_host = 127.0.0.1 +billing_port = 3306 +billing_databasename = accounting +billing_username = root +billing_password = + +##NGCP REST-API connectivity: +ngcprestapi_uri = https://127.0.0.1:443 +ngcprestapi_username = administrator +ngcprestapi_password = administrator +ngcprestapi_realm = api_admin_http + +##sending email: +emailenable = 0 +erroremailrecipient = +warnemailrecipient = +completionemailrecipient = rkrenn@sipwise.com +successemailrecipient = + +##logging: +fileloglevel = OFF +screenloglevel = INFO +emailloglevel = OFF diff --git a/Projects/Migration/IPGallery/default.cfg b/Projects/Migration/IPGallery/default.cfg deleted file mode 100644 index 50866fb3..00000000 --- a/Projects/Migration/IPGallery/default.cfg +++ /dev/null @@ -1 +0,0 @@ -working_path = /var/sipwise/Migration/IPGallery \ No newline at end of file diff --git a/Projects/Migration/IPGallery/narf.pl b/Projects/Migration/IPGallery/narf.pl index b7b2f031..53f1f045 100644 --- a/Projects/Migration/IPGallery/narf.pl +++ b/Projects/Migration/IPGallery/narf.pl @@ -6,45 +6,62 @@ use File::Basename; use Cwd; use lib Cwd::abs_path(File::Basename::dirname(__FILE__) . '/../../../'); -use Getopt::Long; +use Getopt::Long qw(GetOptions); +use Fcntl qw(LOCK_EX LOCK_NB); -use Globals qw( - $defaultconfig -); +use Globals qw(); use Projects::Migration::IPGallery::Settings qw( $defaultsettings + $defaultconfig update_settings + check_dry + $run_id + $features_define_filename + $dry + $force ); use Logging qw( init_log getlogger $attachmentlogfile + scriptinfo ); use LogError qw ( completion success + scriptwarn + scripterror ); use LoadConfig qw( load_config $SIMPLE_CONFIG_TYPE $YAML_CONFIG_TYPE ); -use Utils qw(getscriptpath); +use Array qw(removeduplicates); +use Utils qw(getscriptpath prompt); use Mail qw(wrap_mailbody $signature $normalpriority $lowpriority $highpriority); -#use ConnectorPool qw(); +use ConnectorPool qw(); +use Projects::Migration::IPGallery::ProjectConnectorPool qw(); use Projects::Migration::IPGallery::Import qw( import_features_define ); -my @MODES = (); +scripterror(getscriptpath() . ' already running',getlogger(getscriptpath())) unless flock DATA, LOCK_EX | LOCK_NB; # not tested on windows yet -if (init() && main()) { +my @TASK_OPTS = (); + +my $tasks = []; +my $import_features_define_task_opt = 'import_features_define'; +push(@TASK_OPTS,$import_features_define_task_opt); + +if (init()) { + main(); exit(0); } else { exit(1); @@ -52,27 +69,27 @@ if (init() && main()) { sub init { - #GetOptions ("host=s" => \$host, - # "port=i" => \$port, - # "file=s" => \$output_filename, - # "dir=s" => \$output_dir, - # "user=s" => \$user, - # "pass=s" => \$pass, - # "period=s" => \$period, - # 'verbose+' => \$verbose) or fatal("Error in command line arguments"); my $configfile = $defaultconfig; my $settingsfile = $defaultsettings; + GetOptions ("config=s" => \$configfile, + "settings=s" => \$settingsfile, + "task=s" => $tasks, + "run=s" => \$run_id, + "dry" => \$dry, + "force" => \$force, + ) or scripterror('error in command line arguments',getlogger(getscriptpath())); + + $tasks = removeduplicates($tasks,1); + my $result = load_config($configfile); init_log(); $result &= load_config($settingsfile,\&update_settings,$SIMPLE_CONFIG_TYPE); - #update_working_path('/var/sipwise'); -#my $logger = getlogger(getscriptpath()); + #$result &= load_config($settingsfile,\&update_settings,$SIMPLE_CONFIG_TYPE); return $result; } - sub main() { my @messages = (); @@ -80,22 +97,38 @@ sub main() { my $result = 0; my $completion = 0; - if (1 or ('xx' eq "mode")) { #$mode) { - $result = import_features_define_task(\@messages); - $completion = 1; + if ('ARRAY' eq ref $tasks and (scalar @$tasks) > 0) { + foreach my $task (@$tasks) { + if (lc($import_features_define_task_opt) eq lc($task)) { + scriptinfo('task: ' . $import_features_define_task_opt,getlogger(getscriptpath())); + $result |= import_features_define_task(\@messages); + } elsif (lc('blah') eq lc($task)) { + scriptinfo('task: ' . 'balh',getlogger(getscriptpath())); + next unless check_dry(); + $result |= import_features_define_task(\@messages); + $completion |= 1; + } else { + $result = 0; + scripterror("unknow task option '" . $task . "', must be one of " . join(', ',@TASK_OPTS),getlogger(getscriptpath())); + last; + } + } } else { - push(@messages,'unknow option yy, must be one of' . @MODES); + $result = 0; + scripterror('at least one task option is required. supported tasks: ' . join(', ',@TASK_OPTS),getlogger(getscriptpath())); } push(@attachmentfiles,$attachmentlogfile); - if ($completion) { - completion(join("\n\n",@messages),\@attachmentfiles,getlogger(getscriptpath())); + if ($result) { + if ($completion) { + completion(join("\n\n",@messages),\@attachmentfiles,getlogger(getscriptpath())); + } else { + success(join("\n\n",@messages),\@attachmentfiles,getlogger(getscriptpath())); + } } else { success(join("\n\n",@messages),\@attachmentfiles,getlogger(getscriptpath())); } - #ConnectorPool::destroy_dbs(); - return $result; } @@ -107,13 +140,24 @@ sub import_features_define_task { my ($messages) = shift; if (import_features_define( - '/home/rkrenn/test/Features_Define.cfg' + $features_define_filename )) { push(@$messages,'sucessfully inserted x records...'); return 1; } else { - push(@$messages,'some error happened'); + push(@$messages,'was not executed'); return 0; } } + +END { + # this should not be required explicitly, but prevents Log4Perl's + # "rootlogger not initialized error upon exit.. + Projects::Migration::IPGallery::ProjectConnectorPool::destroy_dbs(); + ConnectorPool::destroy_dbs(); +} + +__DATA__ +This exists to allow the locking code at the beginning of the file to work. +DO NOT REMOVE THESE LINES! diff --git a/Projects/Migration/IPGallery/settings.cfg b/Projects/Migration/IPGallery/settings.cfg index 7349a498..08228858 100644 --- a/Projects/Migration/IPGallery/settings.cfg +++ b/Projects/Migration/IPGallery/settings.cfg @@ -1 +1,7 @@ -x = y \ No newline at end of file + +features_define_filename = /home/rkrenn/test/Features_Define.cfg + +import_multithreading = 1 +feature_define_import_numofthreads = 10 + +#dry=0 diff --git a/SqlConnectors/SQLiteDB.pm b/SqlConnectors/SQLiteDB.pm index f2733e2d..95ec32f8 100644 --- a/SqlConnectors/SQLiteDB.pm +++ b/SqlConnectors/SQLiteDB.pm @@ -1,609 +1,612 @@ -package SqlConnectors::SQLiteDB; -use strict; - -## no critic - +package SqlConnectors::SQLiteDB; +use strict; + +## no critic + use File::Basename; use Cwd; -use lib Cwd::abs_path(File::Basename::dirname(__FILE__) . '/../'); - -use Globals qw( - $local_db_path - $LongReadLen_limit); -use Logging qw( - getlogger - dbinfo - dbdebug - texttablecreated - temptablecreated - indexcreated - tabletruncated - tabledropped); -use LogError qw( - dberror - fieldnamesdiffer - dbwarn - fileerror - filewarn); - -use DBI 1.608 qw(:sql_types); -use DBD::SQLite 1.29; -use Array qw(arrayeq contains setcontains); - -use Utils qw( - tempfilename - timestampdigits - timestamp); - -use SqlConnectors::SQLiteVarianceAggregate; - -use SqlConnector; - -require Exporter; -our @ISA = qw(Exporter SqlConnector); -our @EXPORT_OK = qw($staticdbfilemode - $timestampdbfilemode - $temporarydbfilemode - $memorydbfilemode - $privatedbfilemode - get_tableidentifier - cleanupdbfiles); - -our $staticdbfilemode = 0; #remains on disk after shutdown -our $timestampdbfilemode = 1; #remains on disk after shutdown -our $temporarydbfilemode = 2; #cleaned on shutdown -our $memorydbfilemode = 3; #never on disk -our $privatedbfilemode = 4; #somewhere on disk, cleaned on shutdown - -my $cachesize = 16384; #40000; -my $pagesize = 2048; #8192; -my $busytimeout = 20000; #msecs - -my $dbextension = '.db'; -my $journalpostfix = '-journal'; - -my $texttable_encoding = 'UTF-8'; # sqlite returns whats inserted... - -$DBD::SQLite::COLLATION{no_accents} = sub { - my ( $a, $b ) = map lc, @_; - tr[àâáäåãçðèêéëìîíïñòôóöõøùûúüý] - [aaaaaacdeeeeiiiinoooooouuuuy] for $a, $b; - $a cmp $b; - }; - -my $LongReadLen = $LongReadLen_limit; #bytes -my $LongTruncOk = 0; - -#my $logger = getlogger(__PACKAGE__); - -my $lock_do_chunk = 1; -my $lock_get_chunk = 1; - -#SQLite transactions are always serializable. - -sub new { - - my $class = shift; - - my $self = SqlConnector->new(@_); - - $self->{filemode} = undef; - $self->{dbfilename} = undef; - - $self->{drh} = DBI->install_driver('SQLite'); - - bless($self,$class); - - dbdebug($self,__PACKAGE__ . ' connector created',getlogger(__PACKAGE__)); - - return $self; - -} - -sub _connectidentifier { - - my $self = shift; - return _get_connectidentifier($self->{filemode},$self->{dbfilename}); - -} - -sub tableidentifier { - - my $self = shift; - my $tablename = shift; - return $tablename; - -} - -sub columnidentifier { - - my $self = shift; - my $columnname = shift; - - return $columnname; - -} - -sub get_tableidentifier { - - my ($tablename,$filemode, $filename) = @_; - my $connectionidentifier = _get_connectidentifier($filemode, $filename); - if (defined $connectionidentifier) { - return $connectionidentifier . '.' . $tablename; - } else { - return $tablename; - } - -} - -sub getsafetablename { - - my $self = shift; - my $tableidentifier = shift; - - return $self->SUPER::getsafetablename($tableidentifier); - -} - -sub _force_numeric_column { - my $self = shift; - my $column = shift; - return 'CAST(' . $column . ' AS REAL)'; -} - -sub getdatabases { - - my $self = shift; - - my $rdbextension = quotemeta($dbextension); - my $ucrdbextension = quotemeta(uc($dbextension)); - #my $rjournalpostfix = quotemeta($journalpostfix); - local *DBDIR; - if (not opendir(DBDIR, $local_db_path)) { - fileerror('cannot opendir ' . $local_db_path . ': ' . $!,getlogger(__PACKAGE__)); - return []; - } - my @files = grep { /($rdbextension|$ucrdbextension)$/ && -f $local_db_path . $_ } readdir(DBDIR); - closedir DBDIR; - my @databases = (); - foreach my $file (@files) { - #print $file; - my $databasename = $file; - $databasename =~ s/($rdbextension|$ucrdbextension)$//g; - push @databases,$databasename; - } - return \@databases; - -} - -sub _createdatabase { - - my $self = shift; - my ($filename) = @_; - my $dbfilename = _getdbfilename($self->{filemode},$filename); - - if ($self->_is_filebased() and not -e $dbfilename) { - my $dbh = DBI->connect( - 'dbi:SQLite:dbname=' . $dbfilename, '', '', - { - PrintError => 0, - RaiseError => 0, - #sqlite_unicode => 1, latin 1 chars - #AutoCommit => 0, - } - ) or dberror($self,'error connecting: ' . $self->{drh}->errstr(),getlogger(__PACKAGE__)); - $dbh->disconnect() or dbwarn($self,'error disconnecting: ' . $dbh->errstr(),getlogger(__PACKAGE__)); - dbinfo($self,'database \'' . $dbfilename . '\' created',getlogger(__PACKAGE__)); - } - - return $dbfilename; - -} - -sub db_connect { - - my $self = shift; - my ($filemode, $filename) = @_; - - $self->SUPER::db_connect($filemode, $filename); - - #if (defined $self->{dbh}) { - # $self->db_disconnect(); - #} - - $self->{filemode} = $filemode; - $self->{dbfilename} = $self->_createdatabase($filename); - - - my $dbh = DBI->connect( - 'dbi:SQLite:dbname=' . $self->{dbfilename}, '', '', - { - PrintError => 0, - RaiseError => 0, - #sqlite_unicode => 1, latin 1 chars - #AutoCommit => 0, - } - ) or dberror($self,'error connecting: ' . $self->{drh}->errstr(),getlogger(__PACKAGE__)); - #or sqlitedberror($dbfilename,'error connecting to sqlite db',getlogger(__PACKAGE__)); - - $dbh->{InactiveDestroy} = 1; - - $dbh->{LongReadLen} = $LongReadLen; - $dbh->{LongTruncOk} = $LongTruncOk; - - $dbh->{AutoCommit} = 1; - # we use a mysql style - $dbh->sqlite_create_function('now', 0, \×tamp ); - $dbh->sqlite_create_function('concat', 2, \&_concat ); - #$dbh->sqlite_create_function(float_equal ?? - $dbh->sqlite_create_aggregate( 'variance', 1, 'SQLiteVarianceAggregate' ); - - $dbh->sqlite_busy_timeout($busytimeout); - - $self->{dbh} = $dbh; - - #SQLite transactions are always serializable. - - $self->db_do('PRAGMA foreign_keys = OFF'); - #$self->db_do('PRAGMA default_synchronous = OFF'); - $self->db_do('PRAGMA synchronous = OFF'); - $self->db_do('PRAGMA page_size = ' . $pagesize); - $self->db_do('PRAGMA cache_size = ' . $cachesize); - #$self->db_do('PRAGMA encoding = "UTF-8"'); # only new databases! - $self->db_do('PRAGMA encoding = "' . $texttable_encoding . '"'); # only new databases! - #PRAGMA locking_mode = NORMAL ... by default - #$self->db_do('PRAGMA auto_vacuum = INCREMENTAL'); - - dbinfo($self,'connected',getlogger(__PACKAGE__)); - -} - -sub _concat { - - return $_[0] . $_[1]; - -} - -sub vacuum { - - my $self = shift; - my $tablename = shift; - - $self->db_finish(); - - if (defined $self->{dbh}) { - if ($self->{filemode} == $staticdbfilemode or $self->{filemode} == $timestampdbfilemode) { - $self->db_do('VACUUM'); # or sqlitedberror($self,"failed to VACUUM\nDBI error:\n" . $self->{dbh}->errstr(),getlogger(__PACKAGE__)); - dbinfo($self,'VACUUMed',getlogger(__PACKAGE__)); - } - } - -} - -sub _db_disconnect { - - my $self = shift; - - $self->SUPER::_db_disconnect(); - - if ($self->{filemode} == $temporarydbfilemode and defined $self->{dbfilename} and -e $self->{dbfilename}) { - if ((unlink $self->{dbfilename}) > 0) { - dbinfo($self,'db file removed',getlogger(__PACKAGE__)); - } else { - dbwarn($self,'cannot remove db file: ' . $!,getlogger(__PACKAGE__)); - } - my $journalfilename = $self->{dbfilename} . '-journal'; - if (-e $journalfilename) { - if ((unlink $journalfilename) > 0) { - dbinfo($self,'journal file removed',getlogger(__PACKAGE__)); - } else { - dbwarn($self,'cannot remove journal file: ' . $!,getlogger(__PACKAGE__)); - } - } - } - -} - - -sub cleanupdbfiles { - - my (@remainingdbfilenames) = @_; - my $rdbextension = quotemeta($dbextension); - my $ucrdbextension = quotemeta(uc($dbextension)); - my $rjournalpostfix = quotemeta($journalpostfix); - local *DBDIR; - if (not opendir(DBDIR, $local_db_path)) { - fileerror('cannot opendir ' . $local_db_path . ': ' . $!,getlogger(__PACKAGE__)); - return; - } - my @files = grep { /($rdbextension|$ucrdbextension)($rjournalpostfix)?$/ && -f $local_db_path . $_ } readdir(DBDIR); - closedir DBDIR; - my @remainingdbfiles = (); - foreach my $filename (@remainingdbfilenames) { - push @remainingdbfiles,$local_db_path . $filename . $dbextension; - push @remainingdbfiles,$local_db_path . $filename . $dbextension . $journalpostfix; - push @remainingdbfiles,$local_db_path . uc($filename . $dbextension) . $journalpostfix; - } - foreach my $file (@files) { - #print $file; - my $filepath = $local_db_path . $file; - if (not contains($filepath,\@remainingdbfiles)) { - if ((unlink $filepath) == 0) { - filewarn('cannot remove ' . $filepath . ': ' . $!,getlogger(__PACKAGE__)); - } - } - } - -} - -sub getfieldnames { - - my $self = shift; - my $tablename = shift; - my @fieldnames = keys %{$self->db_get_all_hashref('PRAGMA table_info(' . $tablename . ')','name')}; - return \@fieldnames; - -} - -sub getprimarykeycols { - - my $self = shift; - my $tablename = shift; - #return $self->db_get_col('SHOW FIELDS FROM ' . $tablename); - my $fieldinfo = $self->db_get_all_hashref('PRAGMA table_info(' . $tablename . ')','name'); - my @keycols = (); - foreach my $fieldname (keys %$fieldinfo) { - if ($fieldinfo->{$fieldname}->{'pk'}) { - push @keycols,$fieldname; - } - } - return \@keycols; - -} - -sub create_primarykey { - - my $self = shift; - my ($tablename,$keycols,$fieldnames) = @_; - - #not supported by sqlite - - return 0; -} - -sub create_indexes { - my $self = shift; - my ($tablename,$indexes,$keycols) = @_; - - my $index_count = 0; - if (length($tablename) > 0) { - - if (defined $indexes and ref $indexes eq 'HASH' and scalar keys %$indexes > 0) { - foreach my $indexname (keys %$indexes) { - my $indexcols = $self->_extract_indexcols($indexes->{$indexname}); - if (not arrayeq($indexcols,$keycols,1)) { - #$statement .= ', INDEX ' . $indexname . ' (' . join(', ',@{$indexes->{$indexname}}) . ')'; - $self->db_do('CREATE INDEX ' . $indexname . ' ON ' . $self->tableidentifier($tablename) . ' (' . join(', ',map { local $_ = $_; $_ = $self->columnidentifier($_); $_; } @$indexcols) . ')'); - indexcreated($self,$tablename,$indexname,getlogger(__PACKAGE__)); - } - } - } - - } - - return $index_count; -} - -sub create_temptable { - - my $self = shift; - my $select_stmt = shift; - my $indexes = shift; - - my $index_tablename = $self->_gettemptablename(); - my $temp_tablename = $self->tableidentifier($index_tablename); - - $self->db_do('CREATE TEMPORARY TABLE ' . $temp_tablename . ' AS ' . $select_stmt); - #push(@{$self->{temp_tables}},$temp_tablename); - temptablecreated($self,$index_tablename,getlogger(__PACKAGE__)); - - #$self->{temp_table_count} += 1; - - if (defined $indexes and ref $indexes eq 'HASH' and scalar keys %$indexes > 0) { - foreach my $indexname (keys %$indexes) { - my $indexcols = $self->_extract_indexcols($indexes->{$indexname}); - #if (not arrayeq($indexcols,$keycols,1)) { - #$statement .= ', INDEX ' . $indexname . ' (' . join(', ',@{$indexes->{$indexname}}) . ')'; - $indexname = lc($index_tablename) . '_' . $indexname; - $self->db_do('CREATE INDEX ' . $indexname . ' ON ' . $temp_tablename . ' (' . join(', ',map { local $_ = $_; $_ = $self->columnidentifier($_); $_; } @$indexcols) . ')'); - indexcreated($self,$index_tablename,$indexname,getlogger(__PACKAGE__)); - #} - } - } - - return $temp_tablename; - -} - -sub create_texttable { - - my $self = shift; - my ($tablename,$fieldnames,$keycols,$indexes,$truncate,$defer_indexes) = @_; - #my ($tableidentifier,$fieldnames,$keycols,$indexes,$truncate) = @_; - - #my $tablename = $self->getsafetablename($tableidentifier); - - if (length($tablename) > 0 and defined $fieldnames and ref $fieldnames eq 'ARRAY') { - - my $created = 0; - if ($self->table_exists($tablename) == 0) { - my $statement = 'CREATE TABLE ' . $self->tableidentifier($tablename) . ' ('; - $statement .= join(' TEXT, ',map { local $_ = $_; $_ = $self->columnidentifier($_); $_; } @$fieldnames) . ' TEXT'; # sqlite_unicode off... outcoming strings not marked utf8 - #$statement .= join(' BLOB, ',@$fieldnames) . ' BLOB'; #to maintain source char encoding when inserting? - #if (not $defer_indexes and defined $keycols and ref $keycols eq 'ARRAY' and scalar @$keycols > 0 and setcontains($keycols,$fieldnames,1)) { - if (defined $keycols and ref $keycols eq 'ARRAY' and scalar @$keycols > 0 and setcontains($keycols,$fieldnames,1)) { - $statement .= ', PRIMARY KEY (' . join(', ',map { local $_ = $_; $_ = $self->columnidentifier($_); $_; } @$keycols) . ')'; - } - $statement .= ')'; - - $self->db_do($statement); - texttablecreated($self,$tablename,getlogger(__PACKAGE__)); - - if (not $defer_indexes and defined $indexes and ref $indexes eq 'HASH' and scalar keys %$indexes > 0) { - foreach my $indexname (keys %$indexes) { - my $indexcols = $self->_extract_indexcols($indexes->{$indexname}); - if (not arrayeq($indexcols,$keycols,1)) { - #$statement .= ', INDEX ' . $indexname . ' (' . join(', ',@{$indexes->{$indexname}}) . ')'; - $self->db_do('CREATE INDEX ' . $indexname . ' ON ' . $self->tableidentifier($tablename) . ' (' . join(', ',map { local $_ = $_; $_ = $self->columnidentifier($_); $_; } @$indexcols) . ')'); - indexcreated($self,$tablename,$indexname,getlogger(__PACKAGE__)); - } - } - } - $created = 1; - } else { - my $fieldnamesfound = $self->getfieldnames($tablename); - if (not setcontains($fieldnames,$fieldnamesfound,1)) { - fieldnamesdiffer($self,$tablename,$fieldnames,$fieldnamesfound,getlogger(__PACKAGE__)); - return 0; - } - } - - if (not $created and $truncate) { - $self->truncate_table($tablename); - } - return 1; - } else { - return 0; - } - - #return $tablename; - -} - -sub multithreading_supported { - - my $self = shift; - return 1; - -} - -sub truncate_table { - - my $self = shift; - my $tablename = shift; - - $self->db_do('DELETE FROM ' . $self->tableidentifier($tablename)); - #$self->db_do('VACUUM'); - tabletruncated($self,$tablename,getlogger(__PACKAGE__)); - -} - -sub table_exists { - - my $self = shift; - my $tablename = shift; - - return $self->db_get_value('SELECT COUNT(*) FROM sqlite_master WHERE type = \'table\' AND name = ?',$tablename); - -} - -sub drop_table { - - my $self = shift; - my $tablename = shift; - - if ($self->table_exists($tablename) > 0) { - $self->db_do('DROP TABLE ' . $self->tableidentifier($tablename)); - - #my $indexes = $self->db_get_col('SELECT name FROM sqlite_master WHERE type = \'index\' AND tbl_name = ?',$tablename); - #foreach my $indexname (@$indexes) { - # $self->db_do('DROP INDEX IF EXISTS ' . $indexname); - #} - - - #$self->db_do('VACUUM'); - tabledropped($self,$tablename,getlogger(__PACKAGE__)); - return 1; - } - return 0; - -} - -sub _get_connectidentifier { - - my ($filemode, $filename) = @_; - if ($filemode == $staticdbfilemode and defined $filename) { - return $filename; - } elsif ($filemode == $timestampdbfilemode) { - return $filename; - } elsif ($filemode == $temporarydbfilemode) { - return $filename; - } elsif ($filemode == $memorydbfilemode) { - return ''; - } elsif ($filemode == $privatedbfilemode) { - return ''; - } else { - return undef; - } - -} - -sub _getdbfilename { - - my ($filemode,$filename) = @_; - if ($filemode == $staticdbfilemode and defined $filename) { - return $local_db_path . $filename . $dbextension; - } elsif ($filemode == $timestampdbfilemode) { - return $local_db_path . timestampdigits() . $dbextension; - } elsif ($filemode == $temporarydbfilemode) { - return tempfilename('XXXX',$local_db_path,$dbextension); - } elsif ($filemode == $memorydbfilemode) { - return ':memory:'; - } elsif ($filemode == $privatedbfilemode) { - return ''; - } - -} - -sub _is_filebased { - - my $self = shift; - if ($self->{filemode} == $staticdbfilemode or $self->{filemode} == $timestampdbfilemode or $self->{filemode} == $temporarydbfilemode) { - return 1; - } else { - return 0; - } - -} - -sub db_do_begin { - - my $self = shift; - my $query = shift; - my $tablename = shift; - - $self->SUPER::db_do_begin($query,$tablename,$lock_do_chunk,@_); - -} - -sub db_get_begin { - - my $self = shift; - my $query = shift; - my $tablename = shift; - #my $lock = shift; - - $self->SUPER::db_get_begin($query,$tablename,$lock_get_chunk,@_); - -} - -sub db_finish { - - my $self = shift; - #my $unlock = shift; - - $self->SUPER::db_finish($lock_do_chunk | $lock_get_chunk); - -} - -1; \ No newline at end of file +use lib Cwd::abs_path(File::Basename::dirname(__FILE__) . '/../'); + +use Globals qw( + $local_db_path + $LongReadLen_limit); +use Logging qw( + getlogger + dbinfo + dbdebug + texttablecreated + temptablecreated + indexcreated + tabletruncated + tabledropped); +use LogError qw( + dberror + fieldnamesdiffer + dbwarn + fileerror + filewarn); + +use DBI 1.608 qw(:sql_types); +use DBD::SQLite 1.29; +use Array qw(arrayeq contains setcontains); + +use Utils qw( + tempfilename + timestampdigits + timestamp); + +use SqlConnectors::SQLiteVarianceAggregate; + +use SqlConnector; + +require Exporter; +our @ISA = qw(Exporter SqlConnector); +our @EXPORT_OK = qw($staticdbfilemode + $timestampdbfilemode + $temporarydbfilemode + $memorydbfilemode + $privatedbfilemode + get_tableidentifier + cleanupdbfiles); + +our $staticdbfilemode = 0; #remains on disk after shutdown +our $timestampdbfilemode = 1; #remains on disk after shutdown +our $temporarydbfilemode = 2; #cleaned on shutdown +our $memorydbfilemode = 3; #never on disk +our $privatedbfilemode = 4; #somewhere on disk, cleaned on shutdown + +my $cachesize = 16384; #40000; +my $pagesize = 2048; #8192; +my $busytimeout = 60000; #20000; #msecs + +my $dbextension = '.db'; +my $journalpostfix = '-journal'; + +my $texttable_encoding = 'UTF-8'; # sqlite returns whats inserted... + +$DBD::SQLite::COLLATION{no_accents} = sub { + my ( $a, $b ) = map lc, @_; + tr[àâáäåãçðèêéëìîíïñòôóöõøùûúüý] + [aaaaaacdeeeeiiiinoooooouuuuy] for $a, $b; + $a cmp $b; + }; + +my $LongReadLen = $LongReadLen_limit; #bytes +my $LongTruncOk = 0; + +#my $logger = getlogger(__PACKAGE__); + +my $lock_do_chunk = 1; +my $lock_get_chunk = 1; + +#SQLite transactions are always serializable. + +#my $read_uncommitted_isolation_level = 1; + +sub new { + + my $class = shift; + + my $self = SqlConnector->new(@_); + + $self->{filemode} = undef; + $self->{dbfilename} = undef; + + $self->{drh} = DBI->install_driver('SQLite'); + + bless($self,$class); + + dbdebug($self,__PACKAGE__ . ' connector created',getlogger(__PACKAGE__)); + + return $self; + +} + +sub _connectidentifier { + + my $self = shift; + return _get_connectidentifier($self->{filemode},$self->{dbfilename}); + +} + +sub tableidentifier { + + my $self = shift; + my $tablename = shift; + return $tablename; + +} + +sub columnidentifier { + + my $self = shift; + my $columnname = shift; + + return $columnname; + +} + +sub get_tableidentifier { + + my ($tablename,$filemode, $filename) = @_; + my $connectionidentifier = _get_connectidentifier($filemode, $filename); + if (defined $connectionidentifier) { + return $connectionidentifier . '.' . $tablename; + } else { + return $tablename; + } + +} + +sub getsafetablename { + + my $self = shift; + my $tableidentifier = shift; + + return $self->SUPER::getsafetablename($tableidentifier); + +} + +sub _force_numeric_column { + my $self = shift; + my $column = shift; + return 'CAST(' . $column . ' AS REAL)'; +} + +sub getdatabases { + + my $self = shift; + + my $rdbextension = quotemeta($dbextension); + my $ucrdbextension = quotemeta(uc($dbextension)); + #my $rjournalpostfix = quotemeta($journalpostfix); + local *DBDIR; + if (not opendir(DBDIR, $local_db_path)) { + fileerror('cannot opendir ' . $local_db_path . ': ' . $!,getlogger(__PACKAGE__)); + return []; + } + my @files = grep { /($rdbextension|$ucrdbextension)$/ && -f $local_db_path . $_ } readdir(DBDIR); + closedir DBDIR; + my @databases = (); + foreach my $file (@files) { + #print $file; + my $databasename = $file; + $databasename =~ s/($rdbextension|$ucrdbextension)$//g; + push @databases,$databasename; + } + return \@databases; + +} + +sub _createdatabase { + + my $self = shift; + my ($filename) = @_; + my $dbfilename = _getdbfilename($self->{filemode},$filename); + + if ($self->_is_filebased() and not -e $dbfilename) { + my $dbh = DBI->connect( + 'dbi:SQLite:dbname=' . $dbfilename, '', '', + { + PrintError => 0, + RaiseError => 0, + #sqlite_unicode => 1, latin 1 chars + #AutoCommit => 0, + } + ) or dberror($self,'error connecting: ' . $self->{drh}->errstr(),getlogger(__PACKAGE__)); + $dbh->disconnect() or dbwarn($self,'error disconnecting: ' . $dbh->errstr(),getlogger(__PACKAGE__)); + dbinfo($self,'database \'' . $dbfilename . '\' created',getlogger(__PACKAGE__)); + } + + return $dbfilename; + +} + +sub db_connect { + + my $self = shift; + my ($filemode, $filename) = @_; + + $self->SUPER::db_connect($filemode, $filename); + + #if (defined $self->{dbh}) { + # $self->db_disconnect(); + #} + + $self->{filemode} = $filemode; + $self->{dbfilename} = $self->_createdatabase($filename); + + + my $dbh = DBI->connect( + 'dbi:SQLite:dbname=' . $self->{dbfilename}, '', '', + { + PrintError => 0, + RaiseError => 0, + #sqlite_unicode => 1, latin 1 chars + #AutoCommit => 0, + } + ) or dberror($self,'error connecting: ' . $self->{drh}->errstr(),getlogger(__PACKAGE__)); + #or sqlitedberror($dbfilename,'error connecting to sqlite db',getlogger(__PACKAGE__)); + + $dbh->{InactiveDestroy} = 1; + + $dbh->{LongReadLen} = $LongReadLen; + $dbh->{LongTruncOk} = $LongTruncOk; + + $dbh->{AutoCommit} = 1; + # we use a mysql style + $dbh->sqlite_create_function('now', 0, \×tamp ); + $dbh->sqlite_create_function('concat', 2, \&_concat ); + #$dbh->sqlite_create_function(float_equal ?? + $dbh->sqlite_create_aggregate( 'variance', 1, 'SQLiteVarianceAggregate' ); + + $dbh->sqlite_busy_timeout($busytimeout); + + $self->{dbh} = $dbh; + + #SQLite transactions are always serializable. + + $self->db_do('PRAGMA foreign_keys = OFF'); + #$self->db_do('PRAGMA default_synchronous = OFF'); + $self->db_do('PRAGMA synchronous = OFF'); + $self->db_do('PRAGMA page_size = ' . $pagesize); + $self->db_do('PRAGMA cache_size = ' . $cachesize); + #$self->db_do('PRAGMA encoding = "UTF-8"'); # only new databases! + $self->db_do('PRAGMA encoding = "' . $texttable_encoding . '"'); # only new databases! + #PRAGMA locking_mode = NORMAL ... by default + #$self->db_do('PRAGMA auto_vacuum = INCREMENTAL'); + #$self->db_do('PRAGMA read_uncommitted = ' . $read_uncommitted_isolation_level); + + dbinfo($self,'connected',getlogger(__PACKAGE__)); + +} + +sub _concat { + + return $_[0] . $_[1]; + +} + +sub vacuum { + + my $self = shift; + my $tablename = shift; + + $self->db_finish(); + + if (defined $self->{dbh}) { + if ($self->{filemode} == $staticdbfilemode or $self->{filemode} == $timestampdbfilemode) { + $self->db_do('VACUUM'); # or sqlitedberror($self,"failed to VACUUM\nDBI error:\n" . $self->{dbh}->errstr(),getlogger(__PACKAGE__)); + dbinfo($self,'VACUUMed',getlogger(__PACKAGE__)); + } + } + +} + +sub _db_disconnect { + + my $self = shift; + + $self->SUPER::_db_disconnect(); + + if ($self->{filemode} == $temporarydbfilemode and defined $self->{dbfilename} and -e $self->{dbfilename}) { + if ((unlink $self->{dbfilename}) > 0) { + dbinfo($self,'db file removed',getlogger(__PACKAGE__)); + } else { + dbwarn($self,'cannot remove db file: ' . $!,getlogger(__PACKAGE__)); + } + my $journalfilename = $self->{dbfilename} . '-journal'; + if (-e $journalfilename) { + if ((unlink $journalfilename) > 0) { + dbinfo($self,'journal file removed',getlogger(__PACKAGE__)); + } else { + dbwarn($self,'cannot remove journal file: ' . $!,getlogger(__PACKAGE__)); + } + } + } + +} + + +sub cleanupdbfiles { + + my (@remainingdbfilenames) = @_; + my $rdbextension = quotemeta($dbextension); + my $ucrdbextension = quotemeta(uc($dbextension)); + my $rjournalpostfix = quotemeta($journalpostfix); + local *DBDIR; + if (not opendir(DBDIR, $local_db_path)) { + fileerror('cannot opendir ' . $local_db_path . ': ' . $!,getlogger(__PACKAGE__)); + return; + } + my @files = grep { /($rdbextension|$ucrdbextension)($rjournalpostfix)?$/ && -f $local_db_path . $_ } readdir(DBDIR); + closedir DBDIR; + my @remainingdbfiles = (); + foreach my $filename (@remainingdbfilenames) { + push @remainingdbfiles,$local_db_path . $filename . $dbextension; + push @remainingdbfiles,$local_db_path . $filename . $dbextension . $journalpostfix; + push @remainingdbfiles,$local_db_path . uc($filename . $dbextension) . $journalpostfix; + } + foreach my $file (@files) { + #print $file; + my $filepath = $local_db_path . $file; + if (not contains($filepath,\@remainingdbfiles)) { + if ((unlink $filepath) == 0) { + filewarn('cannot remove ' . $filepath . ': ' . $!,getlogger(__PACKAGE__)); + } + } + } + +} + +sub getfieldnames { + + my $self = shift; + my $tablename = shift; + my @fieldnames = keys %{$self->db_get_all_hashref('PRAGMA table_info(' . $tablename . ')','name')}; + return \@fieldnames; + +} + +sub getprimarykeycols { + + my $self = shift; + my $tablename = shift; + #return $self->db_get_col('SHOW FIELDS FROM ' . $tablename); + my $fieldinfo = $self->db_get_all_hashref('PRAGMA table_info(' . $tablename . ')','name'); + my @keycols = (); + foreach my $fieldname (keys %$fieldinfo) { + if ($fieldinfo->{$fieldname}->{'pk'}) { + push @keycols,$fieldname; + } + } + return \@keycols; + +} + +sub create_primarykey { + + my $self = shift; + my ($tablename,$keycols,$fieldnames) = @_; + + #not supported by sqlite + + return 0; +} + +sub create_indexes { + my $self = shift; + my ($tablename,$indexes,$keycols) = @_; + + my $index_count = 0; + if (length($tablename) > 0) { + + if (defined $indexes and ref $indexes eq 'HASH' and scalar keys %$indexes > 0) { + foreach my $indexname (keys %$indexes) { + my $indexcols = $self->_extract_indexcols($indexes->{$indexname}); + if (not arrayeq($indexcols,$keycols,1)) { + #$statement .= ', INDEX ' . $indexname . ' (' . join(', ',@{$indexes->{$indexname}}) . ')'; + $self->db_do('CREATE INDEX ' . $indexname . ' ON ' . $self->tableidentifier($tablename) . ' (' . join(', ',map { local $_ = $_; $_ = $self->columnidentifier($_); $_; } @$indexcols) . ')'); + indexcreated($self,$tablename,$indexname,getlogger(__PACKAGE__)); + } + } + } + + } + + return $index_count; +} + +sub create_temptable { + + my $self = shift; + my $select_stmt = shift; + my $indexes = shift; + + my $index_tablename = $self->_gettemptablename(); + my $temp_tablename = $self->tableidentifier($index_tablename); + + $self->db_do('CREATE TEMPORARY TABLE ' . $temp_tablename . ' AS ' . $select_stmt); + #push(@{$self->{temp_tables}},$temp_tablename); + temptablecreated($self,$index_tablename,getlogger(__PACKAGE__)); + + #$self->{temp_table_count} += 1; + + if (defined $indexes and ref $indexes eq 'HASH' and scalar keys %$indexes > 0) { + foreach my $indexname (keys %$indexes) { + my $indexcols = $self->_extract_indexcols($indexes->{$indexname}); + #if (not arrayeq($indexcols,$keycols,1)) { + #$statement .= ', INDEX ' . $indexname . ' (' . join(', ',@{$indexes->{$indexname}}) . ')'; + $indexname = lc($index_tablename) . '_' . $indexname; + $self->db_do('CREATE INDEX ' . $indexname . ' ON ' . $temp_tablename . ' (' . join(', ',map { local $_ = $_; $_ = $self->columnidentifier($_); $_; } @$indexcols) . ')'); + indexcreated($self,$index_tablename,$indexname,getlogger(__PACKAGE__)); + #} + } + } + + return $temp_tablename; + +} + +sub create_texttable { + + my $self = shift; + my ($tablename,$fieldnames,$keycols,$indexes,$truncate,$defer_indexes) = @_; + #my ($tableidentifier,$fieldnames,$keycols,$indexes,$truncate) = @_; + + #my $tablename = $self->getsafetablename($tableidentifier); + + if (length($tablename) > 0 and defined $fieldnames and ref $fieldnames eq 'ARRAY') { + + my $created = 0; + if ($self->table_exists($tablename) == 0) { + my $statement = 'CREATE TABLE ' . $self->tableidentifier($tablename) . ' ('; + $statement .= join(' TEXT, ',map { local $_ = $_; $_ = $self->columnidentifier($_); $_; } @$fieldnames) . ' TEXT'; # sqlite_unicode off... outcoming strings not marked utf8 + #$statement .= join(' BLOB, ',@$fieldnames) . ' BLOB'; #to maintain source char encoding when inserting? + #if (not $defer_indexes and defined $keycols and ref $keycols eq 'ARRAY' and scalar @$keycols > 0 and setcontains($keycols,$fieldnames,1)) { + if (defined $keycols and ref $keycols eq 'ARRAY' and scalar @$keycols > 0 and setcontains($keycols,$fieldnames,1)) { + $statement .= ', PRIMARY KEY (' . join(', ',map { local $_ = $_; $_ = $self->columnidentifier($_); $_; } @$keycols) . ')'; + } + $statement .= ')'; + + $self->db_do($statement); + texttablecreated($self,$tablename,getlogger(__PACKAGE__)); + + if (not $defer_indexes and defined $indexes and ref $indexes eq 'HASH' and scalar keys %$indexes > 0) { + foreach my $indexname (keys %$indexes) { + my $indexcols = $self->_extract_indexcols($indexes->{$indexname}); + if (not arrayeq($indexcols,$keycols,1)) { + #$statement .= ', INDEX ' . $indexname . ' (' . join(', ',@{$indexes->{$indexname}}) . ')'; + $self->db_do('CREATE INDEX ' . $indexname . ' ON ' . $self->tableidentifier($tablename) . ' (' . join(', ',map { local $_ = $_; $_ = $self->columnidentifier($_); $_; } @$indexcols) . ')'); + indexcreated($self,$tablename,$indexname,getlogger(__PACKAGE__)); + } + } + } + $created = 1; + } else { + my $fieldnamesfound = $self->getfieldnames($tablename); + if (not setcontains($fieldnames,$fieldnamesfound,1)) { + fieldnamesdiffer($self,$tablename,$fieldnames,$fieldnamesfound,getlogger(__PACKAGE__)); + return 0; + } + } + + if (not $created and $truncate) { + $self->truncate_table($tablename); + } + return 1; + } else { + return 0; + } + + #return $tablename; + +} + +sub multithreading_supported { + + my $self = shift; + return 1; + +} + +sub truncate_table { + + my $self = shift; + my $tablename = shift; + + $self->db_do('DELETE FROM ' . $self->tableidentifier($tablename)); + #$self->db_do('VACUUM'); + tabletruncated($self,$tablename,getlogger(__PACKAGE__)); + +} + +sub table_exists { + + my $self = shift; + my $tablename = shift; + + return $self->db_get_value('SELECT COUNT(*) FROM sqlite_master WHERE type = \'table\' AND name = ?',$tablename); + +} + +sub drop_table { + + my $self = shift; + my $tablename = shift; + + if ($self->table_exists($tablename) > 0) { + $self->db_do('DROP TABLE ' . $self->tableidentifier($tablename)); + + #my $indexes = $self->db_get_col('SELECT name FROM sqlite_master WHERE type = \'index\' AND tbl_name = ?',$tablename); + #foreach my $indexname (@$indexes) { + # $self->db_do('DROP INDEX IF EXISTS ' . $indexname); + #} + + + #$self->db_do('VACUUM'); + tabledropped($self,$tablename,getlogger(__PACKAGE__)); + return 1; + } + return 0; + +} + +sub _get_connectidentifier { + + my ($filemode, $filename) = @_; + if ($filemode == $staticdbfilemode and defined $filename) { + return $filename; + } elsif ($filemode == $timestampdbfilemode) { + return $filename; + } elsif ($filemode == $temporarydbfilemode) { + return $filename; + } elsif ($filemode == $memorydbfilemode) { + return ''; + } elsif ($filemode == $privatedbfilemode) { + return ''; + } else { + return undef; + } + +} + +sub _getdbfilename { + + my ($filemode,$filename) = @_; + if ($filemode == $staticdbfilemode and defined $filename) { + return $local_db_path . $filename . $dbextension; + } elsif ($filemode == $timestampdbfilemode) { + return $local_db_path . timestampdigits() . $dbextension; + } elsif ($filemode == $temporarydbfilemode) { + return tempfilename('XXXX',$local_db_path,$dbextension); + } elsif ($filemode == $memorydbfilemode) { + return ':memory:'; + } elsif ($filemode == $privatedbfilemode) { + return ''; + } + +} + +sub _is_filebased { + + my $self = shift; + if ($self->{filemode} == $staticdbfilemode or $self->{filemode} == $timestampdbfilemode or $self->{filemode} == $temporarydbfilemode) { + return 1; + } else { + return 0; + } + +} + +sub db_do_begin { + + my $self = shift; + my $query = shift; + my $tablename = shift; + + $self->SUPER::db_do_begin($query,$tablename,$lock_do_chunk,@_); + +} + +sub db_get_begin { + + my $self = shift; + my $query = shift; + my $tablename = shift; + #my $lock = shift; + + $self->SUPER::db_get_begin($query,$tablename,$lock_get_chunk,@_); + +} + +sub db_finish { + + my $self = shift; + #my $unlock = shift; + + $self->SUPER::db_finish($lock_do_chunk | $lock_get_chunk); + +} + +1; diff --git a/SqlRecord.pm b/SqlRecord.pm index dab93c70..49888820 100644 --- a/SqlRecord.pm +++ b/SqlRecord.pm @@ -67,7 +67,7 @@ use Globals qw( $enablemultithreading $cpucount $cells_transfer_memory_limit -$defer_indexes); +$transfer_defer_indexes); #use Terminate qw(setsigkill); @@ -86,6 +86,8 @@ our @EXPORT_OK = qw( insert_record update_record delete_records + + insert_stmt ); my $table_expected_fieldnames = {}; @@ -725,6 +727,19 @@ sub transfer_records { } +sub insert_stmt { + + my ($get_db,$tablename) = @_; + my $db = (ref $get_db eq 'CODE') ? &$get_db() : $get_db; + my $connectidentifier = $db->connectidentifier(); + my $tid = threadid(); + my $expected_fieldnames = $table_expected_fieldnames->{$tid}->{$connectidentifier}->{$tablename}; + return 'INSERT INTO ' . $db->tableidentifier($tablename) . ' (' . + join(', ',map { local $_ = $_; $_ = $db->columnidentifier($_); $_; } @$expected_fieldnames) . + ') VALUES (' . substr(',?' x scalar @$expected_fieldnames,1) . ')'; + +} + sub transfer_table { my ($get_db,$tablename,$get_target_db,$targettablename,$truncate_targettable,$create_indexes,$texttable_engine,$fixtable_statements,$selectcount,$select,@values) = @_; @@ -754,7 +769,7 @@ sub transfer_table { my $errorstate = $RUNNING; # 1; - $create_indexes = ((defined $create_indexes) ? $create_indexes : $defer_indexes); + $create_indexes = ((defined $create_indexes) ? $create_indexes : $transfer_defer_indexes); if (create_targettable($db,$tablename,$target_db,$targettablename,$truncate_targettable,$create_indexes,$texttable_engine)) { diff --git a/Utils.pm b/Utils.pm index 37cb8e8b..dcf3f5fe 100644 --- a/Utils.pm +++ b/Utils.pm @@ -37,7 +37,8 @@ use File::Path qw(remove_tree make_path); #use Sys::Info::Constants qw( :device_cpu ); # after all, the only reliable way to get the true vCPU count: -use Sys::CpuAffinity; # qw(getNumCpus); not exported? +#use Sys::CpuAffinity; # qw(getNumCpus); not exported? +#disabling for now, no debian package yet. require Exporter; our @ISA = qw(Exporter); @@ -98,6 +99,7 @@ our @EXPORT_OK = qw( $chmod_umask + prompt ); our $chmod_umask = 0644; @@ -542,9 +544,9 @@ sub kbytes2gigs { sub cleanupdir { - my ($dirpath,$keeproot,$cleanupinfocode,$filewarncode,$logger) = @_; + my ($dirpath,$keeproot,$scriptinfocode,$filewarncode,$logger) = @_; if (-d $dirpath) { - remove_tree($dirpath, { + remove_tree($dirpath, { keep_root => $keeproot, error => \my $err }); if (@$err) { @@ -556,11 +558,11 @@ sub cleanupdir { } else { &$filewarncode("problem unlinking $file: $message",$logger); } - } + } } } else { - if (defined $cleanupinfocode and ref $cleanupinfocode eq 'CODE') { - &$cleanupinfocode($dirpath . ' removed',$logger); + if (defined $scriptinfocode and ref $scriptinfocode eq 'CODE') { + &$scriptinfocode($dirpath . ' removed',$logger); } } #if ($restoredir) { @@ -580,7 +582,7 @@ sub makepath { my ($dirpath,$fileerrorcode,$logger) = @_; #print $chmod_umask ."\n"; #changemod($dirpath); - make_path($dirpath,{ + make_path($dirpath,{ 'chmod' => $chmod_umask, 'error' => \my $err }); if (@$err) { @@ -592,7 +594,7 @@ sub makepath { } else { &$fileerrorcode("problem creating $file: $message",$logger); } - } + } } return 0; } @@ -812,7 +814,7 @@ sub secs_to_years { } sub get_cpucount { - my $cpucount = Sys::CpuAffinity::getNumCpus() + 0; + my $cpucount = 0; #Sys::CpuAffinity::getNumCpus() + 0; return ($cpucount > 0) ? $cpucount : 1; #my $info = Sys::Info->new(); #my $cpu = $info->device('CPU'); # => %options ); @@ -824,11 +826,18 @@ sub get_cpucount { # $cpucount *= 2; #} - return ($cpucount > 0) ? $cpucount : 1; #printf "CPU: %s\n", scalar($cpu->identify) || 'N/A'; #printf "CPU speed is %s MHz\n", $cpu->speed || 'N/A'; #printf "There are %d CPUs\n" , $cpu->count || 1; #printf "CPU load: %s\n" , $cpu->load || 0; } -1; \ No newline at end of file +sub prompt { + my ($query) = @_; # take a prompt string as argument + local $| = 1; # activate autoflush to immediately show the prompt + print $query; + chomp(my $answer = ); + return $answer; +} + +1; diff --git a/default.cfg b/default.cfg index 5c546e32..9db8e7fd 100644 --- a/default.cfg +++ b/default.cfg @@ -1 +1,39 @@ -#working_path = /var/sipwise \ No newline at end of file +##general settings: +#working_path = /var/sipwise +cpucount = 1 +enablemultithreading = 1 + +##gearman/service listener config: +jobservers = 127.0.0.1:4730 + +##NGCP MySQL connectivity - "accounting" db: +accounting_host = 127.0.0.1 +accounting_port = 3306 +accounting_databasename = accounting +accounting_username = root +accounting_password = + +##NGCP MySQL connectivity - "billing" db: +billing_host = 127.0.0.1 +billing_port = 3306 +billing_databasename = accounting +billing_username = root +billing_password = + +##NGCP REST-API connectivity: +ngcprestapi_uri = https://127.0.0.1:443 +ngcprestapi_username = administrator +ngcprestapi_password = administrator +ngcprestapi_realm = api_admin_http + +##sending email: +emailenable = 0 +erroremailrecipient = +warnemailrecipient = +completionemailrecipient = rkrenn@sipwise.com +successemailrecipient = + +##logging: +fileloglevel = OFF +screenloglevel = INFO +emailloglevel = OFF