diff --git a/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Dao/import/Batch.pm b/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Dao/import/Batch.pm new file mode 100644 index 00000000..98554a66 --- /dev/null +++ b/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Dao/import/Batch.pm @@ -0,0 +1,267 @@ +package NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Batch; +use strict; + +## no critic + +#use File::Basename; +#use Cwd; +#use lib Cwd::abs_path(File::Basename::dirname(__FILE__) . '/../../../../'); + +use NGCP::BulkProcessor::Projects::Migration::IPGallery::ProjectConnectorPool qw( + get_import_db + +); +#import_db_tableidentifier + +use NGCP::BulkProcessor::SqlProcessor qw( + registertableinfo + create_targettable + checktableinfo + copy_row + + insert_stmt +); +use NGCP::BulkProcessor::SqlRecord qw(); + +use NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber qw(); + +require Exporter; +our @ISA = qw(Exporter NGCP::BulkProcessor::SqlRecord); +our @EXPORT_OK = qw( + create_table + gettablename + check_table + getinsertstatement + getupsertstatement + + findby_number + countby_number + + update_delta + findby_delta + countby_delta + + $deleted_delta + $updated_delta + $added_delta + +); + +my $tablename = 'batch'; +my $get_db = \&get_import_db; +#my $get_tablename = \&import_db_tableidentifier; + + +my $expected_fieldnames = [ + 'number', + 'delta', +]; + +# table creation: +my $primarykey_fieldnames = [ 'number' ]; +my $indexes = { $tablename . '_delta' => [ 'delta(7)' ]}; +#my $fixtable_statements = []; + +our $deleted_delta = 'DELETED'; +our $updated_delta = 'UPDATED'; +our $added_delta = 'ADDED'; + +sub new { + + my $class = shift; + my $self = NGCP::BulkProcessor::SqlRecord->new($get_db, + $tablename, + $expected_fieldnames,$indexes); + + bless($self,$class); + + copy_row($self,shift,$expected_fieldnames); + + return $self; + +} + +sub create_table { + + my ($truncate) = @_; + + my $db = &$get_db(); + + registertableinfo($db,$tablename,$expected_fieldnames,$indexes,$primarykey_fieldnames); + return create_targettable($db,$tablename,$db,$tablename,$truncate,0,undef); + +} + +sub findby_delta { + + my ($delta,$load_recursive) = @_; + + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + + return [] unless defined $delta; + + my $rows = $db->db_get_all_arrayref( + 'SELECT * FROM ' . + $table . + ' WHERE ' . + $db->columnidentifier('delta') . ' = ?' + ,$delta); + + return buildrecords_fromrows($rows,$load_recursive); + +} + +sub findby_number { + + my ($number,$load_recursive) = @_; + + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + + my $rows = $db->db_get_all_arrayref( + 'SELECT * FROM ' . + $table . + ' WHERE ' . + $db->columnidentifier('number') . ' = ?' + ,$number); + + return buildrecords_fromrows($rows,$load_recursive); + +} + +sub update_delta { + + my ($number,$delta) = @_; + + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + + my $stmt = 'UPDATE ' . $table . ' SET delta = ?'; + my @params = (); + push(@params,$delta); + if (defined $number) { + $stmt .= ' WHERE ' . $db->columnidentifier('number') . ' = ?'; + push(@params,$number); + } + + return $db->db_do($stmt,@params); + +} + +sub countby_number { + + my ($number) = @_; + + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + + my $stmt = 'SELECT COUNT(*) FROM ' . $table; + my @params = (); + if (defined $number) { + $stmt .= ' WHERE ' . $db->columnidentifier('number') . ' = ?'; + push(@params,$number); + } + + return $db->db_get_value($stmt,@params); + +} + +sub countby_delta { + + my ($delta) = @_; + + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + + my $stmt = 'SELECT COUNT(*) FROM ' . $table; + my @params = (); + if (defined $delta) { + $stmt .= ' WHERE ' . + $db->columnidentifier('delta') . ' = ?'; + push(@params,$delta); + } + + return $db->db_get_value($stmt,@params); + +} + +sub buildrecords_fromrows { + + my ($rows,$load_recursive) = @_; + + my @records = (); + my $record; + + if (defined $rows and ref $rows eq 'ARRAY') { + foreach my $row (@$rows) { + $record = __PACKAGE__->new($row); + + # transformations go here ... + if ($load_recursive) { + $record->{_subscriber} = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber::findby_subscribernumber( + $record->{number}, + $load_recursive, + ); + } + + push @records,$record; + } + } + + return \@records; + +} + +sub getinsertstatement { + + my ($insert_ignore) = @_; + check_table(); + return insert_stmt($get_db,$tablename,$insert_ignore); + +} + +sub getupsertstatement { + + my ($exists_delta,$new_delta) = @_; + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + my $upsert_stmt = 'INSERT OR REPLACE INTO ' . $table . ' (' . + join(', ',map { local $_ = $_; $_ = $db->columnidentifier($_); $_; } @$expected_fieldnames) . ')'; + my @values = (); + foreach my $fieldname (@$expected_fieldnames) { + if ('delta' eq $fieldname) { + my $stmt = 'SELECT \'' . $exists_delta . '\' FROM ' . $table . ' WHERE ' . + $db->columnidentifier('number') . ' = ?'; + push(@values,'COALESCE((' . $stmt . '), \'' . $new_delta . '\')'); + } else { + push(@values,'?'); + } + } + $upsert_stmt .= ' VALUES (' . join(',',@values) . ')'; + return $upsert_stmt; + +} + +sub gettablename { + + return $tablename; + +} + +sub check_table { + + return checktableinfo($get_db, + $tablename, + $expected_fieldnames, + $indexes); + +} + +1; diff --git a/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Dao/import/FeatureOption.pm b/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Dao/import/FeatureOption.pm index 7078535f..291307ea 100644 --- a/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Dao/import/FeatureOption.pm +++ b/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Dao/import/FeatureOption.pm @@ -32,9 +32,18 @@ our @EXPORT_OK = qw( gettablename check_table getinsertstatement + getupsertstatement - findby_subscribernumber - countby_subscribernumber + findby_subscribernumber_option + countby_subscribernumber_option + + update_delta + findby_delta + countby_delta + + $deleted_delta + $updated_delta + $added_delta ); my $tablename = 'feature_option'; @@ -44,14 +53,19 @@ my $get_db = \&get_import_db; my $expected_fieldnames = [ 'subscribernumber', - 'option' + 'option', + 'delta', ]; # table creation: my $primarykey_fieldnames = [ 'subscribernumber', 'option' ]; -my $indexes = {}; +my $indexes = { $tablename . '_delta' => [ 'delta(7)' ]}; #my $fixtable_statements = []; +our $deleted_delta = 'DELETED'; +our $updated_delta = 'UPDATED'; +our $added_delta = 'ADDED'; + sub new { my $class = shift; @@ -78,28 +92,75 @@ sub create_table { } -sub findby_subscribernumber { +sub findby_delta { - my ($subscribernumber,$load_recursive) = @_; + my ($delta,$load_recursive) = @_; check_table(); my $db = &$get_db(); my $table = $db->tableidentifier($tablename); + return [] unless defined $delta; + my $rows = $db->db_get_all_arrayref( 'SELECT * FROM ' . $table . ' WHERE ' . - $db->columnidentifier('subscribernumber') . ' = ?' - ,$subscribernumber); + $db->columnidentifier('delta') . ' = ?' + ,$delta); + + return buildrecords_fromrows($rows,$load_recursive); + +} + +sub findby_subscribernumber_option { + + my ($subscribernumber,$option,$load_recursive) = @_; + + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + + my $stmt = 'SELECT * FROM ' . $table . ' WHERE ' . + $db->columnidentifier('subscribernumber') . ' = ?'; + my @params = ($subscribernumber); + if (defined $option) { + $stmt .= ' AND ' . $db->columnidentifier('option') . ' = ?'; + push(@params,$option); + } + my $rows = $db->db_get_all_arrayref($stmt,@params); return buildrecords_fromrows($rows,$load_recursive); } -sub countby_subscribernumber { +sub update_delta { + + my ($subscribernumber,$option,$delta) = @_; + + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + + my $stmt = 'UPDATE ' . $table . ' SET delta = ?'; + my @params = (); + push(@params,$delta); + if (defined $subscribernumber) { + $stmt .= ' WHERE ' . $db->columnidentifier('subscribernumber') . ' = ?'; + push(@params,$subscribernumber); + if (defined $option) { + $stmt .= ' AND ' . $db->columnidentifier('option') . ' = ?'; + push(@params,$option); + } + } + + return $db->db_do($stmt,@params); + +} + +sub countby_subscribernumber_option { - my ($subscribernumber) = @_; + my ($subscribernumber,$option) = @_; check_table(); my $db = &$get_db(); @@ -110,6 +171,30 @@ sub countby_subscribernumber { if (defined $subscribernumber) { $stmt .= ' WHERE ' . $db->columnidentifier('subscribernumber') . ' = ?'; push(@params,$subscribernumber); + if (defined $option) { + $stmt .= ' AND ' . $db->columnidentifier('option') . ' = ?'; + push(@params,$option); + } + } + + return $db->db_get_value($stmt,@params); + +} + +sub countby_delta { + + my ($delta) = @_; + + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + + my $stmt = 'SELECT COUNT(*) FROM ' . $table; + my @params = (); + if (defined $delta) { + $stmt .= ' WHERE ' . + $db->columnidentifier('delta') . ' = ?'; + push(@params,$delta); } return $db->db_get_value($stmt,@params); @@ -129,9 +214,10 @@ sub buildrecords_fromrows { # transformations go here ... if ($load_recursive) { - $record->{_optionsetitems} = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOptionSetItem::findby_subscribernumber_option( + $record->{_optionsetitems} = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOptionSetItem::findby_subscribernumber_option_optionsetitem( $record->{subscribernumber}, $record->{option}, + undef, $load_recursive ); } @@ -152,6 +238,30 @@ sub getinsertstatement { } +sub getupsertstatement { + + my ($exists_delta,$new_delta) = @_; + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + my $upsert_stmt = 'INSERT OR REPLACE INTO ' . $table . ' (' . + join(', ',map { local $_ = $_; $_ = $db->columnidentifier($_); $_; } @$expected_fieldnames) . ')'; + my @values = (); + foreach my $fieldname (@$expected_fieldnames) { + if ('delta' eq $fieldname) { + my $stmt = 'SELECT \'' . $exists_delta . '\' FROM ' . $table . ' WHERE ' . + $db->columnidentifier('subscribernumber') . ' = ?' . + ' AND ' . $db->columnidentifier('option') . ' = ?'; + push(@values,'COALESCE((' . $stmt . '), \'' . $new_delta . '\')'); + } else { + push(@values,'?'); + } + } + $upsert_stmt .= ' VALUES (' . join(',',@values) . ')'; + return $upsert_stmt; + +} + sub gettablename { return $tablename; diff --git a/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Dao/import/FeatureOptionSetItem.pm b/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Dao/import/FeatureOptionSetItem.pm index 4ba73863..80cef0cc 100644 --- a/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Dao/import/FeatureOptionSetItem.pm +++ b/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Dao/import/FeatureOptionSetItem.pm @@ -30,9 +30,20 @@ our @EXPORT_OK = qw( gettablename check_table getinsertstatement + getupsertstatement - findby_subscribernumber_option - countby_subscribernumber_option + findby_subscribernumber_option_optionsetitem + countby_subscribernumber_option_optionsetitem + + update_delta + findby_delta + countby_delta + + $deleted_delta + $updated_delta + $added_delta + + $PRE_PAID_SERVICE_OPTION_SET ); my $tablename = 'feature_option_set_item'; @@ -43,16 +54,23 @@ my $get_db = \&get_import_db; my $expected_fieldnames = [ 'subscribernumber', 'option', - 'optionsetitem' + 'optionsetitem', + 'delta', ]; # table creation: -my $primarykey_fieldnames = []; #[ 'subscribernumber', 'option', 'optionsetitem' ]; -my $indexes = { - $tablename . '_subscribernumber_option_optionsetitem' => ['subscribernumber(11)', 'option(32)', 'optionsetitem(32)'], #(25),(27) -}; +my $primarykey_fieldnames = [ 'subscribernumber', 'option', 'optionsetitem' ]; +my $indexes = { $tablename . '_delta' => [ 'delta(7)' ]}; +# $tablename . '_subscribernumber_option_optionsetitem' => ['subscribernumber(11)', 'option(32)', 'optionsetitem(32)'], #(25),(27) +#}; #my $fixtable_statements = []; +our $deleted_delta = 'DELETED'; +our $updated_delta = 'UPDATED'; +our $added_delta = 'ADDED'; + +our $PRE_PAID_SERVICE_OPTION_SET = 'Pre_Paid_Service'; + sub new { my $class = shift; @@ -79,29 +97,81 @@ sub create_table { } -sub findby_subscribernumber_option { +sub findby_delta { - my ($subscribernumber,$option,$load_recursive) = @_; + my ($delta,$load_recursive) = @_; check_table(); my $db = &$get_db(); my $table = $db->tableidentifier($tablename); + return [] unless defined $delta; + my $rows = $db->db_get_all_arrayref( 'SELECT * FROM ' . $table . ' WHERE ' . - $db->columnidentifier('subscribernumber') . ' = ? ' . - ' AND ' . $db->columnidentifier('option') . ' = ?' - ,$subscribernumber,$option); + $db->columnidentifier('delta') . ' = ?' + ,$delta); return buildrecords_fromrows($rows,$load_recursive); } -sub countby_subscribernumber_option { +sub findby_subscribernumber_option_optionsetitem { - my ($subscribernumber,$option) = @_; + my ($subscribernumber,$option,$optionsetitem,$load_recursive) = @_; + + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + + my $stmt = 'SELECT * FROM ' . $table . + ' WHERE ' . $db->columnidentifier('subscribernumber') . ' = ? ' . + ' AND ' . $db->columnidentifier('option') . ' = ?'; + my @params = ($subscribernumber,$option); + if (defined $optionsetitem and length($optionsetitem) > 0) { + $stmt .= ' AND ' . $db->columnidentifier('optionsetitem') . ' = ?'; + push(@params,$optionsetitem); + } + + my $rows = $db->db_get_all_arrayref($stmt,@params); + + return buildrecords_fromrows($rows,$load_recursive); + +} + +sub update_delta { + + my ($subscribernumber,$option,$optionsetitem,$delta) = @_; + + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + + my $stmt = 'UPDATE ' . $table . ' SET delta = ?'; + my @params = (); + push(@params,$delta); + if (defined $subscribernumber) { + $stmt .= ' WHERE ' . $db->columnidentifier('subscribernumber') . ' = ?'; + push(@params,$subscribernumber); + if (defined $option) { + $stmt .= ' AND ' . $db->columnidentifier('option') . ' = ?'; + push(@params,$option); + if (defined $optionsetitem and length($optionsetitem) > 0) { + $stmt .= ' AND ' . $db->columnidentifier('optionsetitem') . ' = ?'; + push(@params,$optionsetitem); + } + } + } + + return $db->db_do($stmt,@params); + +} + +sub countby_subscribernumber_option_optionsetitem { + + my ($subscribernumber,$option,$optionsetitem) = @_; check_table(); my $db = &$get_db(); @@ -115,6 +185,10 @@ sub countby_subscribernumber_option { if (defined $option) { $stmt .= ' AND ' . $db->columnidentifier('option') . ' = ?'; push(@params,$option); + if (defined $optionsetitem and length($optionsetitem) > 0) { + $stmt .= ' AND ' . $db->columnidentifier('optionsetitem') . ' = ?'; + push(@params,$optionsetitem); + } } } @@ -122,6 +196,26 @@ sub countby_subscribernumber_option { } +sub countby_delta { + + my ($delta) = @_; + + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + + my $stmt = 'SELECT COUNT(*) FROM ' . $table; + my @params = (); + if (defined $delta) { + $stmt .= ' WHERE ' . + $db->columnidentifier('delta') . ' = ?'; + push(@params,$delta); + } + + return $db->db_get_value($stmt,@params); + +} + sub buildrecords_fromrows { my ($rows,$load_recursive) = @_; @@ -151,6 +245,31 @@ sub getinsertstatement { } +sub getupsertstatement { + + my ($exists_delta,$new_delta) = @_; + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + my $upsert_stmt = 'INSERT OR REPLACE INTO ' . $table . ' (' . + join(', ',map { local $_ = $_; $_ = $db->columnidentifier($_); $_; } @$expected_fieldnames) . ')'; + my @values = (); + foreach my $fieldname (@$expected_fieldnames) { + if ('delta' eq $fieldname) { + my $stmt = 'SELECT \'' . $exists_delta . '\' FROM ' . $table . ' WHERE ' . + $db->columnidentifier('subscribernumber') . ' = ?' . + ' AND ' . $db->columnidentifier('option') . ' = ?' . + ' AND ' . $db->columnidentifier('optionsetitem') . ' = ?'; + push(@values,'COALESCE((' . $stmt . '), \'' . $new_delta . '\')'); + } else { + push(@values,'?'); + } + } + $upsert_stmt .= ' VALUES (' . join(',',@values) . ')'; + return $upsert_stmt; + +} + sub gettablename { return $tablename; diff --git a/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Dao/import/Lnp.pm b/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Dao/import/Lnp.pm index 2e2ac79b..e952f4bb 100644 --- a/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Dao/import/Lnp.pm +++ b/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Dao/import/Lnp.pm @@ -30,10 +30,21 @@ our @EXPORT_OK = qw( gettablename check_table getinsertstatement + getupsertstatement findby_lrncode_portednumber countby_lrncode_portednumber count_lrncodes + + update_delta + findby_delta + countby_delta + + $deleted_delta + $updated_delta + $added_delta + + $IN_TYPE ); my $tablename = 'lnp'; @@ -45,13 +56,20 @@ my $expected_fieldnames = [ 'ported_number', 'type', 'lrn_code', + 'delta', ]; # table creation: my $primarykey_fieldnames = [ 'lrn_code', 'ported_number' ]; -my $indexes = {}; +my $indexes = { $tablename . '_delta' => [ 'delta(7)' ]}; #my $fixtable_statements = []; +our $deleted_delta = 'DELETED'; +our $updated_delta = 'UPDATED'; +our $added_delta = 'ADDED'; + +our $IN_TYPE = 'In'; + sub new { my $class = shift; @@ -78,6 +96,27 @@ sub create_table { } +sub findby_delta { + + my ($delta,$load_recursive) = @_; + + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + + return [] unless defined $delta; + + my $rows = $db->db_get_all_arrayref( + 'SELECT * FROM ' . + $table . + ' WHERE ' . + $db->columnidentifier('delta') . ' = ?' + ,$delta); + + return buildrecords_fromrows($rows,$load_recursive); + +} + sub findby_lrncode_portednumber { my ($lrncode,$portednumber,$load_recursive) = @_; @@ -98,6 +137,30 @@ sub findby_lrncode_portednumber { } +sub update_delta { + + my ($lrncode,$portednumber,$delta) = @_; + + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + + my $stmt = 'UPDATE ' . $table . ' SET delta = ?'; + my @params = (); + push(@params,$delta); + if (defined $lrncode) { + $stmt .= ' WHERE ' . $db->columnidentifier('lrn_code') . ' = ?'; + push(@params,$lrncode); + if (defined $portednumber) { + $stmt .= ' AND ' . $db->columnidentifier('ported_number') . ' = ?'; + push(@params,$portednumber); + } + } + + return $db->db_do($stmt,@params); + +} + sub countby_lrncode_portednumber { my ($lrncode,$portednumber) = @_; @@ -132,6 +195,26 @@ sub count_lrncodes { } +sub countby_delta { + + my ($delta) = @_; + + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + + my $stmt = 'SELECT COUNT(*) FROM ' . $table; + my @params = (); + if (defined $delta) { + $stmt .= ' WHERE ' . + $db->columnidentifier('delta') . ' = ?'; + push(@params,$delta); + } + + return $db->db_get_value($stmt,@params); + +} + sub buildrecords_fromrows { my ($rows,$load_recursive) = @_; @@ -161,6 +244,30 @@ sub getinsertstatement { } +sub getupsertstatement { + + my ($exists_delta,$new_delta) = @_; + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + my $upsert_stmt = 'INSERT OR REPLACE INTO ' . $table . ' (' . + join(', ',map { local $_ = $_; $_ = $db->columnidentifier($_); $_; } @$expected_fieldnames) . ')'; + my @values = (); + foreach my $fieldname (@$expected_fieldnames) { + if ('delta' eq $fieldname) { + my $stmt = 'SELECT \'' . $exists_delta . '\' FROM ' . $table . ' WHERE ' . + $db->columnidentifier('lrn_code') . ' = ?' . + ' AND ' . $db->columnidentifier('ported_number') . ' = ?'; + push(@values,'COALESCE((' . $stmt . '), \'' . $new_delta . '\')'); + } else { + push(@values,'?'); + } + } + $upsert_stmt .= ' VALUES (' . join(',',@values) . ')'; + return $upsert_stmt; + +} + sub gettablename { return $tablename; diff --git a/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Dao/import/Subscriber.pm b/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Dao/import/Subscriber.pm index e6c5b100..febdbd81 100644 --- a/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Dao/import/Subscriber.pm +++ b/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Dao/import/Subscriber.pm @@ -23,6 +23,9 @@ use NGCP::BulkProcessor::SqlProcessor qw( ); use NGCP::BulkProcessor::SqlRecord qw(); +use NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOption qw(); +use NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::UsernamePassword qw(); + require Exporter; our @ISA = qw(Exporter NGCP::BulkProcessor::SqlRecord); our @EXPORT_OK = qw( @@ -38,6 +41,8 @@ our @EXPORT_OK = qw( findby_delta countby_delta + split_subscribernumber + $deleted_delta $updated_delta $added_delta @@ -219,9 +224,14 @@ sub buildrecords_fromrows { # transformations go here ... if ($load_recursive) { - $record->{_features} = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOption::findby_subscribernumber( + $record->{_features} = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOption::findby_subscribernumber_option( $record->subscribernumber(), - $load_recursive + undef, + $load_recursive, + ); + $record->{_userpassword} = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::UsernamePassword::findby_fqdn( + $record->subscribernumber(), #$record->{rgw_fqdn} + $load_recursive, ); } diff --git a/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Dao/import/UsernamePassword.pm b/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Dao/import/UsernamePassword.pm new file mode 100644 index 00000000..d2239d31 --- /dev/null +++ b/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Dao/import/UsernamePassword.pm @@ -0,0 +1,262 @@ +package NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::UsernamePassword; +use strict; + +## no critic + +#use File::Basename; +#use Cwd; +#use lib Cwd::abs_path(File::Basename::dirname(__FILE__) . '/../../../../'); + +use NGCP::BulkProcessor::Projects::Migration::IPGallery::ProjectConnectorPool qw( + get_import_db + +); +#import_db_tableidentifier + +use NGCP::BulkProcessor::SqlProcessor qw( + registertableinfo + create_targettable + checktableinfo + copy_row + + insert_stmt +); +use NGCP::BulkProcessor::SqlRecord qw(); + +require Exporter; +our @ISA = qw(Exporter NGCP::BulkProcessor::SqlRecord); +our @EXPORT_OK = qw( + create_table + gettablename + check_table + getinsertstatement + getupsertstatement + + findby_fqdn + countby_fqdn + + update_delta + findby_delta + countby_delta + + $deleted_delta + $updated_delta + $added_delta + +); + +my $tablename = 'user_password'; +my $get_db = \&get_import_db; +#my $get_tablename = \&import_db_tableidentifier; + + +my $expected_fieldnames = [ + 'fqdn', + 'username', + 'password', + 'auth_level', + 'delta', +]; + +# table creation: +my $primarykey_fieldnames = [ 'fqdn' ]; +my $indexes = { $tablename . '_delta' => [ 'delta(7)' ]}; +#my $fixtable_statements = []; + +our $deleted_delta = 'DELETED'; +our $updated_delta = 'UPDATED'; +our $added_delta = 'ADDED'; + +sub new { + + my $class = shift; + my $self = NGCP::BulkProcessor::SqlRecord->new($get_db, + $tablename, + $expected_fieldnames,$indexes); + + bless($self,$class); + + copy_row($self,shift,$expected_fieldnames); + + return $self; + +} + +sub create_table { + + my ($truncate) = @_; + + my $db = &$get_db(); + + registertableinfo($db,$tablename,$expected_fieldnames,$indexes,$primarykey_fieldnames); + return create_targettable($db,$tablename,$db,$tablename,$truncate,0,undef); + +} + +sub findby_delta { + + my ($delta,$load_recursive) = @_; + + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + + return [] unless defined $delta; + + my $rows = $db->db_get_all_arrayref( + 'SELECT * FROM ' . + $table . + ' WHERE ' . + $db->columnidentifier('delta') . ' = ?' + ,$delta); + + return buildrecords_fromrows($rows,$load_recursive); + +} + +sub findby_fqdn { + + my ($fqdn,$load_recursive) = @_; + + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + + my $rows = $db->db_get_all_arrayref( + 'SELECT * FROM ' . + $table . + ' WHERE ' . + $db->columnidentifier('fqdn') . ' = ?' + ,$fqdn); + + return buildrecords_fromrows($rows,$load_recursive); + +} + +sub update_delta { + + my ($fqdn,$delta) = @_; + + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + + my $stmt = 'UPDATE ' . $table . ' SET delta = ?'; + my @params = (); + push(@params,$delta); + if (defined $fqdn) { + $stmt .= ' WHERE ' . $db->columnidentifier('fqdn') . ' = ?'; + push(@params,$fqdn); + } + + return $db->db_do($stmt,@params); + +} + +sub countby_fqdn { + + my ($fqdn) = @_; + + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + + my $stmt = 'SELECT COUNT(*) FROM ' . $table; + my @params = (); + if (defined $fqdn) { + $stmt .= ' WHERE ' . $db->columnidentifier('fqdn') . ' = ?'; + push(@params,$fqdn); + } + + return $db->db_get_value($stmt,@params); + +} + +sub countby_delta { + + my ($delta) = @_; + + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + + my $stmt = 'SELECT COUNT(*) FROM ' . $table; + my @params = (); + if (defined $delta) { + $stmt .= ' WHERE ' . + $db->columnidentifier('delta') . ' = ?'; + push(@params,$delta); + } + + return $db->db_get_value($stmt,@params); + +} + +sub buildrecords_fromrows { + + my ($rows,$load_recursive) = @_; + + my @records = (); + my $record; + + if (defined $rows and ref $rows eq 'ARRAY') { + foreach my $row (@$rows) { + $record = __PACKAGE__->new($row); + + # transformations go here ... + + push @records,$record; + } + } + + return \@records; + +} + +sub getinsertstatement { + + my ($insert_ignore) = @_; + check_table(); + return insert_stmt($get_db,$tablename,$insert_ignore); + +} + +sub getupsertstatement { + + my ($exists_delta,$new_delta) = @_; + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + my $upsert_stmt = 'INSERT OR REPLACE INTO ' . $table . ' (' . + join(', ',map { local $_ = $_; $_ = $db->columnidentifier($_); $_; } @$expected_fieldnames) . ')'; + my @values = (); + foreach my $fieldname (@$expected_fieldnames) { + if ('delta' eq $fieldname) { + my $stmt = 'SELECT \'' . $exists_delta . '\' FROM ' . $table . ' WHERE ' . + $db->columnidentifier('fqdn') . ' = ?'; + push(@values,'COALESCE((' . $stmt . '), \'' . $new_delta . '\')'); + } else { + push(@values,'?'); + } + } + $upsert_stmt .= ' VALUES (' . join(',',@values) . ')'; + return $upsert_stmt; + +} + +sub gettablename { + + return $tablename; + +} + +sub check_table { + + return checktableinfo($get_db, + $tablename, + $expected_fieldnames, + $indexes); + +} + +1; diff --git a/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/FileProcessors/BatchFile.pm b/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/FileProcessors/BatchFile.pm new file mode 100644 index 00000000..2ba3d8da --- /dev/null +++ b/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/FileProcessors/BatchFile.pm @@ -0,0 +1,68 @@ +package NGCP::BulkProcessor::Projects::Migration::IPGallery::FileProcessors::BatchFile; +use strict; + +## no critic + +#use File::Basename; +#use Cwd; +#use lib Cwd::abs_path(File::Basename::dirname(__FILE__) . '/../../../../'); + +use NGCP::BulkProcessor::Logging qw( + getlogger +); +use NGCP::BulkProcessor::LogError qw( + fileprocessingerror + fileprocessingwarn +); + +use NGCP::BulkProcessor::FileProcessor; + +require Exporter; +our @ISA = qw(Exporter NGCP::BulkProcessor::FileProcessor); +our @EXPORT_OK = qw(); + +my $lineseparator = '\\n'; +my $fieldseparator = " +"; +my $encoding = 'UTF-8'; + +my $buffersize = 100 * 1024; +my $threadqueuelength = 10; +my $default_numofthreads = 3; +#my $multithreading = 0; +my $blocksize = 500; + +sub new { + + my $class = shift; + + my $self = NGCP::BulkProcessor::FileProcessor->new(@_); + + $self->{numofthreads} = shift // $default_numofthreads; + $self->{line_separator} = $lineseparator; + $self->{field_separator} = $fieldseparator; + $self->{encoding} = $encoding; + $self->{buffersize} = $buffersize; + $self->{threadqueuelength} = $threadqueuelength; + #$self->{multithreading} = $multithreading; + $self->{blocksize} = $blocksize; + + bless($self,$class); + + #restdebug($self,__PACKAGE__ . ' file processor created',getlogger(__PACKAGE__)); + + return $self; + +} + +sub extractfields { + my ($context,$line_ref) = @_; + my $separator = $context->{instance}->{field_separator}; + $$line_ref =~ s/^ +//; + $$line_ref =~ s/ +$//; + return undef if length($$line_ref) == 0; + return undef if $$line_ref =~ /^#/; + my @fields = split(/$separator/,$$line_ref,-1); + return \@fields; +} + +1; diff --git a/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/FileProcessors/UserPasswordFile.pm b/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/FileProcessors/UserPasswordFile.pm new file mode 100644 index 00000000..3bfa0d66 --- /dev/null +++ b/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/FileProcessors/UserPasswordFile.pm @@ -0,0 +1,68 @@ +package NGCP::BulkProcessor::Projects::Migration::IPGallery::FileProcessors::UserPasswordFile; +use strict; + +## no critic + +#use File::Basename; +#use Cwd; +#use lib Cwd::abs_path(File::Basename::dirname(__FILE__) . '/../../../../'); + +use NGCP::BulkProcessor::Logging qw( + getlogger +); +use NGCP::BulkProcessor::LogError qw( + fileprocessingerror + fileprocessingwarn +); + +use NGCP::BulkProcessor::FileProcessor; + +require Exporter; +our @ISA = qw(Exporter NGCP::BulkProcessor::FileProcessor); +our @EXPORT_OK = qw(); + +my $lineseparator = '\\r\\n'; +my $fieldseparator = " +"; +my $encoding = 'UTF-8'; + +my $buffersize = 100 * 1024; +my $threadqueuelength = 10; +my $default_numofthreads = 3; +#my $multithreading = 0; +my $blocksize = 500; + +sub new { + + my $class = shift; + + my $self = NGCP::BulkProcessor::FileProcessor->new(@_); + + $self->{numofthreads} = shift // $default_numofthreads; + $self->{line_separator} = $lineseparator; + $self->{field_separator} = $fieldseparator; + $self->{encoding} = $encoding; + $self->{buffersize} = $buffersize; + $self->{threadqueuelength} = $threadqueuelength; + #$self->{multithreading} = $multithreading; + $self->{blocksize} = $blocksize; + + bless($self,$class); + + #restdebug($self,__PACKAGE__ . ' file processor created',getlogger(__PACKAGE__)); + + return $self; + +} + +sub extractfields { + my ($context,$line_ref) = @_; + my $separator = $context->{instance}->{field_separator}; + $$line_ref =~ s/^ +//; + $$line_ref =~ s/ +$//; + return undef if length($$line_ref) == 0; + return undef if $$line_ref =~ /^#/; + my @fields = split(/$separator/,$$line_ref,-1); + return \@fields; +} + +1; diff --git a/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Import.pm b/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Import.pm index dea57dc9..57bf66dc 100644 --- a/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Import.pm +++ b/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Import.pm @@ -20,8 +20,13 @@ use NGCP::BulkProcessor::Projects::Migration::IPGallery::Settings qw( $subscribernumer_exclude_pattern $subscribernumer_exclude_exception_pattern $ignore_subscriber_unique + $skip_prepaid_subscribers $lnp_define_import_numofthreads $ignore_lnp_unique + $user_password_import_numofthreads + $ignore_user_password_unique + $batch_import_numofthreads + $ignore_batch_unique $dry ); use NGCP::BulkProcessor::Logging qw ( @@ -37,6 +42,8 @@ use NGCP::BulkProcessor::Projects::Migration::IPGallery::FileProcessors::Feature use NGCP::BulkProcessor::Projects::Migration::IPGallery::FeaturesDefineParser qw(); use NGCP::BulkProcessor::Projects::Migration::IPGallery::FileProcessors::SubscriberDefineFile qw(); use NGCP::BulkProcessor::Projects::Migration::IPGallery::FileProcessors::LnpDefineFile qw(); +use NGCP::BulkProcessor::Projects::Migration::IPGallery::FileProcessors::UserPasswordFile qw(); +use NGCP::BulkProcessor::Projects::Migration::IPGallery::FileProcessors::BatchFile qw(); use NGCP::BulkProcessor::Projects::Migration::IPGallery::ProjectConnectorPool qw( get_import_db @@ -47,6 +54,8 @@ use NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOpt use NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOptionSetItem qw(); use NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber qw(); use NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Lnp qw(); +use NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::UsernamePassword qw(); +use NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Batch qw(); use NGCP::BulkProcessor::Array qw(removeduplicates); use NGCP::BulkProcessor::Utils qw(threadid); @@ -57,14 +66,16 @@ our @EXPORT_OK = qw( import_features_define import_subscriber_define import_lnp_define + import_user_password + import_batch ); sub import_features_define { my ($file) = @_; # create tables: - my $result = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOption::create_table(1); - $result &= NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOptionSetItem::create_table(1); + my $result = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOption::create_table(0); + $result &= NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOptionSetItem::create_table(0); # checks, e.g. other table must be present: # ..none.. @@ -73,6 +84,8 @@ sub import_features_define { my $importer = NGCP::BulkProcessor::Projects::Migration::IPGallery::FileProcessors::FeaturesDefineFile->new($features_define_import_numofthreads); $importer->stoponparseerrors(!$dry); + my $upsert = _import_features_define_reset_delta(); + # launch: destroy_all_dbs(); #close all db connections before forking.. return $result && $importer->process( @@ -103,12 +116,30 @@ sub import_features_define { if ('HASH' eq ref $option) { foreach my $setoption (keys %$option) { foreach my $setoptionitem (@{$skip_duplicate_setoptionitems ? removeduplicates($option->{$setoption}) : $option->{$setoption}}) { - push(@featureoptionsetitem_rows,[ $subscriber_number, $setoption, $setoptionitem ]); + if ($context->{upsert}) { + push(@featureoptionsetitem_rows,[ $subscriber_number, $setoption, $setoptionitem, + $subscriber_number, $setoption, $setoptionitem ]); + } else { + push(@featureoptionsetitem_rows,[ $subscriber_number, $setoption, $setoptionitem, + $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOptionSetItem::added_delta ]); + } + } + if ($context->{upsert}) { + push(@featureoption_rows,[ $subscriber_number, $setoption, + $subscriber_number, $setoption ]); + } else { + push(@featureoption_rows,[ $subscriber_number, $setoption, + $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOption::added_delta ]); } - push(@featureoption_rows,[ $subscriber_number, $setoption ]); } } else { - push(@featureoption_rows,[ $subscriber_number, $option ]); + if ($context->{upsert}) { + push(@featureoption_rows,[ $subscriber_number, $option, + $subscriber_number, $option ]); + } else { + push(@featureoption_rows,[ $subscriber_number, $option, + $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOption::added_delta ]); + } } } } @@ -141,6 +172,7 @@ sub import_features_define { } } $context->{db} = &get_import_db(); # keep ref count low.. + $context->{upsert} = $upsert; }, uninit_process_context_code => sub { my ($context)= @_; @@ -152,10 +184,33 @@ sub import_features_define { } +sub _import_features_define_reset_delta { + my $upsert = 0; + if (NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOption::countby_subscribernumber_option() > 0) { + processing_info(threadid(),'resetting delta of ' . + NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOption::update_delta(undef,undef, + $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOption::deleted_delta) . + ' feature option records',getlogger(__PACKAGE__)); + $upsert |= 1; + } + if (NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOptionSetItem::countby_subscribernumber_option_optionsetitem() > 0) { + processing_info(threadid(),'resetting delta of ' . + NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOptionSetItem::update_delta(undef,undef,undef, + $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOptionSetItem::deleted_delta) . + ' feature set option item records',getlogger(__PACKAGE__)); + $upsert |= 1; + } + return $upsert; +} + sub _insert_featureoption_rows { my ($context,$featureoption_rows) = @_; $context->{db}->db_do_begin( - NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOption::getinsertstatement($ignore_options_unique), + ($context->{upsert} ? + NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOption::getupsertstatement( + $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOption::updated_delta, + $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOption::added_delta + ) : NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOption::getinsertstatement($ignore_options_unique)), #NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOption::gettablename(), #lock - $import_multithreading ); @@ -166,7 +221,11 @@ sub _insert_featureoption_rows { sub _insert_featureoptionsetitem_rows { my ($context,$featureoptionsetitem_rows) = @_; $context->{db}->db_do_begin( - NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOptionSetItem::getinsertstatement($ignore_setoptionitems_unique), + ($context->{upsert} ? + NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOptionSetItem::getupsertstatement( + $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOptionSetItem::updated_delta, + $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOptionSetItem::added_delta + ) : NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOptionSetItem::getinsertstatement($ignore_setoptionitems_unique)), #NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOptionSetItem::gettablename(), #lock ); @@ -210,7 +269,7 @@ sub import_subscriber_define { } my @subscriber_row = @$row; if ($context->{upsert}) { - push(@subscriber_row,$record->{country_code},$record->{area_code},$record->{dial_number}); + push(@subscriber_row,NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber::split_subscribernumber($record->subscribernumber())); } else { push(@subscriber_row,$NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber::added_delta); } @@ -244,16 +303,39 @@ sub import_subscriber_define { sub _import_subscriber_define_referential_checks { my ($context,$record,$rownum) = @_; - my $result = 0; - if (NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOption::countby_subscribernumber($record->subscribernumber()) > 0) { - $result = 1; + my $result = 1; + if (NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOption::countby_subscribernumber_option($record->subscribernumber()) > 0) { + if ($skip_prepaid_subscribers) { + my $prepaid_option_set_item = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOptionSetItem::findby_subscribernumber_option_optionsetitem( + $record->subscribernumber(), + $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOptionSetItem::PRE_PAID_SERVICE_OPTION_SET, + )->[0]; + if (defined $prepaid_option_set_item and $prepaid_option_set_item->{delta} ne + $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOptionSetItem::deleted_delta) { + processing_info($context->{tid},'record ' . $rownum . ' - skipped, ' . $prepaid_option_set_item->{optionsetitem} . ': ' . $record->{dial_number},getlogger(__PACKAGE__)); + $result &= 0; + } + } } else { + $result &= 0; if ($dry) { fileprocessingwarn($context->{filename},'record ' . $rownum . ' - no features records for subscriber found: ' . $record->{dial_number},getlogger(__PACKAGE__)); } else { fileprocessingerror($context->{filename},'record ' . $rownum . ' - no features records for subscriber found: ' . $record->{dial_number},getlogger(__PACKAGE__)); } } + + if (NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::UsernamePassword::countby_fqdn($record->subscribernumber()) > 0) { + + } else { + $result &= 0; + if ($dry) { + fileprocessingwarn($context->{filename},'record ' . $rownum . ' - no username password record for subscriber found: ' . $record->{dial_number},getlogger(__PACKAGE__)); + } else { + fileprocessingerror($context->{filename},'record ' . $rownum . ' - no username password record for subscriber found: ' . $record->{dial_number},getlogger(__PACKAGE__)); + } + } + return $result; } @@ -263,24 +345,33 @@ sub _import_subscriber_define_checks { my $result = 1; my $optioncount = 0; eval { - $optioncount = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOption::countby_subscribernumber(); + $optioncount = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOption::countby_subscribernumber_option(); }; if ($@ or $optioncount == 0) { fileprocessingerror($file,'please import subscriber features first',getlogger(__PACKAGE__)); $result = 0; #even in dry mode.. } + my $userpasswordcount = 0; + eval { + $userpasswordcount = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::UsernamePassword::countby_fqdn(); + }; + if ($@ or $userpasswordcount == 0) { + fileprocessingerror($file,'please import user passwords first',getlogger(__PACKAGE__)); + $result = 0; #even in dry mode.. + } return $result; } sub _import_subscriber_define_reset_delta { + my $upsert = 0; if (NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber::countby_subscribernumber() > 0) { - processing_info(threadid(),'resetting subscriber delta of ' . + processing_info(threadid(),'resetting delta of ' . NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber::update_delta(undef, $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber::deleted_delta) . - ' records',getlogger(__PACKAGE__)); - return 1; + ' subscriber records',getlogger(__PACKAGE__)); + $upsert |= 1; } - return 0; + return $upsert; } sub _insert_subscriber_rows { @@ -303,10 +394,12 @@ sub import_lnp_define { my ($file) = @_; - my $result = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Lnp::create_table(1); + my $result = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Lnp::create_table(0); my $importer = NGCP::BulkProcessor::Projects::Migration::IPGallery::FileProcessors::LnpDefineFile->new($lnp_define_import_numofthreads); + my $upsert = _import_lnp_define_reset_delta(); + destroy_all_dbs(); #close all db connections before forking.. return $result && $importer->process( file => $file, @@ -316,10 +409,18 @@ sub import_lnp_define { my @lnp_rows = (); foreach my $row (@$rows) { $rownum++; - next if $row->[2] eq 'In'; - $row->[3] = substr($row->[3],0,4); - shift @$row; #ignore first col - push(@lnp_rows,$row); + my @lnp_row = @$row; + shift @lnp_row; #ignore first col + my $record = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Lnp->new(\@lnp_row); + next if $record->{type} eq $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Lnp::IN_TYPE; + $record->{lrn_code} = substr($record->{lrn_code},0,4); + @lnp_row = ( $record->{ported_number}, $record->{type}, $record->{lrn_code} ); + if ($context->{upsert}) { + push(@lnp_row,$record->{lrn_code}, $record->{ported_number}); + } else { + push(@lnp_row,$NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Lnp::added_delta); + } + push(@lnp_rows,\@lnp_row); } if ((scalar @lnp_rows) > 0) { @@ -335,6 +436,7 @@ sub import_lnp_define { init_process_context_code => sub { my ($context)= @_; $context->{db} = &get_import_db(); # keep ref count low.. + $context->{upsert} = $upsert; }, uninit_process_context_code => sub { my ($context)= @_; @@ -346,10 +448,27 @@ sub import_lnp_define { } +sub _import_lnp_define_reset_delta { + my $upsert = 0; + if (NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Lnp::countby_lrncode_portednumber() > 0) { + processing_info(threadid(),'resetting delta of ' . + NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Lnp::update_delta(undef,undef, + $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Lnp::deleted_delta) . + ' lnp number records',getlogger(__PACKAGE__)); + $upsert |= 1; + } + return $upsert; +} + sub _insert_lnp_rows { my ($context,$lnp_rows) = @_; $context->{db}->db_do_begin( - NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Lnp::getinsertstatement($ignore_lnp_unique), + ($context->{upsert} ? + NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Lnp::getupsertstatement( + $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Lnp::updated_delta, + $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Lnp::added_delta + ) + : NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Lnp::getinsertstatement($ignore_lnp_unique)), #NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Lnp::gettablename(), #lock ); @@ -357,4 +476,208 @@ sub _insert_lnp_rows { $context->{db}->db_finish(); } + +sub import_user_password { + + my ($file) = @_; + # create tables: + my $result = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::UsernamePassword::create_table(0); + + # checks, e.g. other table must be present: + # ..none.. + + # prepare parse: + my $importer = NGCP::BulkProcessor::Projects::Migration::IPGallery::FileProcessors::UserPasswordFile->new($user_password_import_numofthreads); + + my $upsert = _import_user_password_reset_delta(); + + # launch: + destroy_all_dbs(); #close all db connections before forking.. + return $result && $importer->process( + file => $file, + process_code => sub { + my ($context,$rows,$row_offset) = @_; + my $rownum = $row_offset; + my @usernamepassword_rows = (); + foreach my $row (@$rows) { + $rownum++; + my @usernamepassword_row = @$row; + my $record = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::UsernamePassword->new(\@usernamepassword_row); + if ($context->{upsert}) { + push(@usernamepassword_row,$record->{fqdn}); + } else { + push(@usernamepassword_row,$NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::UsernamePassword::added_delta); + } + push(@usernamepassword_rows,\@usernamepassword_row); + } + + if ((scalar @usernamepassword_rows) > 0) { + if ($dry) { + eval { _insert_usernamepassword_rows($context,\@usernamepassword_rows); }; + } else { + _insert_usernamepassword_rows($context,\@usernamepassword_rows); + } + } + + return 1; + }, + init_process_context_code => sub { + my ($context)= @_; + $context->{db} = &get_import_db(); # keep ref count low.. + $context->{upsert} = $upsert; + }, + uninit_process_context_code => sub { + my ($context)= @_; + undef $context->{db}; + destroy_all_dbs(); + }, + multithreading => $import_multithreading + ); + +} + +sub _import_user_password_reset_delta { + my $upsert = 0; + if (NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::UsernamePassword::countby_fqdn() > 0) { + processing_info(threadid(),'resetting delta of ' . + NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::UsernamePassword::update_delta(undef, + $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::UsernamePassword::deleted_delta) . + ' username password records',getlogger(__PACKAGE__)); + $upsert |= 1; + } + return $upsert; +} + +sub _insert_usernamepassword_rows { + my ($context,$usernamepassword_rows) = @_; + $context->{db}->db_do_begin( + ($context->{upsert} ? + NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::UsernamePassword::getupsertstatement( + $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::UsernamePassword::updated_delta, + $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::UsernamePassword::added_delta + ) : NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::UsernamePassword::getinsertstatement($ignore_user_password_unique)), + #NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOption::gettablename(), + #lock - $import_multithreading + ); + $context->{db}->db_do_rowblock($usernamepassword_rows); + $context->{db}->db_finish(); +} + +sub import_batch { + + my ($file) = @_; + + my $result = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Batch::create_table(0); + + $result &= _import_batch_checks($file); + + my $importer = NGCP::BulkProcessor::Projects::Migration::IPGallery::FileProcessors::BatchFile->new($batch_import_numofthreads); + + my $upsert = _import_batch_reset_delta(); + + destroy_all_dbs(); #close all db connections before forking.. + return $result && $importer->process( + file => $file, + process_code => sub { + my ($context,$rows,$row_offset) = @_; + my $rownum = $row_offset; + my @batch_rows = (); + foreach my $row (@$rows) { + $rownum++; + my $record = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Batch->new($row); + next unless _import_batch_referential_checks($context,$record,$rownum); + my @batch_row = @$row; + if ($context->{upsert}) { + push(@batch_row,$record->{number}); + } else { + push(@batch_row,$NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Batch::added_delta); + } + push(@batch_rows,\@batch_row); + } + + if ((scalar @batch_rows) > 0) { + if ($dry) { + eval { _insert_batch_rows($context,\@batch_rows); }; + } else { + _insert_batch_rows($context,\@batch_rows); + } + } + + return 1; + }, + init_process_context_code => sub { + my ($context)= @_; + $context->{db} = &get_import_db(); # keep ref count low.. + $context->{upsert} = $upsert; + }, + uninit_process_context_code => sub { + my ($context)= @_; + undef $context->{db}; + destroy_all_dbs(); + }, + multithreading => $import_multithreading + ); + +} + +sub _import_batch_referential_checks { + my ($context,$record,$rownum) = @_; + my $result = 1; + if (NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber::countby_subscribernumber($record->{number}) > 0) { + + } else { + $result &= 0; + if ($dry) { + fileprocessingwarn($context->{filename},'record ' . $rownum . ' - no subscriber record for batch number found: ' . $record->{number},getlogger(__PACKAGE__)); + } else { + fileprocessingerror($context->{filename},'record ' . $rownum . ' - no subscriber record for batch number found: ' . $record->{number},getlogger(__PACKAGE__)); + } + } + + return $result; + +} + +sub _import_batch_checks { + my ($file) = @_; + my $result = 1; + my $subscribercount = 0; + eval { + $subscribercount = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber::countby_subscribernumber(); + }; + if ($@ or $subscribercount == 0) { + fileprocessingerror($file,'please import subscribers first',getlogger(__PACKAGE__)); + $result = 0; #even in dry mode.. + } + return $result; +} + +sub _import_batch_reset_delta { + my $upsert = 0; + if (NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Batch::countby_number() > 0) { + processing_info(threadid(),'resetting delta of ' . + NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Batch::update_delta(undef, + $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Batch::deleted_delta) . + ' batch records',getlogger(__PACKAGE__)); + $upsert |= 1; + } + return $upsert; +} + +sub _insert_batch_rows { + my ($context,$batch_rows) = @_; + $context->{db}->db_do_begin( + ($context->{upsert} ? + NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Batch::getupsertstatement( + $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Batch::updated_delta, + $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Batch::added_delta + ) + : NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Batch::getinsertstatement($ignore_batch_unique)), + #NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber::gettablename(), + #lock + ); + $context->{db}->db_do_rowblock($batch_rows); + $context->{db}->db_finish(); +} + 1; diff --git a/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Settings.pm b/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Settings.pm index 446db56b..696fe3f5 100644 --- a/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Settings.pm +++ b/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/Settings.pm @@ -62,12 +62,20 @@ our @EXPORT_OK = qw( $subscribernumer_exclude_pattern $subscribernumer_exclude_exception_pattern $ignore_subscriber_unique + $skip_prepaid_subscribers $lnp_define_filename $lnp_define_import_numofthreads $ignore_lnp_unique - $stats_record_list_limit + $user_password_filename + $user_password_import_numofthreads + $ignore_user_password_unique + + $batch_filename + $batch_import_numofthreads + $ignore_batch_unique + ); our $defaultconfig = 'config.cfg'; @@ -85,7 +93,7 @@ our $import_multithreading = $enablemultithreading; our $features_define_filename = undef; our $features_define_import_numofthreads = $cpucount; -our $skip_duplicate_setoptionitems = 0; +our $skip_duplicate_setoptionitems = 1; our $ignore_options_unique = 0; our $ignore_setoptionitems_unique = 0; @@ -94,13 +102,19 @@ our $subscriber_define_import_numofthreads = $cpucount; our $subscribernumer_exclude_pattern = undef; our $subscribernumer_exclude_exception_pattern = undef; our $ignore_subscriber_unique = 0; +our $skip_prepaid_subscribers = 1; our $lnp_define_filename = undef; our $lnp_define_import_numofthreads = $cpucount; our $ignore_lnp_unique = 1; +our $user_password_filename = undef; +our $user_password_import_numofthreads = $cpucount; +our $ignore_user_password_unique = 0; -our $stats_record_list_limit = 1000; +our $batch_filename = undef; +our $batch_import_numofthreads = $cpucount; +our $ignore_batch_unique = 0; sub update_settings { @@ -132,7 +146,14 @@ sub update_settings { $result &= $regexp_result; $lnp_define_filename = _get_import_filename($lnp_define_filename,$data,'lnp_define_filename'); - $lnp_define_import_numofthreads= _get_import_numofthreads($cpucount,$data,'lnp_define_import_numofthreads'); + $lnp_define_import_numofthreads = _get_import_numofthreads($cpucount,$data,'lnp_define_import_numofthreads'); + + $user_password_filename = _get_import_filename($user_password_filename,$data,'user_password_filename'); + $user_password_import_numofthreads = _get_import_numofthreads($cpucount,$data,'user_password_import_numofthreads'); + + $batch_filename = _get_import_filename($batch_filename,$data,'batch_filename'); + $batch_import_numofthreads = _get_import_numofthreads($cpucount,$data,'batch_import_numofthreads'); + return $result; diff --git a/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/process.pl b/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/process.pl index e83f6b61..bed31de0 100644 --- a/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/process.pl +++ b/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/process.pl @@ -23,7 +23,8 @@ use NGCP::BulkProcessor::Projects::Migration::IPGallery::Settings qw( $features_define_filename $subscriber_define_filename $lnp_define_filename - $stats_record_list_limit + $user_password_filename + $batch_filename ); use NGCP::BulkProcessor::Logging qw( init_log @@ -50,11 +51,6 @@ use NGCP::BulkProcessor::Array qw(removeduplicates); use NGCP::BulkProcessor::Utils qw(getscriptpath prompt cleanupdir); use NGCP::BulkProcessor::Mail qw( cleanupmsgfiles - wrap_mailbody - $signature - $normalpriority - $lowpriority - $highpriority ); use NGCP::BulkProcessor::SqlConnectors::CSVDB qw(cleanupcvsdirs); use NGCP::BulkProcessor::SqlConnectors::SQLiteDB qw(cleanupdbfiles); @@ -65,12 +61,16 @@ use NGCP::BulkProcessor::Projects::Migration::IPGallery::Import qw( import_features_define import_subscriber_define import_lnp_define + import_user_password + import_batch ); use NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOption qw(); use NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOptionSetItem qw(); use NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber qw(); use NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Lnp qw(); +use NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::UsernamePassword qw(); +use NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Batch qw(); scripterror(getscriptpath() . ' already running',getlogger(getscriptpath())) unless flock DATA, LOCK_EX | LOCK_NB; # not tested on windows yet @@ -81,14 +81,26 @@ my $cleanup_task_opt = 'cleanup'; push(@TASK_OPTS,$cleanup_task_opt); my $cleanup_all_task_opt = 'cleanup_all'; push(@TASK_OPTS,$cleanup_all_task_opt); -my $import_features_define_task_opt = 'import_features'; +my $import_features_define_task_opt = 'import_feature'; push(@TASK_OPTS,$import_features_define_task_opt); +my $import_truncate_features_task_opt = 'truncate_feature'; +push(@TASK_OPTS,$import_truncate_features_task_opt); my $import_subscriber_define_task_opt = 'import_subscriber'; push(@TASK_OPTS,$import_subscriber_define_task_opt); my $import_truncate_subscriber_task_opt = 'truncate_subscriber'; push(@TASK_OPTS,$import_truncate_subscriber_task_opt); my $import_lnp_define_task_opt = 'import_lnp'; push(@TASK_OPTS,$import_lnp_define_task_opt); +my $import_truncate_lnp_task_opt = 'truncate_lnp'; +push(@TASK_OPTS,$import_truncate_lnp_task_opt); +my $import_user_password_task_opt = 'import_user_password'; +push(@TASK_OPTS,$import_user_password_task_opt); +my $import_truncate_user_password_task_opt = 'truncate_user_password'; +push(@TASK_OPTS,$import_truncate_user_password_task_opt); +my $import_batch_task_opt = 'import_batch'; +push(@TASK_OPTS,$import_batch_task_opt); +my $import_truncate_batch_task_opt = 'truncate_batch'; +push(@TASK_OPTS,$import_truncate_batch_task_opt); if (init()) { @@ -131,18 +143,37 @@ sub main() { if ('ARRAY' eq ref $tasks and (scalar @$tasks) > 0) { foreach my $task (@$tasks) { + if (lc($cleanup_task_opt) eq lc($task)) { $result = cleanup_task(\@messages,0) if taskinfo($cleanup_task_opt,$result); } elsif (lc($cleanup_all_task_opt) eq lc($task)) { $result = cleanup_task(\@messages,1) if taskinfo($cleanup_all_task_opt,$result); + } elsif (lc($import_features_define_task_opt) eq lc($task)) { $result = import_features_define_task(\@messages) if taskinfo($import_features_define_task_opt,$result); + } elsif (lc($import_truncate_features_task_opt) eq lc($task)) { + $result = import_truncate_features_task(\@messages) if taskinfo($import_truncate_features_task_opt,$result); + } elsif (lc($import_subscriber_define_task_opt) eq lc($task)) { $result = import_subscriber_define_task(\@messages) if taskinfo($import_subscriber_define_task_opt,$result); } elsif (lc($import_truncate_subscriber_task_opt) eq lc($task)) { $result = import_truncate_subscriber_task(\@messages) if taskinfo($import_truncate_subscriber_task_opt,$result); + } elsif (lc($import_lnp_define_task_opt) eq lc($task)) { $result = import_lnp_define_task(\@messages) if taskinfo($import_lnp_define_task_opt,$result); + } elsif (lc($import_truncate_lnp_task_opt) eq lc($task)) { + $result = import_truncate_lnp_task(\@messages) if taskinfo($import_truncate_lnp_task_opt,$result); + + } elsif (lc($import_user_password_task_opt) eq lc($task)) { + $result = import_user_password_task(\@messages) if taskinfo($import_user_password_task_opt,$result); + } elsif (lc($import_truncate_user_password_task_opt) eq lc($task)) { + $result = import_truncate_user_password_task(\@messages) if taskinfo($import_truncate_user_password_task_opt,$result); + + } elsif (lc($import_batch_task_opt) eq lc($task)) { + $result = import_batch_task(\@messages) if taskinfo($import_batch_task_opt,$result); + } elsif (lc($import_truncate_batch_task_opt) eq lc($task)) { + $result = import_truncate_batch_task(\@messages) if taskinfo($import_truncate_batch_task_opt,$result); + } elsif (lc('blah') eq lc($task)) { if (taskinfo($cleanup_task_opt,$result)) { next unless check_dry(); @@ -191,7 +222,7 @@ sub cleanup_task { }; } if ($@ or !$result) { - push(@$messages,'working directory cleanup incomplete'); + push(@$messages,'working directory cleanup INCOMPLETE'); return 0; } else { push(@$messages,'working directory folders cleaned up'); @@ -210,12 +241,39 @@ sub import_features_define_task { my $stats = ''; eval { $stats .= "\n total feature option records: " . - NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOption::countby_subscribernumber() . ' rows'; + NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOption::countby_subscribernumber_option() . ' rows'; + my $added_count = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOption::countby_delta( + $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOption::added_delta + ); + $stats .= "\n new: $added_count rows"; + my $existing_count = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOption::countby_delta( + $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOption::updated_delta + ); + $stats .= "\n existing: $existing_count rows"; + my $deleted_count = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOption::countby_delta( + $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOption::deleted_delta + ); + $stats .= "\n removed: $deleted_count rows"; + $stats .= "\n total feature set option item records: " . - NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOptionSetItem::countby_subscribernumber_option() . ' rows'; + NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOptionSetItem::countby_subscribernumber_option_optionsetitem() . ' rows'; + + $added_count = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOptionSetItem::countby_delta( + $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOptionSetItem::added_delta + ); + $stats .= "\n new: $added_count rows"; + $existing_count = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOptionSetItem::countby_delta( + $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOptionSetItem::updated_delta + ); + $stats .= "\n existing: $existing_count rows"; + $deleted_count = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOptionSetItem::countby_delta( + $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOptionSetItem::deleted_delta + ); + $stats .= "\n removed: $deleted_count rows"; + }; if ($err or !$result) { - push(@$messages,"importing subscriber features incomplete$stats"); + push(@$messages,"importing subscriber features INCOMPLETE$stats"); } else { push(@$messages,"importing subscriber features completed$stats"); } @@ -224,6 +282,32 @@ sub import_features_define_task { } +sub import_truncate_features_task { + + my ($messages) = @_; + my $result = 0; + eval { + $result = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOption::create_table(1); + $result &= NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOptionSetItem::create_table(1); + }; + my $err = $@; + my $stats = ''; + eval { + $stats .= "\n total feature option records: " . + NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOption::countby_subscribernumber_option() . ' rows'; + $stats .= "\n total feature set option item records: " . + NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::FeatureOptionSetItem::countby_subscribernumber_option_optionsetitem() . ' rows'; + }; + if ($err or !$result) { + push(@$messages,"truncating imported subscriber features INCOMPLETE$stats"); + } else { + push(@$messages,"truncating imported subscriber features completed$stats"); + } + destroy_all_dbs(); #every task should leave with closed connections. + return $result; + +} + sub import_subscriber_define_task { my ($messages) = @_; @@ -239,39 +323,18 @@ sub import_subscriber_define_task { my $added_count = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber::countby_delta( $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber::added_delta ); - $stats .= "\n new: $added_count rows"; - if ($added_count <= $stats_record_list_limit) { - foreach my $record (@{NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber::findby_delta( - $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber::added_delta - )}) { - $stats .= "\n " . $record->{dial_number}; - } - } + $stats .= "\n new: $added_count rows"; my $existing_count = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber::countby_delta( $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber::updated_delta ); - $stats .= "\n existing: $existing_count rows"; - if ($existing_count <= $stats_record_list_limit) { - foreach my $record (@{NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber::findby_delta( - $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber::updated_delta - )}) { - $stats .= "\n " . $record->{dial_number}; - } - } + $stats .= "\n existing: $existing_count rows"; my $deleted_count = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber::countby_delta( $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber::deleted_delta ); - $stats .= "\n removed: $deleted_count rows"; - if ($deleted_count <= $stats_record_list_limit) { - foreach my $record (@{NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber::findby_delta( - $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber::deleted_delta - )}) { - $stats .= "\n " . $record->{dial_number}; - } - } + $stats .= "\n removed: $deleted_count rows"; }; if ($err or !$result) { - push(@$messages,"importing subscribers incomplete$stats"); + push(@$messages,"importing subscribers INCOMPLETE$stats"); } else { push(@$messages,"importing subscribers completed$stats"); } @@ -294,7 +357,7 @@ sub import_truncate_subscriber_task { NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Subscriber::countby_subscribernumber() . ' rows'; }; if ($err or !$result) { - push(@$messages,"truncating imported subscribers incomplete$stats"); + push(@$messages,"truncating imported subscribers INCOMPLETE$stats"); } else { push(@$messages,"truncating imported subscribers completed$stats"); } @@ -315,13 +378,168 @@ sub import_lnp_define_task { eval { $stats .= "\n total lnp number records: " . NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Lnp::countby_lrncode_portednumber() . ' rows'; + + my $added_count = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Lnp::countby_delta( + $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Lnp::added_delta + ); + $stats .= "\n new: $added_count rows"; + my $existing_count = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Lnp::countby_delta( + $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Lnp::updated_delta + ); + $stats .= "\n existing: $existing_count rows"; + my $deleted_count = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Lnp::countby_delta( + $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Lnp::deleted_delta + ); + $stats .= "\n removed: $deleted_count rows"; + $stats .= "\n total lrn codes: " . NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Lnp::count_lrncodes(); }; if ($err or !$result) { - push(@$messages,"importing lnp numbers incomplete$stats"); + push(@$messages,"importing lnp numbers INCOMPLETE$stats"); + } else { + push(@$messages,"importing lnp numbers completed$stats"); + } + destroy_all_dbs(); #every task should leave with closed connections. + return $result; + +} + +sub import_truncate_lnp_task { + + my ($messages) = @_; + my $result = 0; + eval { + $result = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Lnp::create_table(1); + }; + my $err = $@; + my $stats = ''; + eval { + $stats .= "\n total lnp number records: " . + NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Lnp::countby_lrncode_portednumber() . ' rows'; + }; + if ($err or !$result) { + push(@$messages,"truncating imported lnp numbers INCOMPLETE$stats"); + } else { + push(@$messages,"truncating imported lnp numbers completed$stats"); + } + destroy_all_dbs(); #every task should leave with closed connections. + return $result; + +} + +sub import_user_password_task { + + my ($messages) = @_; + my $result = 0; + eval { + $result = import_user_password($user_password_filename); + }; + my $err = $@; + my $stats = ''; + eval { + $stats .= "\n total username password records: " . + NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::UsernamePassword::countby_fqdn() . ' rows'; + + my $added_count = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::UsernamePassword::countby_delta( + $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::UsernamePassword::added_delta + ); + $stats .= "\n new: $added_count rows"; + my $existing_count = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::UsernamePassword::countby_delta( + $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::UsernamePassword::updated_delta + ); + $stats .= "\n existing: $existing_count rows"; + my $deleted_count = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::UsernamePassword::countby_delta( + $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::UsernamePassword::deleted_delta + ); + $stats .= "\n removed: $deleted_count rows"; + }; + if ($err or !$result) { + push(@$messages,"importing username passwords INCOMPLETE$stats"); + } else { + push(@$messages,"importing username passwords completed$stats"); + } + destroy_all_dbs(); #every task should leave with closed connections. + return $result; + +} + +sub import_truncate_user_password_task { + + my ($messages) = @_; + my $result = 0; + eval { + $result = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::UsernamePassword::create_table(1); + }; + my $err = $@; + my $stats = ''; + eval { + $stats .= "\n total username password records: " . + NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::UsernamePassword::countby_fqdn() . ' rows'; + }; + if ($err or !$result) { + push(@$messages,"truncating imported username passwords INCOMPLETE$stats"); + } else { + push(@$messages,"truncating imported username passwords completed$stats"); + } + destroy_all_dbs(); #every task should leave with closed connections. + return $result; + +} + +sub import_batch_task { + + my ($messages) = @_; + my $result = 0; + eval { + $result = import_batch($batch_filename); + }; + my $err = $@; + my $stats = ''; + eval { + $stats .= "\n total batch records: " . + NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Batch::countby_number() . ' rows'; + + my $added_count = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Batch::countby_delta( + $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Batch::added_delta + ); + $stats .= "\n new: $added_count rows"; + my $existing_count = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Batch::countby_delta( + $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Batch::updated_delta + ); + $stats .= "\n existing: $existing_count rows"; + my $deleted_count = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Batch::countby_delta( + $NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Batch::deleted_delta + ); + $stats .= "\n removed: $deleted_count rows"; + }; + if ($err or !$result) { + push(@$messages,"importing batch INCOMPLETE$stats"); + } else { + push(@$messages,"importing batch completed$stats"); + } + destroy_all_dbs(); #every task should leave with closed connections. + return $result; + +} + +sub import_truncate_batch_task { + + my ($messages) = @_; + my $result = 0; + eval { + $result = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Batch::create_table(1); + }; + my $err = $@; + my $stats = ''; + eval { + $stats .= "\n total batch records: " . + NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::import::Batch::countby_number() . ' rows'; + }; + if ($err or !$result) { + push(@$messages,"truncating imported batch records INCOMPLETE$stats"); } else { - push(@$messages,"importing lnp numbers$stats"); + push(@$messages,"truncating imported batch records completed$stats"); } destroy_all_dbs(); #every task should leave with closed connections. return $result; diff --git a/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/settings.cfg b/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/settings.cfg index 33175b38..6b77a5ce 100644 --- a/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/settings.cfg +++ b/lib/NGCP/BulkProcessor/Projects/Migration/IPGallery/settings.cfg @@ -1,6 +1,6 @@ #dry=0 -import_multithreading = 1 +import_multithreading = 0 features_define_filename = /home/rkrenn/test/Features_Define.cfg features_define_import_numofthreads = 2 @@ -12,3 +12,9 @@ subscribernumer_exclude_exception_pattern = ^35627702770$ lnp_define_filename = /home/rkrenn/test/LNP_Define.cfg lnp_define_import_numofthreads = 2 + +user_password_filename = /home/rkrenn/test/username_passwords.txt +user_password_import_numofthreads = 2 + +batch_filename = /home/rkrenn/test/blah.txt +batch_import_numofthreads = 2