MT#18663 MT#20893 row bulk processing framework WIP #5
+setting to drop duplicate setoptionitems in Feature_Define.cfg +adjust Feature_Define.cfg threads vs block size settings to arrange with a low sqlite busy_timeout +Subscriber_Define.cfg importer +resolve garbage collector problems with log4perl +refactoring to conform with perl module lib/folder structure for installation -runs again Change-Id: I1821cc0085086684c3c1415be4c262453509045achanges/13/6913/6
Rene Krenn
9 years ago
committed by
Victor Seva
parent
cc3e93d77b
commit
52d66ca0de
@ -1,6 +0,0 @@
|
|||||||
package NoSqlConnector;
|
|
||||||
use strict;
|
|
||||||
|
|
||||||
## no critic
|
|
||||||
|
|
||||||
1;
|
|
||||||
@ -1,126 +0,0 @@
|
|||||||
package Projects::Migration::IPGallery::Import;
|
|
||||||
use strict;
|
|
||||||
|
|
||||||
## no critic
|
|
||||||
|
|
||||||
use File::Basename;
|
|
||||||
use Cwd;
|
|
||||||
use lib Cwd::abs_path(File::Basename::dirname(__FILE__) . '/../../../');
|
|
||||||
|
|
||||||
use Globals qw(
|
|
||||||
$cpucount
|
|
||||||
);
|
|
||||||
use Projects::Migration::IPGallery::Settings qw(
|
|
||||||
$import_multithreading
|
|
||||||
$feature_define_import_numofthreads
|
|
||||||
$dry
|
|
||||||
);
|
|
||||||
use Logging qw (
|
|
||||||
getlogger
|
|
||||||
);
|
|
||||||
use LogError qw(
|
|
||||||
fileprocessingwarn
|
|
||||||
fileprocessingerror
|
|
||||||
);
|
|
||||||
|
|
||||||
#use FileProcessors::CSVFile;
|
|
||||||
use Projects::Migration::IPGallery::FileProcessors::FeaturesDefineFile qw();
|
|
||||||
use Projects::Migration::IPGallery::FeaturesDefineParser qw();
|
|
||||||
|
|
||||||
use Projects::Migration::IPGallery::ProjectConnectorPool qw(
|
|
||||||
get_import_db
|
|
||||||
);
|
|
||||||
|
|
||||||
use Projects::Migration::IPGallery::Dao::FeatureOption qw();
|
|
||||||
use Projects::Migration::IPGallery::Dao::FeatureOptionSet qw();
|
|
||||||
|
|
||||||
use Array qw(removeduplicates);
|
|
||||||
|
|
||||||
require Exporter;
|
|
||||||
our @ISA = qw(Exporter);
|
|
||||||
our @EXPORT_OK = qw(
|
|
||||||
import_features_define
|
|
||||||
);
|
|
||||||
|
|
||||||
|
|
||||||
sub import_features_define {
|
|
||||||
|
|
||||||
my ($file) = @_;
|
|
||||||
my $result = Projects::Migration::IPGallery::Dao::FeatureOption::create_table(1);
|
|
||||||
$result &= Projects::Migration::IPGallery::Dao::FeatureOptionSet::create_table(1);
|
|
||||||
my $importer = Projects::Migration::IPGallery::FileProcessors::FeaturesDefineFile->new($feature_define_import_numofthreads);
|
|
||||||
$importer->{stoponparseerrors} = !$dry;
|
|
||||||
return $result && $importer->process($file,sub {
|
|
||||||
my ($context,$rows,$row_offset) = @_;
|
|
||||||
my $rownum = $row_offset;
|
|
||||||
my @featureoption_rows = ();
|
|
||||||
my @featureoptionset_rows = ();
|
|
||||||
foreach my $line (@$rows) {
|
|
||||||
my $row = undef;
|
|
||||||
if (not $importer->{parselines}) {
|
|
||||||
eval {
|
|
||||||
$row = Projects::Migration::IPGallery::FeaturesDefineParser::parse(\$line,$context->{grammar});
|
|
||||||
};
|
|
||||||
if ($@) {
|
|
||||||
if ($importer->{stoponparseerrors}) {
|
|
||||||
fileprocessingerror($context->{filename},'record ' . ($rownum + 1) . ' - ' . $@,getlogger(__PACKAGE__));
|
|
||||||
} else {
|
|
||||||
fileprocessingwarn($context->{filename},'record ' . ($rownum + 1) . ' - ' . $@,getlogger(__PACKAGE__));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
next unless defined $row;
|
|
||||||
$rownum++;
|
|
||||||
foreach my $subscriber_number (keys %$row) {
|
|
||||||
foreach my $option (@{$row->{$subscriber_number}}) {
|
|
||||||
if ('HASH' eq ref $option) {
|
|
||||||
foreach my $setoption (keys %$option) {
|
|
||||||
foreach my $setoptionitem (@{removeduplicates($option->{$setoption})}) {
|
|
||||||
push(@featureoptionset_rows,[ $subscriber_number, $setoption, $setoptionitem ]);
|
|
||||||
}
|
|
||||||
push(@featureoption_rows,[ $subscriber_number, $setoption ]);
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
push(@featureoption_rows,[ $subscriber_number, $option ]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
my $import_db = &get_import_db();
|
|
||||||
if ((scalar @featureoption_rows) > 0) {
|
|
||||||
$import_db->db_do_begin(
|
|
||||||
Projects::Migration::IPGallery::Dao::FeatureOption::getinsertstatement(),
|
|
||||||
Projects::Migration::IPGallery::Dao::FeatureOption::gettablename(),
|
|
||||||
#lock - $import_multithreading
|
|
||||||
);
|
|
||||||
$import_db->db_do_rowblock(\@featureoption_rows);
|
|
||||||
$import_db->db_finish();
|
|
||||||
}
|
|
||||||
if ((scalar @featureoptionset_rows) > 0) {
|
|
||||||
$import_db->db_do_begin(
|
|
||||||
Projects::Migration::IPGallery::Dao::FeatureOptionSet::getinsertstatement(),
|
|
||||||
Projects::Migration::IPGallery::Dao::FeatureOptionSet::gettablename(),
|
|
||||||
#lock
|
|
||||||
);
|
|
||||||
$import_db->db_do_rowblock(\@featureoptionset_rows);
|
|
||||||
$import_db->db_finish();
|
|
||||||
}
|
|
||||||
return 1;
|
|
||||||
}, sub {
|
|
||||||
my ($context)= @_;
|
|
||||||
if (not $importer->{parselines}) {
|
|
||||||
eval {
|
|
||||||
$context->{grammar} = Projects::Migration::IPGallery::FeaturesDefineParser::create_grammar();
|
|
||||||
};
|
|
||||||
if ($@) {
|
|
||||||
fileprocessingerror($context->{filename},$@,getlogger(__PACKAGE__));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},$import_multithreading);
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
1;
|
|
||||||
@ -1,7 +0,0 @@
|
|||||||
|
|
||||||
features_define_filename = /home/rkrenn/test/Features_Define.cfg
|
|
||||||
|
|
||||||
import_multithreading = 1
|
|
||||||
feature_define_import_numofthreads = 10
|
|
||||||
|
|
||||||
#dry=0
|
|
||||||
@ -1,105 +0,0 @@
|
|||||||
#!/usr/bin/perl
|
|
||||||
use warnings;
|
|
||||||
use strict;
|
|
||||||
|
|
||||||
use Marpa::R2;
|
|
||||||
use Data::Dumper;
|
|
||||||
|
|
||||||
|
|
||||||
my $input = do { local $/; <DATA> };
|
|
||||||
|
|
||||||
my $dsl = << '__GRAMMAR__';
|
|
||||||
|
|
||||||
lexeme default = latm => 1
|
|
||||||
|
|
||||||
:start ::= List
|
|
||||||
:default ::= action => ::first
|
|
||||||
|
|
||||||
List ::= Hash+ action => list
|
|
||||||
Hash ::= String '{' Pairs '}' action => hash
|
|
||||||
Pairs ::= Pair+ action => list
|
|
||||||
Pair ::= String Value ';' action => pair
|
|
||||||
| Hash
|
|
||||||
Value ::= Simple
|
|
||||||
| Bracketed
|
|
||||||
Bracketed ::= '[' String ']' action => second
|
|
||||||
Simple ::= String
|
|
||||||
|
|
||||||
String ~ [-a-zA-Z_0-9]+
|
|
||||||
whitespace ~ [\s] +
|
|
||||||
:discard ~ whitespace
|
|
||||||
|
|
||||||
__GRAMMAR__
|
|
||||||
|
|
||||||
sub hash { +{ $_[1] => $_[3] } }
|
|
||||||
|
|
||||||
sub pair { +{ $_[1] => $_[2] } }
|
|
||||||
|
|
||||||
sub second { [ @_[ 2 .. $#_-1 ] ] }
|
|
||||||
|
|
||||||
sub list { shift; \@_ }
|
|
||||||
|
|
||||||
my $grammar = Marpa::R2::Scanless::G->new( { source => \$dsl } );
|
|
||||||
my $recce = Marpa::R2::Scanless::R->new(
|
|
||||||
{ grammar => $grammar, semantics_package => 'main' } );
|
|
||||||
#my $input = '42 * 1 + 7';
|
|
||||||
$recce->read( \$input );
|
|
||||||
|
|
||||||
my $value_ref = $recce->value;
|
|
||||||
my $value = $value_ref ? ${$value_ref} : 'No Parse';
|
|
||||||
|
|
||||||
print Dumper $value;
|
|
||||||
|
|
||||||
#my $parser = 'Marpa::R2::Scanless::G'->new({ source => \$grammar });
|
|
||||||
|
|
||||||
#print Dumper $parser->parse(\$input, 'main', { trace_terminals => 1 });
|
|
||||||
|
|
||||||
__DATA__
|
|
||||||
bob {
|
|
||||||
ed {
|
|
||||||
larry {
|
|
||||||
rule5 {
|
|
||||||
option {
|
|
||||||
disable-server-response-inspection no;
|
|
||||||
}
|
|
||||||
tag [ some_tag ];
|
|
||||||
from [ prod-L3 ];
|
|
||||||
to [ corp-L3 ];
|
|
||||||
source [ any ];
|
|
||||||
destination [ any ];
|
|
||||||
source-user [ any ];
|
|
||||||
category [ any ];
|
|
||||||
application [ any ];
|
|
||||||
service [ any ];
|
|
||||||
hip-profiles [ any ];
|
|
||||||
log-start no;
|
|
||||||
log-end yes;
|
|
||||||
negate-source no;
|
|
||||||
negate-destination no;
|
|
||||||
action allow;
|
|
||||||
log-setting orion_log;
|
|
||||||
}
|
|
||||||
rule6 {
|
|
||||||
option {
|
|
||||||
disable-server-response-inspection no;
|
|
||||||
}
|
|
||||||
tag [ some_tag ];
|
|
||||||
from [ prod-L3 ];
|
|
||||||
to [ corp-L3 ];
|
|
||||||
source [ any ];
|
|
||||||
destination [ any ];
|
|
||||||
source-user [ any ];
|
|
||||||
category [ any ];
|
|
||||||
application [ any ];
|
|
||||||
service [ any ];
|
|
||||||
hip-profiles [ any ];
|
|
||||||
log-start no;
|
|
||||||
log-end yes;
|
|
||||||
negate-source no;
|
|
||||||
negate-destination no;
|
|
||||||
action allow;
|
|
||||||
log-setting orion_log;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -1,62 +0,0 @@
|
|||||||
|
|
||||||
use strict;
|
|
||||||
|
|
||||||
my $buffersize = 100 * 1024;
|
|
||||||
my $default_encoding = 'UTF-8';
|
|
||||||
|
|
||||||
_get_linecount('/home/rkrenn/test/Features_Define.cfg',$default_encoding,\&breaklines);
|
|
||||||
|
|
||||||
exit;
|
|
||||||
|
|
||||||
sub breaklines {
|
|
||||||
my ($buffer_ref) = @_;
|
|
||||||
my $spearator = "\n";
|
|
||||||
my $count = 0;
|
|
||||||
my $last_record;
|
|
||||||
my $records = [];
|
|
||||||
foreach my $record (split(/$spearator(?=(?:\d+$spearator))/,$$buffer_ref)) {
|
|
||||||
$count++;
|
|
||||||
$last_record = $record;
|
|
||||||
push(@$records,$record);
|
|
||||||
}
|
|
||||||
#if ($last_record =~ /$spearator\}\s*$/) {
|
|
||||||
# $$buffer_ref = '';
|
|
||||||
#} else {
|
|
||||||
$count--;
|
|
||||||
$$buffer_ref = $last_record;
|
|
||||||
pop @$records;
|
|
||||||
#}
|
|
||||||
return $count;
|
|
||||||
}
|
|
||||||
|
|
||||||
sub _get_linecount {
|
|
||||||
|
|
||||||
my ($file,$encoding,$breaklines_code) = @_;
|
|
||||||
|
|
||||||
#local $/ = $lineseparator;
|
|
||||||
local *INPUTFILE_LINECOUNT;
|
|
||||||
if (not open (INPUTFILE_LINECOUNT, '<:encoding(' . $encoding . ')', $file)) {
|
|
||||||
print('get line count - cannot open file ' . $file . ': ' . $!);
|
|
||||||
return undef;
|
|
||||||
}
|
|
||||||
binmode INPUTFILE_LINECOUNT;
|
|
||||||
|
|
||||||
my $linecount = 0;
|
|
||||||
|
|
||||||
my $buffer = '';
|
|
||||||
my $chunk = undef;
|
|
||||||
my $n = 0;
|
|
||||||
while (defined ($n = read(INPUTFILE_LINECOUNT,$chunk,$buffersize)) && $n != 0) {
|
|
||||||
$buffer .= $chunk;
|
|
||||||
$linecount += &$breaklines_code(\$buffer);
|
|
||||||
}
|
|
||||||
if (not defined $n) {
|
|
||||||
print('get line count - error reading file ' . $file . ': ' . $!);
|
|
||||||
close(INPUTFILE_LINECOUNT);
|
|
||||||
return undef;
|
|
||||||
}
|
|
||||||
close(INPUTFILE_LINECOUNT);
|
|
||||||
|
|
||||||
return $linecount;
|
|
||||||
|
|
||||||
}
|
|
||||||
@ -0,0 +1,6 @@
|
|||||||
|
package NGCP::BulkProcessor::NoSqlConnector;
|
||||||
|
use strict;
|
||||||
|
|
||||||
|
## no critic
|
||||||
|
|
||||||
|
1;
|
||||||
@ -0,0 +1,133 @@
|
|||||||
|
package NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::Subscriber;
|
||||||
|
use strict;
|
||||||
|
|
||||||
|
## no critic
|
||||||
|
|
||||||
|
#use File::Basename;
|
||||||
|
#use Cwd;
|
||||||
|
#use lib Cwd::abs_path(File::Basename::dirname(__FILE__) . '/../../../../');
|
||||||
|
|
||||||
|
use NGCP::BulkProcessor::Projects::Migration::IPGallery::ProjectConnectorPool qw(
|
||||||
|
get_import_db
|
||||||
|
|
||||||
|
);
|
||||||
|
#import_db_tableidentifier
|
||||||
|
|
||||||
|
use NGCP::BulkProcessor::SqlRecord qw(
|
||||||
|
registertableinfo
|
||||||
|
create_targettable
|
||||||
|
checktableinfo
|
||||||
|
copy_row
|
||||||
|
|
||||||
|
insert_stmt
|
||||||
|
);
|
||||||
|
|
||||||
|
require Exporter;
|
||||||
|
our @ISA = qw(Exporter NGCP::BulkProcessor::SqlRecord);
|
||||||
|
our @EXPORT_OK = qw(
|
||||||
|
create_table
|
||||||
|
gettablename
|
||||||
|
check_table
|
||||||
|
getinsertstatement
|
||||||
|
|
||||||
|
test_table_bycolumn1
|
||||||
|
test_table_local_select
|
||||||
|
test_table_source_select
|
||||||
|
test_table_source_select_temptable
|
||||||
|
);
|
||||||
|
|
||||||
|
my $tablename = 'subscriber';
|
||||||
|
my $get_db = \&get_import_db;
|
||||||
|
#my $get_tablename = \&import_db_tableidentifier;
|
||||||
|
|
||||||
|
|
||||||
|
my $expected_fieldnames = [
|
||||||
|
'country_code', #356
|
||||||
|
'area_code', #None
|
||||||
|
'dial_number', #35627883323
|
||||||
|
'rgw_fqdn', #35627883323
|
||||||
|
'port', #None
|
||||||
|
'region_name', #None
|
||||||
|
'carrier_code', #None
|
||||||
|
'time_zone_name', #malta
|
||||||
|
'lang_code', #eng
|
||||||
|
'barring_profile', #None
|
||||||
|
];
|
||||||
|
|
||||||
|
my $primarykey_fieldnames = [ 'country_code', 'area_code', 'dial_number' ];
|
||||||
|
|
||||||
|
my $indexes = {};
|
||||||
|
|
||||||
|
my $fixtable_statements = [];
|
||||||
|
|
||||||
|
sub new {
|
||||||
|
|
||||||
|
my $class = shift;
|
||||||
|
my $self = NGCP::BulkProcessor::SqlRecord->new($get_db,
|
||||||
|
$tablename,
|
||||||
|
$expected_fieldnames,$indexes);
|
||||||
|
|
||||||
|
bless($self,$class);
|
||||||
|
|
||||||
|
copy_row($self,shift,$expected_fieldnames);
|
||||||
|
|
||||||
|
return $self;
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
sub create_table {
|
||||||
|
|
||||||
|
my ($truncate) = @_;
|
||||||
|
|
||||||
|
my $db = &$get_db();
|
||||||
|
|
||||||
|
registertableinfo($db,$tablename,$expected_fieldnames,$indexes,$primarykey_fieldnames);
|
||||||
|
return create_targettable($db,$tablename,$db,$tablename,$truncate,0,undef);
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
sub buildrecords_fromrows {
|
||||||
|
|
||||||
|
my ($rows,$load_recursive) = @_;
|
||||||
|
|
||||||
|
my @records = ();
|
||||||
|
my $record;
|
||||||
|
|
||||||
|
if (defined $rows and ref $rows eq 'ARRAY') {
|
||||||
|
foreach my $row (@$rows) {
|
||||||
|
$record = __PACKAGE__->new($row);
|
||||||
|
|
||||||
|
# transformations go here ...
|
||||||
|
|
||||||
|
push @records,$record;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return \@records;
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
sub getinsertstatement {
|
||||||
|
|
||||||
|
check_table();
|
||||||
|
return insert_stmt($get_db,$tablename);
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
sub gettablename {
|
||||||
|
|
||||||
|
return $tablename;
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
sub check_table {
|
||||||
|
|
||||||
|
return checktableinfo($get_db,
|
||||||
|
$tablename,
|
||||||
|
$expected_fieldnames,
|
||||||
|
$indexes);
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
1;
|
||||||
@ -1,11 +1,11 @@
|
|||||||
package Projects::Migration::IPGallery::FeaturesDefineParser;
|
package NGCP::BulkProcessor::Projects::Migration::IPGallery::FeaturesDefineParser;
|
||||||
use strict;
|
use strict;
|
||||||
|
|
||||||
## no critic
|
## no critic
|
||||||
|
|
||||||
use File::Basename;
|
#use File::Basename;
|
||||||
use Cwd;
|
#use Cwd;
|
||||||
use lib Cwd::abs_path(File::Basename::dirname(__FILE__) . '/../../../');
|
#use lib Cwd::abs_path(File::Basename::dirname(__FILE__) . '/../../../');
|
||||||
|
|
||||||
use Marpa::R2;
|
use Marpa::R2;
|
||||||
use Data::Dumper::Concise;
|
use Data::Dumper::Concise;
|
||||||
@ -0,0 +1,66 @@
|
|||||||
|
package NGCP::BulkProcessor::Projects::Migration::IPGallery::FileProcessors::SubscriberDefineFile;
|
||||||
|
use strict;
|
||||||
|
|
||||||
|
## no critic
|
||||||
|
|
||||||
|
#use File::Basename;
|
||||||
|
#use Cwd;
|
||||||
|
#use lib Cwd::abs_path(File::Basename::dirname(__FILE__) . '/../../../../');
|
||||||
|
|
||||||
|
use NGCP::BulkProcessor::Logging qw(
|
||||||
|
getlogger
|
||||||
|
);
|
||||||
|
use NGCP::BulkProcessor::LogError qw(
|
||||||
|
fileprocessingerror
|
||||||
|
fileprocessingwarn
|
||||||
|
);
|
||||||
|
|
||||||
|
use NGCP::BulkProcessor::FileProcessor;
|
||||||
|
|
||||||
|
require Exporter;
|
||||||
|
our @ISA = qw(Exporter NGCP::BulkProcessor::FileProcessor);
|
||||||
|
our @EXPORT_OK = qw();
|
||||||
|
|
||||||
|
my $lineseparator = '\\n';
|
||||||
|
my $fieldseparator = " +";
|
||||||
|
my $encoding = 'UTF-8';
|
||||||
|
|
||||||
|
my $buffersize = 100 * 1024;
|
||||||
|
my $threadqueuelength = 10;
|
||||||
|
my $default_numofthreads = 3;
|
||||||
|
#my $multithreading = 0;
|
||||||
|
my $blocksize = 100;
|
||||||
|
|
||||||
|
sub new {
|
||||||
|
|
||||||
|
my $class = shift;
|
||||||
|
|
||||||
|
my $self = NGCP::BulkProcessor::FileProcessor->new(@_);
|
||||||
|
|
||||||
|
$self->{numofthreads} = shift // $default_numofthreads;
|
||||||
|
$self->{line_separator} = $lineseparator;
|
||||||
|
$self->{field_separator} = $fieldseparator;
|
||||||
|
$self->{encoding} = $encoding;
|
||||||
|
$self->{buffersize} = $buffersize;
|
||||||
|
$self->{threadqueuelength} = $threadqueuelength;
|
||||||
|
#$self->{multithreading} = $multithreading;
|
||||||
|
$self->{blocksize} = $blocksize;
|
||||||
|
|
||||||
|
bless($self,$class);
|
||||||
|
|
||||||
|
#restdebug($self,__PACKAGE__ . ' file processor created',getlogger(__PACKAGE__));
|
||||||
|
|
||||||
|
return $self;
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
sub extractfields {
|
||||||
|
my ($context,$line_ref) = @_;
|
||||||
|
my $separator = $context->{instance}->{field_separator};
|
||||||
|
$$line_ref =~ s/^ +//;
|
||||||
|
$$line_ref =~ s/ +$//;
|
||||||
|
my @fields = split(/$separator/,$$line_ref,-1);
|
||||||
|
return \@fields;
|
||||||
|
}
|
||||||
|
|
||||||
|
1;
|
||||||
@ -0,0 +1,164 @@
|
|||||||
|
package NGCP::BulkProcessor::Projects::Migration::IPGallery::Import;
|
||||||
|
use strict;
|
||||||
|
|
||||||
|
## no critic
|
||||||
|
|
||||||
|
#use File::Basename;
|
||||||
|
#use Cwd;
|
||||||
|
#use lib Cwd::abs_path(File::Basename::dirname(__FILE__) . '/../../../');
|
||||||
|
|
||||||
|
use NGCP::BulkProcessor::Globals qw(
|
||||||
|
$cpucount
|
||||||
|
);
|
||||||
|
use NGCP::BulkProcessor::Projects::Migration::IPGallery::Settings qw(
|
||||||
|
$import_multithreading
|
||||||
|
$features_define_import_numofthreads
|
||||||
|
$skip_duplicate_setoptionitems
|
||||||
|
$subscriber_define_import_numofthreads
|
||||||
|
$dry
|
||||||
|
);
|
||||||
|
use NGCP::BulkProcessor::Logging qw (
|
||||||
|
getlogger
|
||||||
|
);
|
||||||
|
use NGCP::BulkProcessor::LogError qw(
|
||||||
|
fileprocessingwarn
|
||||||
|
fileprocessingerror
|
||||||
|
);
|
||||||
|
|
||||||
|
use NGCP::BulkProcessor::Projects::Migration::IPGallery::FileProcessors::FeaturesDefineFile qw();
|
||||||
|
use NGCP::BulkProcessor::Projects::Migration::IPGallery::FeaturesDefineParser qw();
|
||||||
|
use NGCP::BulkProcessor::Projects::Migration::IPGallery::FileProcessors::SubscriberDefineFile qw();
|
||||||
|
|
||||||
|
use NGCP::BulkProcessor::Projects::Migration::IPGallery::ProjectConnectorPool qw(
|
||||||
|
get_import_db
|
||||||
|
destroy_dbs
|
||||||
|
);
|
||||||
|
|
||||||
|
use NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::FeatureOption qw();
|
||||||
|
use NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::FeatureOptionSet qw();
|
||||||
|
use NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::Subscriber qw();
|
||||||
|
|
||||||
|
use NGCP::BulkProcessor::Array qw(removeduplicates);
|
||||||
|
|
||||||
|
require Exporter;
|
||||||
|
our @ISA = qw(Exporter);
|
||||||
|
our @EXPORT_OK = qw(
|
||||||
|
import_features_define
|
||||||
|
import_subscriber_define
|
||||||
|
);
|
||||||
|
|
||||||
|
sub import_features_define {
|
||||||
|
|
||||||
|
my ($file) = @_;
|
||||||
|
my $result = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::FeatureOption::create_table(1);
|
||||||
|
$result &= NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::FeatureOptionSet::create_table(1);
|
||||||
|
destroy_dbs(); #close all db connections before forking..
|
||||||
|
my $importer = NGCP::BulkProcessor::Projects::Migration::IPGallery::FileProcessors::FeaturesDefineFile->new($features_define_import_numofthreads);
|
||||||
|
$importer->{stoponparseerrors} = !$dry;
|
||||||
|
return $result && $importer->process($file,sub {
|
||||||
|
my ($context,$rows,$row_offset) = @_;
|
||||||
|
my $rownum = $row_offset;
|
||||||
|
my @featureoption_rows = ();
|
||||||
|
my @featureoptionset_rows = ();
|
||||||
|
foreach my $line (@$rows) {
|
||||||
|
my $row = undef;
|
||||||
|
if (not $importer->{parselines}) {
|
||||||
|
eval {
|
||||||
|
$row = NGCP::BulkProcessor::Projects::Migration::IPGallery::FeaturesDefineParser::parse(\$line,$context->{grammar});
|
||||||
|
};
|
||||||
|
if ($@) {
|
||||||
|
if ($importer->{stoponparseerrors}) {
|
||||||
|
fileprocessingerror($context->{filename},'record ' . ($rownum + 1) . ' - ' . $@,getlogger(__PACKAGE__));
|
||||||
|
} else {
|
||||||
|
fileprocessingwarn($context->{filename},'record ' . ($rownum + 1) . ' - ' . $@,getlogger(__PACKAGE__));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
next unless defined $row;
|
||||||
|
$rownum++;
|
||||||
|
foreach my $subscriber_number (keys %$row) {
|
||||||
|
foreach my $option (@{$row->{$subscriber_number}}) {
|
||||||
|
if ('HASH' eq ref $option) {
|
||||||
|
foreach my $setoption (keys %$option) {
|
||||||
|
foreach my $setoptionitem (@{$skip_duplicate_setoptionitems ? removeduplicates($option->{$setoption}) : $option->{$setoption}}) {
|
||||||
|
push(@featureoptionset_rows,[ $subscriber_number, $setoption, $setoptionitem ]);
|
||||||
|
}
|
||||||
|
push(@featureoption_rows,[ $subscriber_number, $setoption ]);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
push(@featureoption_rows,[ $subscriber_number, $option ]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if ((scalar @featureoption_rows) > 0) {
|
||||||
|
$context->{db}->db_do_begin(
|
||||||
|
NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::FeatureOption::getinsertstatement(),
|
||||||
|
NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::FeatureOption::gettablename(),
|
||||||
|
#lock - $import_multithreading
|
||||||
|
);
|
||||||
|
$context->{db}->db_do_rowblock(\@featureoption_rows);
|
||||||
|
$context->{db}->db_finish();
|
||||||
|
}
|
||||||
|
if ((scalar @featureoptionset_rows) > 0) {
|
||||||
|
$context->{db}->db_do_begin(
|
||||||
|
NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::FeatureOptionSet::getinsertstatement(),
|
||||||
|
NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::FeatureOptionSet::gettablename(),
|
||||||
|
#lock
|
||||||
|
);
|
||||||
|
$context->{db}->db_do_rowblock(\@featureoptionset_rows);
|
||||||
|
$context->{db}->db_finish();
|
||||||
|
}
|
||||||
|
return 1;
|
||||||
|
}, sub {
|
||||||
|
my ($context)= @_;
|
||||||
|
if (not $importer->{parselines}) {
|
||||||
|
eval {
|
||||||
|
$context->{grammar} = NGCP::BulkProcessor::Projects::Migration::IPGallery::FeaturesDefineParser::create_grammar();
|
||||||
|
};
|
||||||
|
if ($@) {
|
||||||
|
fileprocessingerror($context->{filename},$@,getlogger(__PACKAGE__));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
$context->{db} = &get_import_db(); # keep ref count low..
|
||||||
|
}, sub {
|
||||||
|
my ($context)= @_;
|
||||||
|
undef $context->{db};
|
||||||
|
destroy_dbs();
|
||||||
|
},$import_multithreading);
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
sub import_subscriber_define {
|
||||||
|
|
||||||
|
my ($file) = @_;
|
||||||
|
my $result = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::Subscriber::create_table(1);
|
||||||
|
my $importer = NGCP::BulkProcessor::Projects::Migration::IPGallery::FileProcessors::SubscriberDefineFile->new($subscriber_define_import_numofthreads);
|
||||||
|
destroy_dbs(); #close all db connections before forking..
|
||||||
|
return $result && $importer->process($file,sub {
|
||||||
|
my ($context,$rows,$row_offset) = @_;
|
||||||
|
|
||||||
|
if ((scalar @$rows) > 0) {
|
||||||
|
$context->{db}->db_do_begin(
|
||||||
|
NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::Subscriber::getinsertstatement(),
|
||||||
|
NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::Subscriber::gettablename(),
|
||||||
|
#lock - $import_multithreading
|
||||||
|
);
|
||||||
|
$context->{db}->db_do_rowblock($rows);
|
||||||
|
$context->{db}->db_finish();
|
||||||
|
}
|
||||||
|
|
||||||
|
return 1;
|
||||||
|
}, sub {
|
||||||
|
my ($context)= @_;
|
||||||
|
$context->{db} = &get_import_db(); # keep ref count low..
|
||||||
|
}, sub {
|
||||||
|
my ($context)= @_;
|
||||||
|
undef $context->{db};
|
||||||
|
destroy_dbs();
|
||||||
|
}, $import_multithreading);
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
1;
|
||||||
@ -0,0 +1,10 @@
|
|||||||
|
|
||||||
|
features_define_filename = /home/rkrenn/test/Features_Define.cfg
|
||||||
|
|
||||||
|
import_multithreading = 1
|
||||||
|
features_define_import_numofthreads = 2
|
||||||
|
|
||||||
|
subscriber_define_filename = /home/rkrenn/test/Subscriber_Define.cfg
|
||||||
|
subscriber_define_import_numofthreads = 2
|
||||||
|
|
||||||
|
#dry=0
|
||||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -1,67 +1,67 @@
|
|||||||
# helper module to implement variance aggregate function with SQLite
|
# helper module to implement variance aggregate function with SQLite
|
||||||
# adamk@cpan.org
|
# adamk@cpan.org
|
||||||
# 2009
|
# 2009
|
||||||
|
|
||||||
# used only for testing custom functions ...
|
# used only for testing custom functions ...
|
||||||
|
|
||||||
package SqlConnectors::SQLiteVarianceAggregate;
|
package NGCP::BulkProcessor::SqlConnectors::SQLiteVarianceAggregate;
|
||||||
use strict;
|
use strict;
|
||||||
|
|
||||||
## no critic
|
## no critic
|
||||||
|
|
||||||
use File::Basename;
|
#use File::Basename;
|
||||||
use Cwd;
|
#use Cwd;
|
||||||
use lib Cwd::abs_path(File::Basename::dirname(__FILE__) . '/../');
|
#use lib Cwd::abs_path(File::Basename::dirname(__FILE__) . '/../');
|
||||||
|
|
||||||
#sub new { bless [], shift; }
|
#sub new { bless [], shift; }
|
||||||
#
|
#
|
||||||
#sub step {
|
#sub step {
|
||||||
# my ( $self, $value ) = @_;
|
# my ( $self, $value ) = @_;
|
||||||
#
|
#
|
||||||
# push @$self, $value;
|
# push @$self, $value;
|
||||||
#}
|
#}
|
||||||
#
|
#
|
||||||
#sub finalize {
|
#sub finalize {
|
||||||
# my $self = $_[0];
|
# my $self = $_[0];
|
||||||
#
|
#
|
||||||
# my $n = @$self;
|
# my $n = @$self;
|
||||||
#
|
#
|
||||||
# # Variance is NULL unless there is more than one row
|
# # Variance is NULL unless there is more than one row
|
||||||
# return undef unless $n || $n == 1;
|
# return undef unless $n || $n == 1;
|
||||||
#
|
#
|
||||||
# my $mu = 0;
|
# my $mu = 0;
|
||||||
# foreach my $v ( @$self ) {
|
# foreach my $v ( @$self ) {
|
||||||
# $mu += $v;
|
# $mu += $v;
|
||||||
# }
|
# }
|
||||||
# $mu /= $n;
|
# $mu /= $n;
|
||||||
#
|
#
|
||||||
# my $sigma = 0;
|
# my $sigma = 0;
|
||||||
# foreach my $v ( @$self ) {
|
# foreach my $v ( @$self ) {
|
||||||
# #$sigma += ($x - $mu)**2;
|
# #$sigma += ($x - $mu)**2;
|
||||||
# $sigma += ($v - $mu)**2;
|
# $sigma += ($v - $mu)**2;
|
||||||
# }
|
# }
|
||||||
# $sigma = $sigma / ($n - 1);
|
# $sigma = $sigma / ($n - 1);
|
||||||
#
|
#
|
||||||
# return $sigma;
|
# return $sigma;
|
||||||
#}
|
#}
|
||||||
|
|
||||||
my $mu = 0;
|
my $mu = 0;
|
||||||
my $count = 0;
|
my $count = 0;
|
||||||
my $S = 0;
|
my $S = 0;
|
||||||
|
|
||||||
sub new { bless [], shift; }
|
sub new { bless [], shift; }
|
||||||
|
|
||||||
sub step {
|
sub step {
|
||||||
my ( $self, $value ) = @_;
|
my ( $self, $value ) = @_;
|
||||||
$count++;
|
$count++;
|
||||||
my $delta = $value - $mu;
|
my $delta = $value - $mu;
|
||||||
$mu += $delta / $count;
|
$mu += $delta / $count;
|
||||||
$S += $delta * ($value - $mu);
|
$S += $delta * ($value - $mu);
|
||||||
}
|
}
|
||||||
|
|
||||||
sub finalize {
|
sub finalize {
|
||||||
my $self = $_[0];
|
my $self = $_[0];
|
||||||
return $S / ($count - 1);
|
return $S / ($count - 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
1;
|
1;
|
||||||
Loading…
Reference in new issue