MT#18663 MT#20893 row bulk processing framework WIP #5
+setting to drop duplicate setoptionitems in Feature_Define.cfg +adjust Feature_Define.cfg threads vs block size settings to arrange with a low sqlite busy_timeout +Subscriber_Define.cfg importer +resolve garbage collector problems with log4perl +refactoring to conform with perl module lib/folder structure for installation -runs again Change-Id: I1821cc0085086684c3c1415be4c262453509045achanges/13/6913/6
Rene Krenn
9 years ago
committed by
Victor Seva
parent
cc3e93d77b
commit
52d66ca0de
@ -1,6 +0,0 @@
|
||||
package NoSqlConnector;
|
||||
use strict;
|
||||
|
||||
## no critic
|
||||
|
||||
1;
|
||||
@ -1,126 +0,0 @@
|
||||
package Projects::Migration::IPGallery::Import;
|
||||
use strict;
|
||||
|
||||
## no critic
|
||||
|
||||
use File::Basename;
|
||||
use Cwd;
|
||||
use lib Cwd::abs_path(File::Basename::dirname(__FILE__) . '/../../../');
|
||||
|
||||
use Globals qw(
|
||||
$cpucount
|
||||
);
|
||||
use Projects::Migration::IPGallery::Settings qw(
|
||||
$import_multithreading
|
||||
$feature_define_import_numofthreads
|
||||
$dry
|
||||
);
|
||||
use Logging qw (
|
||||
getlogger
|
||||
);
|
||||
use LogError qw(
|
||||
fileprocessingwarn
|
||||
fileprocessingerror
|
||||
);
|
||||
|
||||
#use FileProcessors::CSVFile;
|
||||
use Projects::Migration::IPGallery::FileProcessors::FeaturesDefineFile qw();
|
||||
use Projects::Migration::IPGallery::FeaturesDefineParser qw();
|
||||
|
||||
use Projects::Migration::IPGallery::ProjectConnectorPool qw(
|
||||
get_import_db
|
||||
);
|
||||
|
||||
use Projects::Migration::IPGallery::Dao::FeatureOption qw();
|
||||
use Projects::Migration::IPGallery::Dao::FeatureOptionSet qw();
|
||||
|
||||
use Array qw(removeduplicates);
|
||||
|
||||
require Exporter;
|
||||
our @ISA = qw(Exporter);
|
||||
our @EXPORT_OK = qw(
|
||||
import_features_define
|
||||
);
|
||||
|
||||
|
||||
sub import_features_define {
|
||||
|
||||
my ($file) = @_;
|
||||
my $result = Projects::Migration::IPGallery::Dao::FeatureOption::create_table(1);
|
||||
$result &= Projects::Migration::IPGallery::Dao::FeatureOptionSet::create_table(1);
|
||||
my $importer = Projects::Migration::IPGallery::FileProcessors::FeaturesDefineFile->new($feature_define_import_numofthreads);
|
||||
$importer->{stoponparseerrors} = !$dry;
|
||||
return $result && $importer->process($file,sub {
|
||||
my ($context,$rows,$row_offset) = @_;
|
||||
my $rownum = $row_offset;
|
||||
my @featureoption_rows = ();
|
||||
my @featureoptionset_rows = ();
|
||||
foreach my $line (@$rows) {
|
||||
my $row = undef;
|
||||
if (not $importer->{parselines}) {
|
||||
eval {
|
||||
$row = Projects::Migration::IPGallery::FeaturesDefineParser::parse(\$line,$context->{grammar});
|
||||
};
|
||||
if ($@) {
|
||||
if ($importer->{stoponparseerrors}) {
|
||||
fileprocessingerror($context->{filename},'record ' . ($rownum + 1) . ' - ' . $@,getlogger(__PACKAGE__));
|
||||
} else {
|
||||
fileprocessingwarn($context->{filename},'record ' . ($rownum + 1) . ' - ' . $@,getlogger(__PACKAGE__));
|
||||
}
|
||||
}
|
||||
}
|
||||
next unless defined $row;
|
||||
$rownum++;
|
||||
foreach my $subscriber_number (keys %$row) {
|
||||
foreach my $option (@{$row->{$subscriber_number}}) {
|
||||
if ('HASH' eq ref $option) {
|
||||
foreach my $setoption (keys %$option) {
|
||||
foreach my $setoptionitem (@{removeduplicates($option->{$setoption})}) {
|
||||
push(@featureoptionset_rows,[ $subscriber_number, $setoption, $setoptionitem ]);
|
||||
}
|
||||
push(@featureoption_rows,[ $subscriber_number, $setoption ]);
|
||||
}
|
||||
} else {
|
||||
push(@featureoption_rows,[ $subscriber_number, $option ]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
my $import_db = &get_import_db();
|
||||
if ((scalar @featureoption_rows) > 0) {
|
||||
$import_db->db_do_begin(
|
||||
Projects::Migration::IPGallery::Dao::FeatureOption::getinsertstatement(),
|
||||
Projects::Migration::IPGallery::Dao::FeatureOption::gettablename(),
|
||||
#lock - $import_multithreading
|
||||
);
|
||||
$import_db->db_do_rowblock(\@featureoption_rows);
|
||||
$import_db->db_finish();
|
||||
}
|
||||
if ((scalar @featureoptionset_rows) > 0) {
|
||||
$import_db->db_do_begin(
|
||||
Projects::Migration::IPGallery::Dao::FeatureOptionSet::getinsertstatement(),
|
||||
Projects::Migration::IPGallery::Dao::FeatureOptionSet::gettablename(),
|
||||
#lock
|
||||
);
|
||||
$import_db->db_do_rowblock(\@featureoptionset_rows);
|
||||
$import_db->db_finish();
|
||||
}
|
||||
return 1;
|
||||
}, sub {
|
||||
my ($context)= @_;
|
||||
if (not $importer->{parselines}) {
|
||||
eval {
|
||||
$context->{grammar} = Projects::Migration::IPGallery::FeaturesDefineParser::create_grammar();
|
||||
};
|
||||
if ($@) {
|
||||
fileprocessingerror($context->{filename},$@,getlogger(__PACKAGE__));
|
||||
}
|
||||
}
|
||||
},$import_multithreading);
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
1;
|
||||
@ -1,7 +0,0 @@
|
||||
|
||||
features_define_filename = /home/rkrenn/test/Features_Define.cfg
|
||||
|
||||
import_multithreading = 1
|
||||
feature_define_import_numofthreads = 10
|
||||
|
||||
#dry=0
|
||||
@ -1,105 +0,0 @@
|
||||
#!/usr/bin/perl
|
||||
use warnings;
|
||||
use strict;
|
||||
|
||||
use Marpa::R2;
|
||||
use Data::Dumper;
|
||||
|
||||
|
||||
my $input = do { local $/; <DATA> };
|
||||
|
||||
my $dsl = << '__GRAMMAR__';
|
||||
|
||||
lexeme default = latm => 1
|
||||
|
||||
:start ::= List
|
||||
:default ::= action => ::first
|
||||
|
||||
List ::= Hash+ action => list
|
||||
Hash ::= String '{' Pairs '}' action => hash
|
||||
Pairs ::= Pair+ action => list
|
||||
Pair ::= String Value ';' action => pair
|
||||
| Hash
|
||||
Value ::= Simple
|
||||
| Bracketed
|
||||
Bracketed ::= '[' String ']' action => second
|
||||
Simple ::= String
|
||||
|
||||
String ~ [-a-zA-Z_0-9]+
|
||||
whitespace ~ [\s] +
|
||||
:discard ~ whitespace
|
||||
|
||||
__GRAMMAR__
|
||||
|
||||
sub hash { +{ $_[1] => $_[3] } }
|
||||
|
||||
sub pair { +{ $_[1] => $_[2] } }
|
||||
|
||||
sub second { [ @_[ 2 .. $#_-1 ] ] }
|
||||
|
||||
sub list { shift; \@_ }
|
||||
|
||||
my $grammar = Marpa::R2::Scanless::G->new( { source => \$dsl } );
|
||||
my $recce = Marpa::R2::Scanless::R->new(
|
||||
{ grammar => $grammar, semantics_package => 'main' } );
|
||||
#my $input = '42 * 1 + 7';
|
||||
$recce->read( \$input );
|
||||
|
||||
my $value_ref = $recce->value;
|
||||
my $value = $value_ref ? ${$value_ref} : 'No Parse';
|
||||
|
||||
print Dumper $value;
|
||||
|
||||
#my $parser = 'Marpa::R2::Scanless::G'->new({ source => \$grammar });
|
||||
|
||||
#print Dumper $parser->parse(\$input, 'main', { trace_terminals => 1 });
|
||||
|
||||
__DATA__
|
||||
bob {
|
||||
ed {
|
||||
larry {
|
||||
rule5 {
|
||||
option {
|
||||
disable-server-response-inspection no;
|
||||
}
|
||||
tag [ some_tag ];
|
||||
from [ prod-L3 ];
|
||||
to [ corp-L3 ];
|
||||
source [ any ];
|
||||
destination [ any ];
|
||||
source-user [ any ];
|
||||
category [ any ];
|
||||
application [ any ];
|
||||
service [ any ];
|
||||
hip-profiles [ any ];
|
||||
log-start no;
|
||||
log-end yes;
|
||||
negate-source no;
|
||||
negate-destination no;
|
||||
action allow;
|
||||
log-setting orion_log;
|
||||
}
|
||||
rule6 {
|
||||
option {
|
||||
disable-server-response-inspection no;
|
||||
}
|
||||
tag [ some_tag ];
|
||||
from [ prod-L3 ];
|
||||
to [ corp-L3 ];
|
||||
source [ any ];
|
||||
destination [ any ];
|
||||
source-user [ any ];
|
||||
category [ any ];
|
||||
application [ any ];
|
||||
service [ any ];
|
||||
hip-profiles [ any ];
|
||||
log-start no;
|
||||
log-end yes;
|
||||
negate-source no;
|
||||
negate-destination no;
|
||||
action allow;
|
||||
log-setting orion_log;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,62 +0,0 @@
|
||||
|
||||
use strict;
|
||||
|
||||
my $buffersize = 100 * 1024;
|
||||
my $default_encoding = 'UTF-8';
|
||||
|
||||
_get_linecount('/home/rkrenn/test/Features_Define.cfg',$default_encoding,\&breaklines);
|
||||
|
||||
exit;
|
||||
|
||||
sub breaklines {
|
||||
my ($buffer_ref) = @_;
|
||||
my $spearator = "\n";
|
||||
my $count = 0;
|
||||
my $last_record;
|
||||
my $records = [];
|
||||
foreach my $record (split(/$spearator(?=(?:\d+$spearator))/,$$buffer_ref)) {
|
||||
$count++;
|
||||
$last_record = $record;
|
||||
push(@$records,$record);
|
||||
}
|
||||
#if ($last_record =~ /$spearator\}\s*$/) {
|
||||
# $$buffer_ref = '';
|
||||
#} else {
|
||||
$count--;
|
||||
$$buffer_ref = $last_record;
|
||||
pop @$records;
|
||||
#}
|
||||
return $count;
|
||||
}
|
||||
|
||||
sub _get_linecount {
|
||||
|
||||
my ($file,$encoding,$breaklines_code) = @_;
|
||||
|
||||
#local $/ = $lineseparator;
|
||||
local *INPUTFILE_LINECOUNT;
|
||||
if (not open (INPUTFILE_LINECOUNT, '<:encoding(' . $encoding . ')', $file)) {
|
||||
print('get line count - cannot open file ' . $file . ': ' . $!);
|
||||
return undef;
|
||||
}
|
||||
binmode INPUTFILE_LINECOUNT;
|
||||
|
||||
my $linecount = 0;
|
||||
|
||||
my $buffer = '';
|
||||
my $chunk = undef;
|
||||
my $n = 0;
|
||||
while (defined ($n = read(INPUTFILE_LINECOUNT,$chunk,$buffersize)) && $n != 0) {
|
||||
$buffer .= $chunk;
|
||||
$linecount += &$breaklines_code(\$buffer);
|
||||
}
|
||||
if (not defined $n) {
|
||||
print('get line count - error reading file ' . $file . ': ' . $!);
|
||||
close(INPUTFILE_LINECOUNT);
|
||||
return undef;
|
||||
}
|
||||
close(INPUTFILE_LINECOUNT);
|
||||
|
||||
return $linecount;
|
||||
|
||||
}
|
||||
@ -0,0 +1,6 @@
|
||||
package NGCP::BulkProcessor::NoSqlConnector;
|
||||
use strict;
|
||||
|
||||
## no critic
|
||||
|
||||
1;
|
||||
@ -0,0 +1,133 @@
|
||||
package NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::Subscriber;
|
||||
use strict;
|
||||
|
||||
## no critic
|
||||
|
||||
#use File::Basename;
|
||||
#use Cwd;
|
||||
#use lib Cwd::abs_path(File::Basename::dirname(__FILE__) . '/../../../../');
|
||||
|
||||
use NGCP::BulkProcessor::Projects::Migration::IPGallery::ProjectConnectorPool qw(
|
||||
get_import_db
|
||||
|
||||
);
|
||||
#import_db_tableidentifier
|
||||
|
||||
use NGCP::BulkProcessor::SqlRecord qw(
|
||||
registertableinfo
|
||||
create_targettable
|
||||
checktableinfo
|
||||
copy_row
|
||||
|
||||
insert_stmt
|
||||
);
|
||||
|
||||
require Exporter;
|
||||
our @ISA = qw(Exporter NGCP::BulkProcessor::SqlRecord);
|
||||
our @EXPORT_OK = qw(
|
||||
create_table
|
||||
gettablename
|
||||
check_table
|
||||
getinsertstatement
|
||||
|
||||
test_table_bycolumn1
|
||||
test_table_local_select
|
||||
test_table_source_select
|
||||
test_table_source_select_temptable
|
||||
);
|
||||
|
||||
my $tablename = 'subscriber';
|
||||
my $get_db = \&get_import_db;
|
||||
#my $get_tablename = \&import_db_tableidentifier;
|
||||
|
||||
|
||||
my $expected_fieldnames = [
|
||||
'country_code', #356
|
||||
'area_code', #None
|
||||
'dial_number', #35627883323
|
||||
'rgw_fqdn', #35627883323
|
||||
'port', #None
|
||||
'region_name', #None
|
||||
'carrier_code', #None
|
||||
'time_zone_name', #malta
|
||||
'lang_code', #eng
|
||||
'barring_profile', #None
|
||||
];
|
||||
|
||||
my $primarykey_fieldnames = [ 'country_code', 'area_code', 'dial_number' ];
|
||||
|
||||
my $indexes = {};
|
||||
|
||||
my $fixtable_statements = [];
|
||||
|
||||
sub new {
|
||||
|
||||
my $class = shift;
|
||||
my $self = NGCP::BulkProcessor::SqlRecord->new($get_db,
|
||||
$tablename,
|
||||
$expected_fieldnames,$indexes);
|
||||
|
||||
bless($self,$class);
|
||||
|
||||
copy_row($self,shift,$expected_fieldnames);
|
||||
|
||||
return $self;
|
||||
|
||||
}
|
||||
|
||||
sub create_table {
|
||||
|
||||
my ($truncate) = @_;
|
||||
|
||||
my $db = &$get_db();
|
||||
|
||||
registertableinfo($db,$tablename,$expected_fieldnames,$indexes,$primarykey_fieldnames);
|
||||
return create_targettable($db,$tablename,$db,$tablename,$truncate,0,undef);
|
||||
|
||||
}
|
||||
|
||||
|
||||
sub buildrecords_fromrows {
|
||||
|
||||
my ($rows,$load_recursive) = @_;
|
||||
|
||||
my @records = ();
|
||||
my $record;
|
||||
|
||||
if (defined $rows and ref $rows eq 'ARRAY') {
|
||||
foreach my $row (@$rows) {
|
||||
$record = __PACKAGE__->new($row);
|
||||
|
||||
# transformations go here ...
|
||||
|
||||
push @records,$record;
|
||||
}
|
||||
}
|
||||
|
||||
return \@records;
|
||||
|
||||
}
|
||||
|
||||
sub getinsertstatement {
|
||||
|
||||
check_table();
|
||||
return insert_stmt($get_db,$tablename);
|
||||
|
||||
}
|
||||
|
||||
sub gettablename {
|
||||
|
||||
return $tablename;
|
||||
|
||||
}
|
||||
|
||||
sub check_table {
|
||||
|
||||
return checktableinfo($get_db,
|
||||
$tablename,
|
||||
$expected_fieldnames,
|
||||
$indexes);
|
||||
|
||||
}
|
||||
|
||||
1;
|
||||
@ -1,11 +1,11 @@
|
||||
package Projects::Migration::IPGallery::FeaturesDefineParser;
|
||||
package NGCP::BulkProcessor::Projects::Migration::IPGallery::FeaturesDefineParser;
|
||||
use strict;
|
||||
|
||||
## no critic
|
||||
|
||||
use File::Basename;
|
||||
use Cwd;
|
||||
use lib Cwd::abs_path(File::Basename::dirname(__FILE__) . '/../../../');
|
||||
#use File::Basename;
|
||||
#use Cwd;
|
||||
#use lib Cwd::abs_path(File::Basename::dirname(__FILE__) . '/../../../');
|
||||
|
||||
use Marpa::R2;
|
||||
use Data::Dumper::Concise;
|
||||
@ -0,0 +1,66 @@
|
||||
package NGCP::BulkProcessor::Projects::Migration::IPGallery::FileProcessors::SubscriberDefineFile;
|
||||
use strict;
|
||||
|
||||
## no critic
|
||||
|
||||
#use File::Basename;
|
||||
#use Cwd;
|
||||
#use lib Cwd::abs_path(File::Basename::dirname(__FILE__) . '/../../../../');
|
||||
|
||||
use NGCP::BulkProcessor::Logging qw(
|
||||
getlogger
|
||||
);
|
||||
use NGCP::BulkProcessor::LogError qw(
|
||||
fileprocessingerror
|
||||
fileprocessingwarn
|
||||
);
|
||||
|
||||
use NGCP::BulkProcessor::FileProcessor;
|
||||
|
||||
require Exporter;
|
||||
our @ISA = qw(Exporter NGCP::BulkProcessor::FileProcessor);
|
||||
our @EXPORT_OK = qw();
|
||||
|
||||
my $lineseparator = '\\n';
|
||||
my $fieldseparator = " +";
|
||||
my $encoding = 'UTF-8';
|
||||
|
||||
my $buffersize = 100 * 1024;
|
||||
my $threadqueuelength = 10;
|
||||
my $default_numofthreads = 3;
|
||||
#my $multithreading = 0;
|
||||
my $blocksize = 100;
|
||||
|
||||
sub new {
|
||||
|
||||
my $class = shift;
|
||||
|
||||
my $self = NGCP::BulkProcessor::FileProcessor->new(@_);
|
||||
|
||||
$self->{numofthreads} = shift // $default_numofthreads;
|
||||
$self->{line_separator} = $lineseparator;
|
||||
$self->{field_separator} = $fieldseparator;
|
||||
$self->{encoding} = $encoding;
|
||||
$self->{buffersize} = $buffersize;
|
||||
$self->{threadqueuelength} = $threadqueuelength;
|
||||
#$self->{multithreading} = $multithreading;
|
||||
$self->{blocksize} = $blocksize;
|
||||
|
||||
bless($self,$class);
|
||||
|
||||
#restdebug($self,__PACKAGE__ . ' file processor created',getlogger(__PACKAGE__));
|
||||
|
||||
return $self;
|
||||
|
||||
}
|
||||
|
||||
sub extractfields {
|
||||
my ($context,$line_ref) = @_;
|
||||
my $separator = $context->{instance}->{field_separator};
|
||||
$$line_ref =~ s/^ +//;
|
||||
$$line_ref =~ s/ +$//;
|
||||
my @fields = split(/$separator/,$$line_ref,-1);
|
||||
return \@fields;
|
||||
}
|
||||
|
||||
1;
|
||||
@ -0,0 +1,164 @@
|
||||
package NGCP::BulkProcessor::Projects::Migration::IPGallery::Import;
|
||||
use strict;
|
||||
|
||||
## no critic
|
||||
|
||||
#use File::Basename;
|
||||
#use Cwd;
|
||||
#use lib Cwd::abs_path(File::Basename::dirname(__FILE__) . '/../../../');
|
||||
|
||||
use NGCP::BulkProcessor::Globals qw(
|
||||
$cpucount
|
||||
);
|
||||
use NGCP::BulkProcessor::Projects::Migration::IPGallery::Settings qw(
|
||||
$import_multithreading
|
||||
$features_define_import_numofthreads
|
||||
$skip_duplicate_setoptionitems
|
||||
$subscriber_define_import_numofthreads
|
||||
$dry
|
||||
);
|
||||
use NGCP::BulkProcessor::Logging qw (
|
||||
getlogger
|
||||
);
|
||||
use NGCP::BulkProcessor::LogError qw(
|
||||
fileprocessingwarn
|
||||
fileprocessingerror
|
||||
);
|
||||
|
||||
use NGCP::BulkProcessor::Projects::Migration::IPGallery::FileProcessors::FeaturesDefineFile qw();
|
||||
use NGCP::BulkProcessor::Projects::Migration::IPGallery::FeaturesDefineParser qw();
|
||||
use NGCP::BulkProcessor::Projects::Migration::IPGallery::FileProcessors::SubscriberDefineFile qw();
|
||||
|
||||
use NGCP::BulkProcessor::Projects::Migration::IPGallery::ProjectConnectorPool qw(
|
||||
get_import_db
|
||||
destroy_dbs
|
||||
);
|
||||
|
||||
use NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::FeatureOption qw();
|
||||
use NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::FeatureOptionSet qw();
|
||||
use NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::Subscriber qw();
|
||||
|
||||
use NGCP::BulkProcessor::Array qw(removeduplicates);
|
||||
|
||||
require Exporter;
|
||||
our @ISA = qw(Exporter);
|
||||
our @EXPORT_OK = qw(
|
||||
import_features_define
|
||||
import_subscriber_define
|
||||
);
|
||||
|
||||
sub import_features_define {
|
||||
|
||||
my ($file) = @_;
|
||||
my $result = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::FeatureOption::create_table(1);
|
||||
$result &= NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::FeatureOptionSet::create_table(1);
|
||||
destroy_dbs(); #close all db connections before forking..
|
||||
my $importer = NGCP::BulkProcessor::Projects::Migration::IPGallery::FileProcessors::FeaturesDefineFile->new($features_define_import_numofthreads);
|
||||
$importer->{stoponparseerrors} = !$dry;
|
||||
return $result && $importer->process($file,sub {
|
||||
my ($context,$rows,$row_offset) = @_;
|
||||
my $rownum = $row_offset;
|
||||
my @featureoption_rows = ();
|
||||
my @featureoptionset_rows = ();
|
||||
foreach my $line (@$rows) {
|
||||
my $row = undef;
|
||||
if (not $importer->{parselines}) {
|
||||
eval {
|
||||
$row = NGCP::BulkProcessor::Projects::Migration::IPGallery::FeaturesDefineParser::parse(\$line,$context->{grammar});
|
||||
};
|
||||
if ($@) {
|
||||
if ($importer->{stoponparseerrors}) {
|
||||
fileprocessingerror($context->{filename},'record ' . ($rownum + 1) . ' - ' . $@,getlogger(__PACKAGE__));
|
||||
} else {
|
||||
fileprocessingwarn($context->{filename},'record ' . ($rownum + 1) . ' - ' . $@,getlogger(__PACKAGE__));
|
||||
}
|
||||
}
|
||||
}
|
||||
next unless defined $row;
|
||||
$rownum++;
|
||||
foreach my $subscriber_number (keys %$row) {
|
||||
foreach my $option (@{$row->{$subscriber_number}}) {
|
||||
if ('HASH' eq ref $option) {
|
||||
foreach my $setoption (keys %$option) {
|
||||
foreach my $setoptionitem (@{$skip_duplicate_setoptionitems ? removeduplicates($option->{$setoption}) : $option->{$setoption}}) {
|
||||
push(@featureoptionset_rows,[ $subscriber_number, $setoption, $setoptionitem ]);
|
||||
}
|
||||
push(@featureoption_rows,[ $subscriber_number, $setoption ]);
|
||||
}
|
||||
} else {
|
||||
push(@featureoption_rows,[ $subscriber_number, $option ]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ((scalar @featureoption_rows) > 0) {
|
||||
$context->{db}->db_do_begin(
|
||||
NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::FeatureOption::getinsertstatement(),
|
||||
NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::FeatureOption::gettablename(),
|
||||
#lock - $import_multithreading
|
||||
);
|
||||
$context->{db}->db_do_rowblock(\@featureoption_rows);
|
||||
$context->{db}->db_finish();
|
||||
}
|
||||
if ((scalar @featureoptionset_rows) > 0) {
|
||||
$context->{db}->db_do_begin(
|
||||
NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::FeatureOptionSet::getinsertstatement(),
|
||||
NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::FeatureOptionSet::gettablename(),
|
||||
#lock
|
||||
);
|
||||
$context->{db}->db_do_rowblock(\@featureoptionset_rows);
|
||||
$context->{db}->db_finish();
|
||||
}
|
||||
return 1;
|
||||
}, sub {
|
||||
my ($context)= @_;
|
||||
if (not $importer->{parselines}) {
|
||||
eval {
|
||||
$context->{grammar} = NGCP::BulkProcessor::Projects::Migration::IPGallery::FeaturesDefineParser::create_grammar();
|
||||
};
|
||||
if ($@) {
|
||||
fileprocessingerror($context->{filename},$@,getlogger(__PACKAGE__));
|
||||
}
|
||||
}
|
||||
$context->{db} = &get_import_db(); # keep ref count low..
|
||||
}, sub {
|
||||
my ($context)= @_;
|
||||
undef $context->{db};
|
||||
destroy_dbs();
|
||||
},$import_multithreading);
|
||||
|
||||
}
|
||||
|
||||
sub import_subscriber_define {
|
||||
|
||||
my ($file) = @_;
|
||||
my $result = NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::Subscriber::create_table(1);
|
||||
my $importer = NGCP::BulkProcessor::Projects::Migration::IPGallery::FileProcessors::SubscriberDefineFile->new($subscriber_define_import_numofthreads);
|
||||
destroy_dbs(); #close all db connections before forking..
|
||||
return $result && $importer->process($file,sub {
|
||||
my ($context,$rows,$row_offset) = @_;
|
||||
|
||||
if ((scalar @$rows) > 0) {
|
||||
$context->{db}->db_do_begin(
|
||||
NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::Subscriber::getinsertstatement(),
|
||||
NGCP::BulkProcessor::Projects::Migration::IPGallery::Dao::Subscriber::gettablename(),
|
||||
#lock - $import_multithreading
|
||||
);
|
||||
$context->{db}->db_do_rowblock($rows);
|
||||
$context->{db}->db_finish();
|
||||
}
|
||||
|
||||
return 1;
|
||||
}, sub {
|
||||
my ($context)= @_;
|
||||
$context->{db} = &get_import_db(); # keep ref count low..
|
||||
}, sub {
|
||||
my ($context)= @_;
|
||||
undef $context->{db};
|
||||
destroy_dbs();
|
||||
}, $import_multithreading);
|
||||
|
||||
}
|
||||
|
||||
1;
|
||||
@ -0,0 +1,10 @@
|
||||
|
||||
features_define_filename = /home/rkrenn/test/Features_Define.cfg
|
||||
|
||||
import_multithreading = 1
|
||||
features_define_import_numofthreads = 2
|
||||
|
||||
subscriber_define_filename = /home/rkrenn/test/Subscriber_Define.cfg
|
||||
subscriber_define_import_numofthreads = 2
|
||||
|
||||
#dry=0
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -1,67 +1,67 @@
|
||||
# helper module to implement variance aggregate function with SQLite
|
||||
# adamk@cpan.org
|
||||
# 2009
|
||||
|
||||
# used only for testing custom functions ...
|
||||
|
||||
package SqlConnectors::SQLiteVarianceAggregate;
|
||||
use strict;
|
||||
|
||||
## no critic
|
||||
|
||||
use File::Basename;
|
||||
use Cwd;
|
||||
use lib Cwd::abs_path(File::Basename::dirname(__FILE__) . '/../');
|
||||
|
||||
#sub new { bless [], shift; }
|
||||
#
|
||||
#sub step {
|
||||
# my ( $self, $value ) = @_;
|
||||
#
|
||||
# push @$self, $value;
|
||||
#}
|
||||
#
|
||||
#sub finalize {
|
||||
# my $self = $_[0];
|
||||
#
|
||||
# my $n = @$self;
|
||||
#
|
||||
# # Variance is NULL unless there is more than one row
|
||||
# return undef unless $n || $n == 1;
|
||||
#
|
||||
# my $mu = 0;
|
||||
# foreach my $v ( @$self ) {
|
||||
# $mu += $v;
|
||||
# }
|
||||
# $mu /= $n;
|
||||
#
|
||||
# my $sigma = 0;
|
||||
# foreach my $v ( @$self ) {
|
||||
# #$sigma += ($x - $mu)**2;
|
||||
# $sigma += ($v - $mu)**2;
|
||||
# }
|
||||
# $sigma = $sigma / ($n - 1);
|
||||
#
|
||||
# return $sigma;
|
||||
#}
|
||||
|
||||
my $mu = 0;
|
||||
my $count = 0;
|
||||
my $S = 0;
|
||||
|
||||
sub new { bless [], shift; }
|
||||
|
||||
sub step {
|
||||
my ( $self, $value ) = @_;
|
||||
$count++;
|
||||
my $delta = $value - $mu;
|
||||
$mu += $delta / $count;
|
||||
$S += $delta * ($value - $mu);
|
||||
}
|
||||
|
||||
sub finalize {
|
||||
my $self = $_[0];
|
||||
return $S / ($count - 1);
|
||||
}
|
||||
|
||||
1;
|
||||
# helper module to implement variance aggregate function with SQLite
|
||||
# adamk@cpan.org
|
||||
# 2009
|
||||
|
||||
# used only for testing custom functions ...
|
||||
|
||||
package NGCP::BulkProcessor::SqlConnectors::SQLiteVarianceAggregate;
|
||||
use strict;
|
||||
|
||||
## no critic
|
||||
|
||||
#use File::Basename;
|
||||
#use Cwd;
|
||||
#use lib Cwd::abs_path(File::Basename::dirname(__FILE__) . '/../');
|
||||
|
||||
#sub new { bless [], shift; }
|
||||
#
|
||||
#sub step {
|
||||
# my ( $self, $value ) = @_;
|
||||
#
|
||||
# push @$self, $value;
|
||||
#}
|
||||
#
|
||||
#sub finalize {
|
||||
# my $self = $_[0];
|
||||
#
|
||||
# my $n = @$self;
|
||||
#
|
||||
# # Variance is NULL unless there is more than one row
|
||||
# return undef unless $n || $n == 1;
|
||||
#
|
||||
# my $mu = 0;
|
||||
# foreach my $v ( @$self ) {
|
||||
# $mu += $v;
|
||||
# }
|
||||
# $mu /= $n;
|
||||
#
|
||||
# my $sigma = 0;
|
||||
# foreach my $v ( @$self ) {
|
||||
# #$sigma += ($x - $mu)**2;
|
||||
# $sigma += ($v - $mu)**2;
|
||||
# }
|
||||
# $sigma = $sigma / ($n - 1);
|
||||
#
|
||||
# return $sigma;
|
||||
#}
|
||||
|
||||
my $mu = 0;
|
||||
my $count = 0;
|
||||
my $S = 0;
|
||||
|
||||
sub new { bless [], shift; }
|
||||
|
||||
sub step {
|
||||
my ( $self, $value ) = @_;
|
||||
$count++;
|
||||
my $delta = $value - $mu;
|
||||
$mu += $delta / $count;
|
||||
$S += $delta * ($value - $mu);
|
||||
}
|
||||
|
||||
sub finalize {
|
||||
my $self = $_[0];
|
||||
return $S / ($count - 1);
|
||||
}
|
||||
|
||||
1;
|
||||
Loading…
Reference in new issue