From 01cbb4a9498b425e39bfd964c7066039d73393c8 Mon Sep 17 00:00:00 2001 From: Rene Krenn Date: Mon, 13 May 2024 17:47:39 +0200 Subject: [PATCH] MT#59727 extend cdr data model for cdr exporter support loading the full CDR graph, featuring all relations: -cdr_groups -cdr_tags -cdr_mos -cdr_status -cdr_relations -cdr_presentity -cdr_cash_balances -cdr_time_balances aside various fixes, NGCP::BulkProcessor::Closure is introduced, which provides uniform code execution: -perl code (from YAML) -perl code (from string) -javascript (from string) an identical symbol table is exposed to either language env, including the SQL connector api. Change-Id: If10422df33d996fb6a6a6b53d0ead28ea1cef755 (cherry picked from commit 2f56063aff20c24c7e59f9fa89cebe23bfa40ed1) --- .gitignore | 1 + debian/control | 7 +- lib/NGCP/BulkProcessor/Closure.pm | 551 ++++++++++++++++++ .../BulkProcessor/Dao/Trunk/accounting/cdr.pm | 125 ++-- .../Trunk/accounting/cdr_cash_balance_data.pm | 145 +++++ .../Dao/Trunk/accounting/cdr_direction.pm | 33 ++ .../Dao/Trunk/accounting/cdr_export_status.pm | 70 ++- .../accounting/cdr_export_status_data.pm | 59 ++ .../Dao/Trunk/accounting/cdr_group.pm | 1 + .../Dao/Trunk/accounting/cdr_mos_data.pm | 137 +++++ .../Dao/Trunk/accounting/cdr_presentity.pm | 136 +++++ .../Dao/Trunk/accounting/cdr_provider.pm | 34 ++ .../Dao/Trunk/accounting/cdr_relation.pm | 143 +++++ .../Dao/Trunk/accounting/cdr_relation_data.pm | 184 ++++++ .../Dao/Trunk/accounting/cdr_tag.pm | 33 ++ .../Dao/Trunk/accounting/cdr_tag_data.pm | 25 +- .../Trunk/accounting/cdr_time_balance_data.pm | 145 +++++ .../Dao/Trunk/billing/billing_fees.pm | 8 +- .../Dao/Trunk/billing/billing_fees_history.pm | 131 +++++ .../Dao/Trunk/billing/billing_zones.pm | 5 - .../Trunk/billing/billing_zones_history.pm | 110 ++++ .../Dao/Trunk/billing/contract_balances.pm | 20 + .../Dao/Trunk/billing/contracts.pm | 3 + .../Dao/Trunk/billing/products.pm | 50 ++ .../Dao/Trunk/billing/resellers.pm | 18 + .../Dao/Trunk/billing/voip_subscribers.pm | 18 + .../Trunk/provisioning/voip_peer_groups.pm | 16 + lib/NGCP/BulkProcessor/FileProcessor.pm | 47 +- .../FileProcessors/CSVFileSimple.pm | 4 + lib/NGCP/BulkProcessor/Globals.pm | 57 +- lib/NGCP/BulkProcessor/LoadConfig.pm | 30 +- lib/NGCP/BulkProcessor/LogError.pm | 34 +- lib/NGCP/BulkProcessor/Logging.pm | 136 +++-- lib/NGCP/BulkProcessor/NoSqlConnector.pm | 4 +- .../NoSqlConnectors/RedisEntry.pm | 35 +- .../NoSqlConnectors/RedisProcessor.pm | 47 +- .../Projects/ETL/CDR/Dao/Tabular.pm | 279 --------- .../Projects/ETL/CDR/ExportCDR.pm | 490 ---------------- .../Projects/ETL/CDR/ProjectConnectorPool.pm | 120 ---- .../Projects/ETL/CDR/Settings.pm | 484 --------------- .../BulkProcessor/Projects/ETL/CDR/config.cfg | 61 -- .../Projects/ETL/CDR/config.debug.cfg | 61 -- .../BulkProcessor/Projects/ETL/CDR/graph.yml | 5 - .../BulkProcessor/Projects/ETL/CDR/load.yml | 44 -- .../BulkProcessor/Projects/ETL/CDR/process.pl | 319 ---------- .../Projects/ETL/CDR/settings.cfg | 58 -- .../Projects/ETL/CDR/tabular.yml | 70 --- .../Projects/ETL/EDR/Dao/PeriodEvents.pm | 157 ----- .../Projects/ETL/EDR/ExportEvents.pm | 271 --------- .../Projects/ETL/EDR/ProjectConnectorPool.pm | 120 ---- .../Projects/ETL/EDR/Settings.pm | 235 -------- .../BulkProcessor/Projects/ETL/EDR/config.cfg | 61 -- .../Projects/ETL/EDR/config.debug.cfg | 61 -- .../BulkProcessor/Projects/ETL/EDR/process.pl | 214 ------- .../Projects/ETL/EDR/settings.cfg | 21 - .../Projects/Massive/Generator/process.pl | 4 +- .../Massive/RegistrationMonitoring/process.pl | 2 +- lib/NGCP/BulkProcessor/RestProcessor.pm | 47 +- lib/NGCP/BulkProcessor/SqlConnector.pm | 7 +- lib/NGCP/BulkProcessor/SqlConnectors/CSVDB.pm | 4 +- .../BulkProcessor/SqlConnectors/MySQLDB.pm | 4 +- .../BulkProcessor/SqlConnectors/OracleDB.pm | 7 +- .../SqlConnectors/PostgreSQLDB.pm | 4 +- .../SqlConnectors/SQLServerDB.pm | 7 +- lib/NGCP/BulkProcessor/SqlProcessor.pm | 123 ++-- lib/NGCP/BulkProcessor/SqlRecord.pm | 35 +- lib/NGCP/BulkProcessor/Utils.pm | 26 +- 67 files changed, 2634 insertions(+), 3369 deletions(-) create mode 100644 lib/NGCP/BulkProcessor/Closure.pm create mode 100644 lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_cash_balance_data.pm create mode 100644 lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_mos_data.pm create mode 100644 lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_presentity.pm create mode 100644 lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_relation.pm create mode 100644 lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_relation_data.pm create mode 100644 lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_time_balance_data.pm create mode 100644 lib/NGCP/BulkProcessor/Dao/Trunk/billing/billing_fees_history.pm create mode 100644 lib/NGCP/BulkProcessor/Dao/Trunk/billing/billing_zones_history.pm delete mode 100644 lib/NGCP/BulkProcessor/Projects/ETL/CDR/Dao/Tabular.pm delete mode 100644 lib/NGCP/BulkProcessor/Projects/ETL/CDR/ExportCDR.pm delete mode 100644 lib/NGCP/BulkProcessor/Projects/ETL/CDR/ProjectConnectorPool.pm delete mode 100644 lib/NGCP/BulkProcessor/Projects/ETL/CDR/Settings.pm delete mode 100644 lib/NGCP/BulkProcessor/Projects/ETL/CDR/config.cfg delete mode 100644 lib/NGCP/BulkProcessor/Projects/ETL/CDR/config.debug.cfg delete mode 100644 lib/NGCP/BulkProcessor/Projects/ETL/CDR/graph.yml delete mode 100644 lib/NGCP/BulkProcessor/Projects/ETL/CDR/load.yml delete mode 100644 lib/NGCP/BulkProcessor/Projects/ETL/CDR/process.pl delete mode 100644 lib/NGCP/BulkProcessor/Projects/ETL/CDR/settings.cfg delete mode 100644 lib/NGCP/BulkProcessor/Projects/ETL/CDR/tabular.yml delete mode 100644 lib/NGCP/BulkProcessor/Projects/ETL/EDR/Dao/PeriodEvents.pm delete mode 100644 lib/NGCP/BulkProcessor/Projects/ETL/EDR/ExportEvents.pm delete mode 100644 lib/NGCP/BulkProcessor/Projects/ETL/EDR/ProjectConnectorPool.pm delete mode 100644 lib/NGCP/BulkProcessor/Projects/ETL/EDR/Settings.pm delete mode 100644 lib/NGCP/BulkProcessor/Projects/ETL/EDR/config.cfg delete mode 100644 lib/NGCP/BulkProcessor/Projects/ETL/EDR/config.debug.cfg delete mode 100644 lib/NGCP/BulkProcessor/Projects/ETL/EDR/process.pl delete mode 100644 lib/NGCP/BulkProcessor/Projects/ETL/EDR/settings.cfg diff --git a/.gitignore b/.gitignore index 305c8a9..7591a6b 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,7 @@ /MYMETA.* /_build/ /blib/ +.vscode #/uml #/html diff --git a/debian/control b/debian/control index 6438403..c0fa34d 100644 --- a/debian/control +++ b/debian/control @@ -19,6 +19,7 @@ Homepage: https://www.sipwise.com/ Package: libngcp-bulkprocessor-perl Architecture: all Depends: + libarchive-zip-perl, libconfig-any-perl, libdata-dump-perl, libdata-rmap-perl, @@ -35,12 +36,14 @@ Depends: libdbd-sqlite3-perl, libemail-mime-attachment-stripper-perl, libemail-mime-perl, + libeval-closure-perl, libexcel-reader-xlsx-perl, libgearman-client-perl, libhtml-parser-perl, libhttp-message-perl, libintl-perl, libio-socket-ssl-perl, + libje-perl, libjson-perl, liblog-log4perl-perl, libmail-imapclient-perl, @@ -48,6 +51,7 @@ Depends: libmime-lite-perl, libmime-tools-perl, libnet-address-ip-local-perl, + libobject-destroyer-perl, libphp-serialization-perl, libredis-perl, libspreadsheet-parseexcel-perl, @@ -62,11 +66,12 @@ Depends: libwww-perl, libxml-dumper-perl, libyaml-libyaml-perl, + libyaml-perl, ${misc:Depends}, ${perl:Depends}, Suggests: libdbd-odbc-perl, - libdbd-orable-perl, + libdbd-oracle-perl, libdbd-pg-perl, tdsodbc, Description: NGCP bulk processor framework perl modules diff --git a/lib/NGCP/BulkProcessor/Closure.pm b/lib/NGCP/BulkProcessor/Closure.pm new file mode 100644 index 0000000..99ee930 --- /dev/null +++ b/lib/NGCP/BulkProcessor/Closure.pm @@ -0,0 +1,551 @@ +package NGCP::BulkProcessor::Closure; +use strict; + +use warnings; + +no warnings 'uninitialized'; ## no critic (ProhibitNoWarnings) + +use Scalar::Util qw(blessed reftype); +use Eval::Closure qw(eval_closure); +my $eval_closure_make_lexical_assignment = sub { + my ($key, $index, $alias) = @_; + my $sigil = substr($key, 0, 1); + my $name = substr($key, 1); + if (Eval::Closure::HAS_LEXICAL_SUBS && $sigil eq '&') { + my $tmpname = '$__' . $name . '__' . $index; + return 'use feature "lexical_subs"; ' + . 'no warnings "experimental::lexical_subs"; ' + . 'my ' . $tmpname . ' = $_[' . $index . ']; ' + . 'my sub ' . $name . ' { goto ' . $tmpname . ' }'; + } + if ($alias) { + return 'my ' . $key . ';'; + } + else { + return 'my ' . $key . ' = ' . '$_[' . $index . '];'; + #return 'my ' . $key . ' = ' . $sigil . '{$_[' . $index . ']};'; + } +}; +my $eval_closure_validate_env = sub { + my ($env) = @_; + + croak("The 'environment' parameter must be a hashref") + unless reftype($env) eq 'HASH'; + + for my $var (keys %$env) { + if (Eval::Closure::HAS_LEXICAL_SUBS) { + croak("Environment key '$var' should start with \@, \%, \$, or \&") + if index('$@%&', substr($var, 0, 1)) < 0; + } + else { + croak("Environment key '$var' should start with \@, \%, or \$") + if index('$@%', substr($var, 0, 1)) < 0; + } + #croak("Environment values must be references, not $env->{$var}") + # unless ref($env->{$var}); + } +}; + + +#use JE::Destroyer qw(); +use JE qw(); + +{ + no warnings 'redefine'; ## no critic (ProhibitNoWarnings) + *JE::Object::evall = sub { + no warnings; ## no critic (ProhibitNoWarnings) + my $global = shift; + my $v = shift; + my $r = eval 'local *_;' . $v; ## no critic (ProhibitStringyEval) + if ($@) { + my $e = $@; + $r = eval "local *_;'$v'"; ## no critic (ProhibitStringyEval) + if ($@) { + die; + } + } + $r; + }; +} + +use JSON qw(); + +use YAML::Types; +{ + no warnings 'redefine'; ## no critic (ProhibitNoWarnings) + *YAML::Type::code::yaml_load = sub { + my $self = shift; + my ($node, $class, $loader) = @_; + if ($loader->load_code) { + $node = "sub $node" unless $node =~ /^\s*sub/; #upstream backward compat + my $code = eval "package yamlmain; no strict 'vars'; $node"; ## no critic (ProhibitStringyEval) + if ($@) { + die ($@); + #$loader->warn('YAML_LOAD_WARN_PARSE_CODE', $@); + #return sub {}; + } + else { + CORE::bless $code, $class if ($class and $YAML::LoadBlessed); + return $code; + } + } + else { + return CORE::bless sub {}, $class if ($class and $YAML::LoadBlessed); + return sub {}; + } + }; +} + +use NGCP::BulkProcessor::SqlConnector qw(); + +use NGCP::BulkProcessor::Array qw(array_to_map); + +require Exporter; +our @ISA = qw(Exporter); +our @EXPORT_OK = qw( + closure + cleanup + is_code + clear_stash +); + +(my $DISABLED_CORE_FUNCTION_MAP, undef, undef) = array_to_map([ qw( + binmode close closedir dbmclose dbmopen eof fileno flock format getc read + readdir rewinddir say seek seekdir select syscall sysread sysseek + syswrite tell telldir truncate write print printf + + chdir chmod chown chroot fcntl glob ioctl link lstat mkdir open opendir readlink + rename rmdir stat symlink sysopen umask unlink utime + + alarm exec fork getpgrp getppid getpriority kill pipe setpgrp setpriority sleep + system times wait waitpid + + accept bind connect getpeername getsockname getsockopt listen recv send setsockopt + shutdown socket socketpair + + msgctl msgget msgrcv msgsnd semctl semget semop shmctl shmget shmread shmwrite + + endgrent endhostent endnetent endpwent getgrent getgrgid getgrnam getlogin getpwent + getpwnam getpwuid setgrent setpwent + + endprotoent endservent gethostbyaddr gethostbyname gethostent getnetbyaddr + getnetbyname getnetent getprotobyname getprotobynumber getprotoent getservbyname + getservbyport getservent sethostent setnetent setprotoent setservent + + exit goto +)], sub { return shift; }, sub { return 1; }, 'last'); + +my @DISABLED_CORE_FUNCTIONS = grep { $DISABLED_CORE_FUNCTION_MAP->{$_}; } keys %$DISABLED_CORE_FUNCTION_MAP; + +my $PERL_ENV = 'use subs qw(' . join(' ', @DISABLED_CORE_FUNCTIONS) . ");\n"; +foreach my $f (@DISABLED_CORE_FUNCTIONS) { + $PERL_ENV .= 'sub ' . $f . " { die('$f called'); }\n"; +} + +my $JS_ENV = ''; + +my $JE_ANON_CLASS = 'je_anon'; +sub je_anon::TO_JSON { + return _unbless(@_); +}; + +my %interpreter_cache = (); +my %stash = (); +my %je_exported_map = (); + +sub _stash_get { + my $k = shift; + return $stash{$k} if $k; +} +sub _stash_set { + my ($k,$v) = @_; + $stash{$k} = $v if $k; +} + +sub cleanup { + + eval { + #no warnings 'deprecated'; + require JE::Destroyer; + JE::Destroyer->import(); + 1; + } or do { + return; + }; + clear_stash(); + foreach my $code (keys %interpreter_cache) { + JE::Destroyer::destroy($interpreter_cache{$code}) if 'JE' eq ref $interpreter_cache{$code}; # break circular refs + delete $interpreter_cache{$code}; + delete $je_exported_map{$code}; + } + +} + +sub clear_stash { + + %stash = (); + +} + +sub new { + + my $class = shift; + my $self = bless {}, $class; + + my ($code,$context,$description) = @_; + + $self->{description} = $description; + if ('CODE' eq ref $code) { + $self->{description} //= 'coderef'; + $self->{type} = "coderef"; + $self->{exported_map} = (); + foreach my $key (_get_public_vars($context = { + get_env => sub { + return _filter_perl_env_symbols(keys %yamlmain::); + }, + to_json => \&_unbless_to_json, + stash_get => \&_stash_get, + stash_set => \&_stash_set, + %{$context // {}}, + })) { + _register_closure_var($key,$context->{$key}); + $self->{exported_map}->{$key} = 1; + } + $self->{code} = $code; + } elsif ($code =~ /^\s*sub/) { #perl + $self->{source} = $code; + $self->{description} //= 'perl function'; + $self->{type} = "perl"; + unless (exists $interpreter_cache{$code}) { + local *Eval::Closure::_make_lexical_assignment = $eval_closure_make_lexical_assignment; + local *Eval::Closure::_validate_env = $eval_closure_validate_env; + my @exported = (); + eval { + $interpreter_cache{$code} = eval_closure( + source => ($PERL_ENV . $code), + environment => { + map { if ('ARRAY' eq ref $context->{$_}) { + push(@exported,$_); + ('$' . $_) => $context->{$_}; + } elsif ('HASH' eq ref $context->{$_}) { + push(@exported,$_); + ('$' . $_) => $context->{$_}; + } elsif ($JE_ANON_CLASS eq ref $context->{$_}) { + push(@exported,$_); + ('$' . $_) => _unbless($context->{$_}); + } elsif ('CODE' eq ref $context->{$_}) { + push(@exported,$_); + ('&' . $_) => $context->{$_}; + } elsif (ref $context->{$_}) { + push(@exported,$_); + ('$' . $_) => $context->{$_}; + } else { + push(@exported,$_); + ('$' . $_) => $context->{$_}; + } } _get_public_vars($context = { + get_env => sub { + no strict "refs"; ## no critic (ProhibitNoStrict) + return (@exported,_filter_perl_env_symbols(keys %{caller() .'::'})); + }, + to_json => \&_unbless_to_json, + stash_get => \&_stash_get, + stash_set => \&_stash_set, + %{$context // {}}, + }) + }, + terse_error => 1, + description => $self->{description}, + alias => 0, + ); + }; + if ($@) { + die("$self->{description}: " . $@); + } + } + } elsif ($code =~ /^\s*function/) { #javascript + $self->{source} = $code; + $self->{description} //= 'javascript function'; + $self->{type} = "js"; + my $je; + if (exists $interpreter_cache{$code}) { + $je = $interpreter_cache{$code}; + } else { + $je_exported_map{$code} = {}; + $je = JE->new(); + $je->eval($JS_ENV . "\nvar _func = " . $code . ';'); + $interpreter_cache{$code} = $je; + } + $je->eval(_serialize_je_args($je,{ + get_env => sub { + return [ _filter_js_env_symbols(keys %$je) ]; + }, + to_json => sub { + my ($obj,$pretty, $canonical) = @_; + return _to_json(_unbox_je_value($obj), _unbox_je_value($pretty), _unbox_je_value($canonical)); + }, + quotemeta => sub { + my $s = shift; + return quotemeta(_unbox_je_value($s)); + }, + sprintf => sub { + my ($f,@p) = @_; + return sprintf(_unbox_je_value($f), map { + _unbox_je_value($_); + } @p); + }, + stash_get => sub { + my $k = shift; + return _stash_get(_unbox_je_value($k)); + }, + stash_set => sub { + my ($k,$v) = @_; + _stash_set(_unbox_je_value($k),_unbox_je_value($v)); + }, + %{$context // {}}, + },$je_exported_map{$code})); + die("$self->{description}: " . $@) if $@; + } else { + die("unsupported expression langage"); + } + + return $self; + +} + +sub _register_closure_var { + + my ($key,$value) = @_; + # modified globally? + no strict "refs"; ## no critic (ProhibitNoStrict) + if ('CODE' eq ref $value) { + no warnings 'redefine'; ## no critic (ProhibitNoWarnings) + *{"yamlmain::$key"} = $value; + } else { + ${"yamlmain::$key"} = $value; + } + +} + +sub _get_public_vars { + + my $args = shift; + return grep { substr($_,0,1) ne '_'; } keys %$args; + +} + +sub _serialize_je_args { + + my ($je,$args,$je_env) = @_; + my $sep; + my @args; + if ('HASH' eq ref $args and $je_env) { + $sep = ";\n"; + @args = map { { k => $_, v => $args->{$_}, }; } _get_public_vars($args); + } else { + $sep = ","; + @args = map { { k => undef, v => $_, }; } @$args; + } + return join ($sep,map { + if ('CODE' eq ref $_->{v}) { + if ($_->{k} and not $je_env->{$_->{k}}) { + $je_env->{$_->{k}} = 1; + my $sub = $_->{v}; + $je->new_function($_->{k} => sub { + return $sub->(map { _unbox_je_value($_); } @_); + }); + } + (); + } elsif (blessed $_->{v} and $_->{v}->isa('NGCP::BulkProcessor::SqlConnector')) { + if ($_->{k} and not $je_env->{$_->{k}}) { + $je_env->{$_->{k}} = 1; + my $db = $_->{v}; + no strict 'refs'; ## no critic (ProhibitNoStrict) + foreach my $k (keys %NGCP::BulkProcessor::SqlConnector::) { + next unless substr($k,0,3) eq "db_"; + if (exists &{"NGCP::BulkProcessor::SqlConnector::$k"}) { # check if symbol is method + $je->new_function($k => sub { + return $db->$k(map { _unbox_je_value($_); } @_); + }); + } + } + } + (); + } elsif (('ARRAY' eq ref $_->{v}) + or ('HASH' eq ref $_->{v}) + or ($JE_ANON_CLASS eq ref $_->{v})) { + if (not $_->{k}) { + _to_json($_->{v}); + } elsif ($je_env->{$_->{k}}) { + $_->{k} . ' = ' . _to_json($_->{v}); + } else { + $je_env->{$_->{k}} = 1; + 'var ' . $_->{k} . ' = ' . _to_json($_->{v}); + } + } elsif (('ARRAY' eq reftype($_->{v})) + or ('HASH' eq reftype($_->{v}))) { + if (not $_->{k}) { + _unbless_to_json($_->{v}); + } elsif ($je_env->{$_->{k}}) { + $_->{k} . ' = ' . _unbless_to_json($_->{v}); + } else { + $je_env->{$_->{k}} = 1; + 'var ' . $_->{k} . ' = ' . _unbless_to_json($_->{v}); + } + } elsif (ref $_->{v}) { + warn((ref $_->{v}) . ' objects not available in javascript'); + } else { + if (not $_->{k}) { + "'" . _escape_js($_->{v}) . "'"; + } elsif ($je_env->{$_->{k}}) { + $_->{k} . " = '" . _escape_js($_->{v}) . "'"; + } else { + $je_env->{$_->{k}} = 1; + 'var ' . $_->{k} . " = '" . _escape_js($_->{v}) . "'"; + } + } + } @args); + +} + +sub calc { + + my $self = shift; + my $context = shift; + my @v; + if ("coderef" eq $self->{type}) { + foreach my $key (_get_public_vars($context)) { + unless ($self->{exported_map}->{$key}) { + _register_closure_var($key,$context->{$key}); + $self->{exported_map}->{$key} = 1; + } + } + eval { + @v = $self->{code}->(@_); + $v[0] = _unbless($v[0]) if ($JE_ANON_CLASS eq ref $v[0]); + }; + if ($@) { + die("$self->{description}: " . $@); + } + } elsif ("perl" eq $self->{type}) { + @v = $interpreter_cache{$self->{source}}->(@_); + $v[0] = _unbless($v[0]) if ($JE_ANON_CLASS eq ref $v[0]); + if ($@) { + die("$self->{description}: " . $@); + } + } elsif ("js" eq $self->{type}) { + my $je = $interpreter_cache{$self->{source}}; + my $updated_je_env = ''; + $updated_je_env = _serialize_je_args($je,$context,$je_exported_map{$self->{source}}) if $context; + $updated_je_env .= ";\n" if length($updated_je_env); + my $call; + if (scalar @_) { + $call = "_func(" . _serialize_je_args($je,[ @_ ],$je_exported_map{$self->{source}}) . ");"; + } else { + $call = "_func();" + } + $v[0] = _unbox_je_value($interpreter_cache{$self->{source}}->eval($updated_je_env . $call)); + if ($@) { + die("$self->{description}: " . $@); + } + } + + return @v if wantarray; + return $v[0]; + +} + +sub is_code { + + my $code = shift; + return unless defined $code; + if ('CODE' eq ref $code) { + return 1; + } elsif (not ref $code) { + if ($code =~ /^\s*function/) { + return 1; + } elsif ($code =~ /^\s*sub/) { + return 1; + } + } + return 0; + +} + +sub _unbox_je_value { + + my $v = shift; + return undef unless defined $v; ## no critic (ProhibitExplicitReturnUndef) + if ((ref $v) =~ /^JE::/) { + $v = $v->value; + } elsif ($JE_ANON_CLASS eq ref $v) { + $v = _unbless($v); + } + if ('ARRAY' eq ref $v) { + return [ map { _unbox_je_value($_); } @$v ]; + } elsif ('HASH' eq ref $v) { + return { map { $_ => _unbox_je_value($v->{$_}); } keys %$v }; + } else { + return $v; + } + +} + +sub _unbless { + + my $obj = shift; + if ('HASH' eq reftype($obj)) { + return { map { $_ => _unbless($obj->{$_}); } keys %$obj }; + } elsif ('ARRAY' eq reftype($obj)) { + return [ map { _unbless($_); } @$obj ]; + } else { + return $obj; + } + +}; + +sub _escape_js { + + my $str = shift // ''; + my $quote_char = shift; + $quote_char //= "'"; + $str =~ s/\\/\\\\/g; + $str =~ s/$quote_char/\\$quote_char/g; + return $str; + +} + +sub _to_json { + + my ($obj,$pretty,$canonical) = @_; + return JSON::to_json($obj, { + allow_nonref => 1, allow_blessed => 1, allow_unknown => 1, + convert_blessed => 1, pretty => $pretty, canonical => $canonical, }); + +} + +sub _filter_perl_env_symbols { + + return grep { + $_ !~ /^__ANON__/ + and $_ !~ /^BEGIN/ + and not (exists $DISABLED_CORE_FUNCTION_MAP->{$_} and $DISABLED_CORE_FUNCTION_MAP->{$_}) + ; } @_; + +} + +sub _filter_js_env_symbols { + + return grep { + $_ !~ /^_func/ + ; } @_; + +} + +sub _unbless_to_json { + + my $obj = shift; + return _to_json(_unbless($obj),@_); + +} + +1; \ No newline at end of file diff --git a/lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr.pm b/lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr.pm index f121b13..dd84f04 100644 --- a/lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr.pm +++ b/lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr.pm @@ -25,6 +25,11 @@ use NGCP::BulkProcessor::SqlProcessor qw( use NGCP::BulkProcessor::SqlRecord qw(); use NGCP::BulkProcessor::Dao::Trunk::accounting::cdr_export_status_data qw(); +use NGCP::BulkProcessor::Dao::Trunk::accounting::cdr_mos_data qw(); +use NGCP::BulkProcessor::Dao::Trunk::accounting::cdr_tag_data qw(); +use NGCP::BulkProcessor::Dao::Trunk::accounting::cdr_relation_data qw(); +use NGCP::BulkProcessor::Dao::Trunk::accounting::cdr_cash_balance_data qw(); +use NGCP::BulkProcessor::Dao::Trunk::accounting::cdr_time_balance_data qw(); require Exporter; our @ISA = qw(Exporter NGCP::BulkProcessor::SqlRecord); @@ -272,7 +277,7 @@ sub get_callidprefix { sub findby_callidprefix { - my ($xa_db,$call_id,$joins,$conditions,$load_recursive) = @_; + my ($xa_db,$call_id,$alias,$joins,$conditions,$load_recursive) = @_; check_table(); my $db = &$get_db(); @@ -285,7 +290,7 @@ sub findby_callidprefix { my @conditions = @{$conditions // []}; push(@conditions,{ $table . '.call_id' => { 'LIKE' => '?' } }); my $stmt = 'SELECT ' . join(',', map { $table . '.' . $db->columnidentifier($_); } @$expected_fieldnames) . ' ' . - _get_export_stmt($db,$joins,\@conditions) . + _get_export_stmt($db,$alias,$joins,\@conditions) . ' ORDER BY LENGTH(' . $table . '.call_id' . ') ASC, ' . $table . '.start_time ASC'; my @params = ($call_id . '%'); my $rows = $xa_db->db_get_all_arrayref($stmt,@params); @@ -296,7 +301,7 @@ sub findby_callidprefix { sub findby_callid { - my ($xa_db,$call_id,$joins,$conditions,$load_recursive) = @_; + my ($xa_db,$call_id,$alias,$joins,$conditions,$load_recursive) = @_; check_table(); my $db = &$get_db(); @@ -306,7 +311,7 @@ sub findby_callid { my @conditions = @{$conditions // []}; push(@conditions,{ $table . '.call_id' => { '=' => '?' } }); my $stmt = 'SELECT ' . join(',', map { $table . '.' . $db->columnidentifier($_); } @$expected_fieldnames) . ' ' . - _get_export_stmt($db,$joins,\@conditions) . + _get_export_stmt($db,$alias,$joins,\@conditions) . ' ORDER BY ' . $table . '.start_time ASC'; my @params = ($call_id); my $rows = $xa_db->db_get_all_arrayref($stmt,@params); @@ -342,23 +347,27 @@ sub process_unexported { my %params = @_; my ($process_code, + $name, $static_context, $init_process_context_code, $uninit_process_context_code, $multithreading, $numofthreads, $blocksize, + $alias, $joins, $conditions, #$sort, $limit) = @params{qw/ process_code + name static_context init_process_context_code uninit_process_context_code multithreading numofthreads blocksize + alias joins conditions limit @@ -369,32 +378,41 @@ sub process_unexported { NGCP::BulkProcessor::Dao::Trunk::accounting::cdr_export_status_data::check_table(); my $db = &$get_db(); my $table = $db->tableidentifier($tablename); - + my $select_stmt; my $count_stmt; - my $select_format = 'SELECT ' . $table . '.' . $db->columnidentifier('id') . ', ' . $table . '.' . $db->columnidentifier('call_id') . ' %s ORDER BY ' . $table . '.' . $db->columnidentifier('id'); - my $count_format = 'SELECT COUNT(1) FROM (%s) AS __cnt'; + my $select_format = 'SELECT ' . ($alias ? $alias : $table) . '.id, ' . ($alias ? $alias : $table) . '.call_id, ' . ($alias ? $alias : $table) . '.start_time %s'; # ORDER BY ' . $table . '.' . $db->columnidentifier('id'); + my $count_format = 'SELECT COUNT(1) %s'; + if (defined $limit) { + $select_format .= ' LIMIT ' . $limit; + $count_format = sprintf($count_format,'FROM (SELECT 1 %s LIMIT ' . $limit . ') as q'); + } if ($static_context) { $static_context->{part} = 'A'; - $select_stmt = sprintf('(' . $select_format . ')',_get_export_stmt_part($db,$static_context,$joins,$conditions)); - $count_stmt = sprintf('(' . $count_format . ')',$db->paginate_sort_query('SELECT 1 ' . _get_export_stmt_part($db,$static_context,$joins,$conditions),0,$limit,undef)); - $select_stmt .= ' UNION ALL '; - $count_stmt .= ' + '; + $select_stmt = '(' . sprintf($select_format,_get_export_stmt_part($db,$static_context,$alias,$joins,$conditions,1)); + $count_stmt = '(' . sprintf($count_format,_get_export_stmt_part($db,$static_context,$alias,$joins,$conditions,0)); + $select_stmt .= ') UNION ALL ('; + $count_stmt .= ') + ('; $static_context->{part} = 'B'; - $select_stmt .= sprintf('(' . $select_format . ')',_get_export_stmt_part($db,$static_context,$joins,$conditions)); - $count_stmt .= sprintf('(' . $count_format . ')',$db->paginate_sort_query('SELECT 1 ' . _get_export_stmt_part($db,$static_context,$joins,$conditions),0,$limit,undef)); + my $b = _get_export_stmt_part($db,$static_context,$alias,$joins,$conditions,1); + $select_stmt .= sprintf($select_format,_get_export_stmt_part($db,$static_context,$alias,$joins,$conditions,1)) . ')'; + $select_stmt = 'SELECT id, call_id, start_time FROM (' . $select_stmt . ') as q ORDER BY id'; + $count_stmt .= sprintf($count_format,_get_export_stmt_part($db,$static_context,$alias,$joins,$conditions,0)) . ')'; if (defined $limit) { $count_stmt = 'SELECT LEAST(' . $count_stmt . ', ' . $limit . ')'; + $select_stmt .= ' LIMIT ' . $limit; } else { $count_stmt = 'SELECT ' . $count_stmt; - } + } + delete $static_context->{part}; } else { - $select_stmt = sprintf($select_format,_get_export_stmt_part($db,undef,$joins,$conditions)); - $count_stmt = sprintf($count_format,$db->paginate_sort_query('SELECT 1 ' . _get_export_stmt_part($db,undef,$joins,$conditions),0,$limit,undef)); + $select_stmt = sprintf($select_format,_get_export_stmt_part($db,undef,$alias,$joins,$conditions,1)); + $count_stmt = sprintf($count_format,_get_export_stmt_part($db,undef,$alias,$joins,$conditions,0)); } return process_table( + name => $name, get_db => $get_db, class => __PACKAGE__, process_code => sub { @@ -484,41 +502,49 @@ sub process_fromto { sub _get_export_stmt { - my ($db,$joins,$conditions) = @_; - return _get_export_stmt_part($db,undef,$joins,$conditions); + my ($db,$alias,$joins,$conditions) = @_; + return _get_export_stmt_part($db,undef,$alias,$joins,$conditions); } sub _get_export_stmt_part { - my ($db,$static_context,$joins,$conditions) = @_; + my ($db,$static_context,$alias,$joins,$conditions,$order_by_id) = @_; my $table = $db->tableidentifier($tablename); - my $stmt = "FROM " . $table; + my $table_aliased = $table; + $table_aliased .= ' AS ' . $alias if $alias; + my $stmt = "FROM " . $table_aliased; my @intjoins = (); if (defined $joins and (scalar @$joins) > 0) { foreach my $f (@$joins) { - my ($table, $keys) = %{ $f }; - my ($foreign_key, $own_key) = %{ $keys }; - push @intjoins, "LEFT JOIN $table ON $foreign_key = $own_key"; + my ($tbl, $keys) = %{ $f }; + my $j = "LEFT JOIN $tbl"; + if ($keys) { + my ($foreign_key, $own_key) = %{ $keys }; + $j .= " ON $foreign_key = $own_key"; + } + push @intjoins, $j; } } + my $order_by = ' ORDER BY ' . ($alias ? $alias : $table) . '.id'; my @conds = (); if (defined $static_context and $static_context->{export_status_id} and $static_context->{part}) { unless (defined $static_context->{last_processed_cdr_id}) { $static_context->{last_processed_cdr_id} = NGCP::BulkProcessor::Dao::Trunk::accounting::cdr_export_status_data::find_last_processed_cdrid($static_context->{export_status_id}); } if ('b' eq lc($static_context->{part})) { - push @conds, $table . '.id > ' . $static_context->{last_processed_cdr_id}; + push @conds, ($alias ? $alias : $table) . '.id > ' . $static_context->{last_processed_cdr_id}; } elsif ('a' eq lc($static_context->{part})) { $stmt = "FROM " . $db->tableidentifier(NGCP::BulkProcessor::Dao::Trunk::accounting::cdr_export_status_data::gettablename()) . ' AS __cesd FORCE INDEX (PRIMARY)'; - unshift @intjoins, 'LEFT JOIN ' . $table . ' FORCE INDEX (PRIMARY) ON ' . $table . '.id = __cesd.cdr_id'; + unshift @intjoins, 'LEFT JOIN ' . $table_aliased . ' FORCE INDEX (PRIMARY) ON ' . ($alias ? $alias : $table) . '.id = __cesd.cdr_id'; push @conds, '__cesd.cdr_id <= ' . $static_context->{last_processed_cdr_id}; push @conds, '__cesd.status_id = ' . $static_context->{export_status_id}; push @conds, '__cesd.export_status = "' . $NGCP::BulkProcessor::Dao::Trunk::accounting::cdr_export_status_data::UNEXPORTED . '"'; + $order_by = ' ORDER BY __cesd.cdr_id'; } } @@ -526,11 +552,19 @@ sub _get_export_stmt_part { if (defined $conditions and (scalar @$conditions) > 0) { foreach my $f (@$conditions) { my ($field, $match) = %{ $f }; - my ($op, $val) = %{ $match }; - push @conds, "$field $op $val"; + my ($op, $val); + if (ref $match) { + ($op, $val) = %{ $match }; + } else { + $op = $match; + } + my $c = "$field $op"; + $c .= " $val" if defined $val; + push @conds, $c; } } $stmt .= " WHERE " . join(" AND ", @conds) if (scalar @conds) > 0; + $stmt .= $order_by if $order_by_id; return $stmt; } @@ -547,18 +581,37 @@ sub buildrecords_fromrows { $record = __PACKAGE__->new($row); # transformations go here ... + $record->load_relation($load_recursive,'source_user','NGCP::BulkProcessor::Dao::Trunk::billing::voip_subscribers::findby_uuid',$record->{source_user_id},$load_recursive) if $record->{source_user_id}; + $record->load_relation($load_recursive,'source_provider','NGCP::BulkProcessor::Dao::Trunk::billing::contracts::findby_id',$record->{source_provider_id},$load_recursive) if $record->{source_provider_id}; + $record->load_relation($load_recursive,'source_account','NGCP::BulkProcessor::Dao::Trunk::billing::contracts::findby_id',$record->{source_account_id},$load_recursive) if $record->{source_account_id}; + + $record->load_relation($load_recursive,'destination_user','NGCP::BulkProcessor::Dao::Trunk::billing::voip_subscribers::findby_uuid',$record->{destination_user_id},$load_recursive) if $record->{destination_user_id}; + $record->load_relation($load_recursive,'destination_provider','NGCP::BulkProcessor::Dao::Trunk::billing::contracts::findby_id',$record->{destination_provider_id},$load_recursive) if $record->{destination_provider_id}; + $record->load_relation($load_recursive,'destination_account','NGCP::BulkProcessor::Dao::Trunk::billing::contracts::findby_id',$record->{destination_account_id},$load_recursive) if $record->{destination_account_id}; + + $record->load_relation($load_recursive,'source_carrier_billing_fee','NGCP::BulkProcessor::Dao::Trunk::billing::billing_fees_history::findby_id',$record->{source_carrier_billing_fee_id},$load_recursive) if $record->{source_carrier_billing_fee_id}; + $record->load_relation($load_recursive,'source_reseller_billing_fee','NGCP::BulkProcessor::Dao::Trunk::billing::billing_fees_history::findby_id',$record->{source_reseller_billing_fee_id},$load_recursive) if $record->{source_reseller_billing_fee_id}; + $record->load_relation($load_recursive,'source_customer_billing_fee','NGCP::BulkProcessor::Dao::Trunk::billing::billing_fees_history::findby_id',$record->{source_customer_billing_fee_id},$load_recursive) if $record->{source_customer_billing_fee_id}; + $record->load_relation($load_recursive,'source_carrier_billing_zone','NGCP::BulkProcessor::Dao::Trunk::billing::billing_zones_history::findby_id',$record->{source_carrier_billing_zone_id},$load_recursive) if $record->{source_carrier_billing_zone_id}; + $record->load_relation($load_recursive,'source_reseller_billing_zone','NGCP::BulkProcessor::Dao::Trunk::billing::billing_zones_history::findby_id',$record->{source_reseller_billing_zone_id},$load_recursive) if $record->{source_reseller_billing_zone_id}; + $record->load_relation($load_recursive,'source_customer_billing_zone','NGCP::BulkProcessor::Dao::Trunk::billing::billing_zones_history::findby_id',$record->{source_customer_billing_zone_id},$load_recursive) if $record->{source_customer_billing_zone_id}; + + $record->load_relation($load_recursive,'destination_carrier_billing_fee','NGCP::BulkProcessor::Dao::Trunk::billing::billing_fees_history::findby_id',$record->{destination_carrier_billing_fee_id},$load_recursive) if $record->{destination_carrier_billing_fee_id}; + $record->load_relation($load_recursive,'destination_reseller_billing_fee','NGCP::BulkProcessor::Dao::Trunk::billing::billing_fees_history::findby_id',$record->{destination_reseller_billing_fee_id},$load_recursive) if $record->{destination_reseller_billing_fee_id}; + $record->load_relation($load_recursive,'destination_customer_billing_fee','NGCP::BulkProcessor::Dao::Trunk::billing::billing_fees_history::findby_id',$record->{destination_customer_billing_fee_id},$load_recursive) if $record->{destination_customer_billing_fee_id}; + $record->load_relation($load_recursive,'destination_carrier_billing_zone','NGCP::BulkProcessor::Dao::Trunk::billing::billing_zones_history::findby_id',$record->{destination_carrier_billing_zone_id},$load_recursive) if $record->{destination_carrier_billing_zone_id}; + $record->load_relation($load_recursive,'destination_reseller_billing_zone','NGCP::BulkProcessor::Dao::Trunk::billing::billing_zones_history::findby_id',$record->{destination_reseller_billing_zone_id},$load_recursive) if $record->{destination_reseller_billing_zone_id}; + $record->load_relation($load_recursive,'destination_customer_billing_zone','NGCP::BulkProcessor::Dao::Trunk::billing::billing_zones_history::findby_id',$record->{destination_customer_billing_zone_id},$load_recursive) if $record->{destination_customer_billing_zone_id}; + $record->load_relation($load_recursive,'cdr_groups','NGCP::BulkProcessor::Dao::Trunk::accounting::cdr_group::findby_cdrid',undef,$record->{id},$load_recursive); $record->load_relation($load_recursive,'cdr_tags','NGCP::BulkProcessor::Dao::Trunk::accounting::cdr_tag_data::findby_cdrid',undef,$record->{id},$load_recursive); + $record->load_relation($load_recursive,'cdr_mos','NGCP::BulkProcessor::Dao::Trunk::accounting::cdr_mos_data::findby_cdrid',undef,$record->{id},$load_recursive); + $record->load_relation($load_recursive,'cdr_status','NGCP::BulkProcessor::Dao::Trunk::accounting::cdr_export_status_data::findby_cdrid',undef,$record->{id},$load_recursive); + $record->load_relation($load_recursive,'cdr_relations','NGCP::BulkProcessor::Dao::Trunk::accounting::cdr_relation_data::findby_cdrid',undef,$record->{id},$load_recursive); + $record->load_relation($load_recursive,'cdr_presentity','NGCP::BulkProcessor::Dao::Trunk::accounting::cdr_presentity::findby_callid',undef,$record->{call_id},$load_recursive); + $record->load_relation($load_recursive,'cdr_cash_balances','NGCP::BulkProcessor::Dao::Trunk::accounting::cdr_cash_balance_data::findby_cdrid',undef,$record->{id},$load_recursive); + $record->load_relation($load_recursive,'cdr_time_balances','NGCP::BulkProcessor::Dao::Trunk::accounting::cdr_time_balance_data::findby_cdrid',undef,$record->{id},$load_recursive); - - #$record->load_relation($load_recursive,'cdr_cash_balance - #$record->load_relation($load_recursive,'cdr_export_status - #$record->load_relation($load_recursive,'cdr_mos_data - #$record->load_relation($load_recursive,'cdr_relation - #$record->load_relation($load_recursive,'cdr_time_balance - - - push @records,$record; } } diff --git a/lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_cash_balance_data.pm b/lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_cash_balance_data.pm new file mode 100644 index 0000000..690e525 --- /dev/null +++ b/lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_cash_balance_data.pm @@ -0,0 +1,145 @@ +package NGCP::BulkProcessor::Dao::Trunk::accounting::cdr_cash_balance_data; +use strict; + +## no critic + +use NGCP::BulkProcessor::Logging qw( + getlogger +); + +use NGCP::BulkProcessor::ConnectorPool qw( + get_accounting_db + destroy_dbs +); + +use NGCP::BulkProcessor::SqlProcessor qw( + checktableinfo + copy_row + + insert_record +); +use NGCP::BulkProcessor::SqlRecord qw(); + +use NGCP::BulkProcessor::Dao::Trunk::billing::contract_balances qw(); +use NGCP::BulkProcessor::Dao::Trunk::accounting::cdr_provider qw(); +use NGCP::BulkProcessor::Dao::Trunk::accounting::cdr_direction qw(); + +require Exporter; +our @ISA = qw(Exporter NGCP::BulkProcessor::SqlRecord); +our @EXPORT_OK = qw( + gettablename + settablename + check_table + + findby_cdrid + + insert_row +); + +my $tablename = 'cdr_cash_balance_data'; +my $get_db = \&get_accounting_db; + +my $expected_fieldnames = [ + "cdr_id", + "provider_id", + "direction_id", + "cash_balance_id", + "val_before", + "val_after", + "cdr_start_time", +]; + +my $indexes = {}; + +my $insert_unique_fields = []; + +sub new { + + my $class = shift; + my $self = NGCP::BulkProcessor::SqlRecord->new($class,$get_db, + $tablename,$expected_fieldnames,$indexes); + + copy_row($self,shift,$expected_fieldnames); + + return $self; + +} + +sub findby_cdrid { + + my ($xa_db,$cdrid,$load_recursive) = @_; + + check_table(); + my $db = &$get_db(); + $xa_db //= $db; + my $table = $db->tableidentifier($tablename); + + my $stmt = 'SELECT * FROM ' . $table . ' WHERE ' . + $db->columnidentifier('cdr_id') . ' = ?'; + my @params = ($cdrid); + my $rows = $xa_db->db_get_all_arrayref($stmt,@params); + + return buildrecords_fromrows($rows,$load_recursive); + +} + +sub insert_row { + + my $db = &$get_db(); + my $xa_db = shift // $db; + + my ($data,$insert_ignore) = @_; + check_table(); + if (insert_record($db,$xa_db,__PACKAGE__,$data,$insert_ignore,$insert_unique_fields)) { + return $xa_db->db_last_insert_id() || 1; + } + return undef; + +} + +sub buildrecords_fromrows { + + my ($rows,$load_recursive) = @_; + + my @records = (); + my $record; + + if (defined $rows and ref $rows eq 'ARRAY') { + foreach my $row (@$rows) { + $record = __PACKAGE__->new($row); + + # transformations go here ... + $record->load_relation($load_recursive,'contract_balance','NGCP::BulkProcessor::Dao::Trunk::billing::contract_balances::findby_id',undef,$record->{cash_balance_id},$load_recursive); + $record->load_relation($load_recursive,'direction','NGCP::BulkProcessor::Dao::Trunk::accounting::cdr_direction::findby_id_cached',$record->{direction_id},$load_recursive); + $record->load_relation($load_recursive,'provider','NGCP::BulkProcessor::Dao::Trunk::accounting::cdr_provider::findby_id_cached',$record->{provider_id},$load_recursive); + + push @records,$record; + } + } + + return \@records; + +} + +sub gettablename { + + return $tablename; + +} + +sub settablename { + + $tablename = shift; + +} + +sub check_table { + + return checktableinfo($get_db, + __PACKAGE__,$tablename, + $expected_fieldnames, + $indexes); + +} + +1; diff --git a/lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_direction.pm b/lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_direction.pm index d0322d6..1bc52c3 100644 --- a/lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_direction.pm +++ b/lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_direction.pm @@ -19,6 +19,8 @@ use NGCP::BulkProcessor::SqlProcessor qw( ); use NGCP::BulkProcessor::SqlRecord qw(); +use NGCP::BulkProcessor::Array qw(array_to_map); + require Exporter; our @ISA = qw(Exporter NGCP::BulkProcessor::SqlRecord); our @EXPORT_OK = qw( @@ -26,6 +28,8 @@ our @EXPORT_OK = qw( check_table findall + findby_id + findby_id_cached $SOURCE $DESTINATION @@ -76,6 +80,35 @@ sub findall { } +my $cdr_direction_map; + +sub findby_id_cached { + my ($id,$load_recursive) = @_; + unless ($cdr_direction_map) { + ($cdr_direction_map, my $directions, my $ids) = array_to_map(findall($load_recursive), + sub { return shift->{id}; }, sub { return shift; }, 'last'); + } + return __PACKAGE__->new($cdr_direction_map->{$id}) if defined $id; + return; +} + +sub findby_id { + + my ($id,$load_recursive) = @_; + + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + + my $stmt = 'SELECT * FROM ' . $table . ' WHERE ' . + $db->columnidentifier('id') . ' = ?'; + my @params = ($id); + my $rows = $db->db_get_all_arrayref($stmt,@params); + + return buildrecords_fromrows($rows,$load_recursive); + +} + sub buildrecords_fromrows { my ($rows,$load_recursive) = @_; diff --git a/lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_export_status.pm b/lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_export_status.pm index aafe4b6..3f5e196 100644 --- a/lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_export_status.pm +++ b/lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_export_status.pm @@ -15,18 +15,24 @@ use NGCP::BulkProcessor::ConnectorPool qw( use NGCP::BulkProcessor::SqlProcessor qw( checktableinfo copy_row - + insert_record ); use NGCP::BulkProcessor::SqlRecord qw(); +use NGCP::BulkProcessor::Array qw(array_to_map); + require Exporter; our @ISA = qw(Exporter NGCP::BulkProcessor::SqlRecord); our @EXPORT_OK = qw( gettablename check_table + insert_row + findall findby_type + findby_id + findby_id_cached ); my $tablename = 'cdr_export_status'; @@ -39,7 +45,7 @@ my $expected_fieldnames = [ my $indexes = {}; -my $insert_unique_fields = []; +my $insert_unique_fields = [ 'type' ]; sub new { @@ -68,6 +74,36 @@ sub findall { } +my $cdr_export_status_map; + +sub findby_id_cached { + my ($id,$load_recursive) = @_; + unless ($cdr_export_status_map) { + ($cdr_export_status_map, my $streams, my $ids) = array_to_map(findall($load_recursive), + sub { return shift->{id}; }, sub { return shift; }, 'last'); + } + return __PACKAGE__->new($cdr_export_status_map->{$id}) if defined $id; + return; +} + +sub findby_id { + + my ($xa_db,$id,$load_recursive) = @_; + + check_table(); + my $db = &$get_db(); + $xa_db //= $db; + my $table = $db->tableidentifier($tablename); + + my $stmt = 'SELECT * FROM ' . $table . ' WHERE ' . + $db->columnidentifier('id') . ' = ?'; + my @params = ($id); + my $rows = $xa_db->db_get_all_arrayref($stmt,@params); + + return buildrecords_fromrows($rows,$load_recursive); + +} + sub findby_type { my ($type,$load_recursive) = @_; @@ -97,6 +133,7 @@ sub buildrecords_fromrows { $record = __PACKAGE__->new($row); # transformations go here ... + push @records,$record; } @@ -106,6 +143,35 @@ sub buildrecords_fromrows { } +sub insert_row { + + my $db = &$get_db(); + my $xa_db = shift // $db; + if ('HASH' eq ref $_[0]) { + my ($data,$insert_ignore) = @_; + check_table(); + if (insert_record($db,$xa_db,__PACKAGE__,$data,$insert_ignore,$insert_unique_fields)) { + return $xa_db->db_last_insert_id(); + } + } else { + my %params = @_; + my ($type) = @params{qw/ + type + /}; + + if ($xa_db->db_do('INSERT INTO ' . $db->tableidentifier($tablename) . ' (' . + $db->columnidentifier('type') . ') VALUES (' . + '?)', + $type, + )) { + rowinserted($db,$tablename,getlogger(__PACKAGE__)); + return $xa_db->db_last_insert_id(); + } + } + return undef; + +} + sub gettablename { return $tablename; diff --git a/lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_export_status_data.pm b/lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_export_status_data.pm index 235a040..20e5b1c 100644 --- a/lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_export_status_data.pm +++ b/lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_export_status_data.pm @@ -8,6 +8,7 @@ use NGCP::BulkProcessor::Logging qw( rowsupdated rowinserted rowupserted + rowsupserted rowupdated ); @@ -25,6 +26,10 @@ use NGCP::BulkProcessor::SqlProcessor qw( ); use NGCP::BulkProcessor::SqlRecord qw(); +use NGCP::BulkProcessor::Dao::Trunk::accounting::cdr_export_status qw(); + +use NGCP::BulkProcessor::Array qw(flatten); + require Exporter; our @ISA = qw(Exporter NGCP::BulkProcessor::SqlRecord); our @EXPORT_OK = qw( @@ -34,15 +39,19 @@ our @EXPORT_OK = qw( update_row insert_row upsert_row + upsert_rows find_last_processed_cdrid update_export_status + findby_cdrid $UNEXPORTED $OK $FAILED $SKIPPED + + @STATES ); our $UNEXPORTED = 'unexported'; @@ -50,6 +59,8 @@ our $OK = 'ok'; our $FAILED = 'failed'; our $SKIPPED = 'skipped'; +our @STATES = ($UNEXPORTED, $OK, $FAILED, $SKIPPED); + my $tablename = 'cdr_export_status_data'; my $get_db = \&get_accounting_db; @@ -100,6 +111,24 @@ sub find_last_processed_cdrid { return $db->db_get_value($stmt,@params); } + +sub findby_cdrid { + + my ($xa_db,$cdrid,$load_recursive) = @_; + + check_table(); + my $db = &$get_db(); + $xa_db //= $db; + my $table = $db->tableidentifier($tablename); + + my $stmt = 'SELECT * FROM ' . $table . ' WHERE ' . + $db->columnidentifier('cdr_id') . ' = ?'; + my @params = ($cdrid); + my $rows = $xa_db->db_get_all_arrayref($stmt,@params); + + return buildrecords_fromrows($rows,$load_recursive); + +} sub update_row { @@ -199,6 +228,34 @@ sub upsert_row { } +sub upsert_rows { + + my $db = &$get_db(); + my $xa_db = shift // $db; + + my ($rows,$export_status) = @_; + + my $result; + if ($result = $xa_db->db_do('INSERT INTO ' . $db->tableidentifier($tablename) . ' (' . + $db->columnidentifier('cdr_id') . ', ' . + $db->columnidentifier('status_id') . ', ' . + $db->columnidentifier('export_status') . ', ' . + $db->columnidentifier('exported_at') . ', ' . + $db->columnidentifier('cdr_start_time') . ') VALUES ' . + join(',',('(?,?,?,NOW(),?)') x scalar @$rows) . + ' ON DUPLICATE KEY UPDATE ' . + 'export_status = ?, ' . + 'exported_at = IF(export_status = "' . $UNEXPORTED . '",NOW(),exported_at)', + flatten(@$rows), + $export_status, + )) { + rowsupserted($db,$tablename,getlogger(__PACKAGE__)); + } + + return $result; # 0 .. no change, 1 .. inserted, 2 .. updated + +} + sub update_export_status { my ($status_id,$export_status,$start_time_from,$start_time_to,$call_ids) = @_; @@ -252,6 +309,8 @@ sub buildrecords_fromrows { $record = __PACKAGE__->new($row); # transformations go here ... + #$record->load_relation($load_recursive,'status','NGCP::BulkProcessor::Dao::Trunk::accounting::cdr_export_status::findby_id',undef,$record->{status_id},$load_recursive); + $record->load_relation($load_recursive,'status','NGCP::BulkProcessor::Dao::Trunk::accounting::cdr_export_status::findby_id_cached',$record->{id},$load_recursive); push @records,$record; } diff --git a/lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_group.pm b/lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_group.pm index d6769e4..fefdb52 100644 --- a/lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_group.pm +++ b/lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_group.pm @@ -120,6 +120,7 @@ sub buildrecords_fromrows { $record = __PACKAGE__->new($row); # transformations go here ... + $record->load_relation($load_recursive,'cdrs','NGCP::BulkProcessor::Dao::Trunk::accounting::cdr::findby_callid',undef,$record->{call_id},undef,undef,undef,$load_recursive); push @records,$record; } diff --git a/lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_mos_data.pm b/lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_mos_data.pm new file mode 100644 index 0000000..8e0ed36 --- /dev/null +++ b/lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_mos_data.pm @@ -0,0 +1,137 @@ +package NGCP::BulkProcessor::Dao::Trunk::accounting::cdr_mos_data; +use strict; + +## no critic + +use NGCP::BulkProcessor::Logging qw( + getlogger +); + +use NGCP::BulkProcessor::ConnectorPool qw( + get_accounting_db + destroy_dbs +); + +use NGCP::BulkProcessor::SqlProcessor qw( + checktableinfo + copy_row + + insert_record +); +use NGCP::BulkProcessor::SqlRecord qw(); + +require Exporter; +our @ISA = qw(Exporter NGCP::BulkProcessor::SqlRecord); +our @EXPORT_OK = qw( + gettablename + settablename + check_table + + findby_cdrid + + insert_row +); + +my $tablename = 'cdr_mos_data'; +my $get_db = \&get_accounting_db; + +my $expected_fieldnames = [ + "cdr_id", + "mos_average", + "mos_average_packetloss", + "mos_average_jitter", + "mos_average_roundtrip", + "cdr_start_time", +]; + +my $indexes = {}; + +my $insert_unique_fields = []; + +sub new { + + my $class = shift; + my $self = NGCP::BulkProcessor::SqlRecord->new($class,$get_db, + $tablename,$expected_fieldnames,$indexes); + + copy_row($self,shift,$expected_fieldnames); + + return $self; + +} + +sub findby_cdrid { + + my ($xa_db,$cdrid,$load_recursive) = @_; + + check_table(); + my $db = &$get_db(); + $xa_db //= $db; + my $table = $db->tableidentifier($tablename); + + my $stmt = 'SELECT * FROM ' . $table . ' WHERE ' . + $db->columnidentifier('cdr_id') . ' = ?'; + my @params = ($cdrid); + my $rows = $xa_db->db_get_all_arrayref($stmt,@params); + + return buildrecords_fromrows($rows,$load_recursive); + +} + +sub insert_row { + + my $db = &$get_db(); + my $xa_db = shift // $db; + + my ($data,$insert_ignore) = @_; + check_table(); + if (insert_record($db,$xa_db,__PACKAGE__,$data,$insert_ignore,$insert_unique_fields)) { + return $xa_db->db_last_insert_id() || 1; + } + return undef; + +} + +sub buildrecords_fromrows { + + my ($rows,$load_recursive) = @_; + + my @records = (); + my $record; + + if (defined $rows and ref $rows eq 'ARRAY') { + foreach my $row (@$rows) { + $record = __PACKAGE__->new($row); + + # transformations go here ... + + push @records,$record; + } + } + + return \@records; + +} + +sub gettablename { + + return $tablename; + +} + +sub settablename { + + $tablename = shift; + +} + +sub check_table { + + return checktableinfo($get_db, + __PACKAGE__,$tablename, + $expected_fieldnames, + $indexes); + +} + +1; diff --git a/lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_presentity.pm b/lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_presentity.pm new file mode 100644 index 0000000..755dd7b --- /dev/null +++ b/lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_presentity.pm @@ -0,0 +1,136 @@ +package NGCP::BulkProcessor::Dao::Trunk::accounting::cdr_presentity; +use strict; + +## no critic + +use NGCP::BulkProcessor::Logging qw( + getlogger +); + +use NGCP::BulkProcessor::ConnectorPool qw( + get_accounting_db + destroy_dbs +); + +use NGCP::BulkProcessor::SqlProcessor qw( + checktableinfo + copy_row + + insert_record +); +use NGCP::BulkProcessor::SqlRecord qw(); + +require Exporter; +our @ISA = qw(Exporter NGCP::BulkProcessor::SqlRecord); +our @EXPORT_OK = qw( + gettablename + settablename + check_table + + findby_callid + + insert_row +); + +my $tablename = 'cdr_presentity'; +my $get_db = \&get_accounting_db; + +my $expected_fieldnames = [ + "call_id", + "event", + "received_time", + "body", +]; + + +my $indexes = {}; + +my $insert_unique_fields = []; + +sub new { + + my $class = shift; + my $self = NGCP::BulkProcessor::SqlRecord->new($class,$get_db, + $tablename,$expected_fieldnames,$indexes); + + copy_row($self,shift,$expected_fieldnames); + + return $self; + +} + +sub findby_callid { + + my ($xa_db,$callid,$load_recursive) = @_; + + check_table(); + my $db = &$get_db(); + $xa_db //= $db; + my $table = $db->tableidentifier($tablename); + + my $stmt = 'SELECT * FROM ' . $table . ' WHERE ' . + $db->columnidentifier('call_id') . ' = ?'; + my @params = ($callid); + my $rows = $xa_db->db_get_all_arrayref($stmt,@params); + + return buildrecords_fromrows($rows,$load_recursive); + +} + +sub insert_row { + + my $db = &$get_db(); + my $xa_db = shift // $db; + + my ($data,$insert_ignore) = @_; + check_table(); + if (insert_record($db,$xa_db,__PACKAGE__,$data,$insert_ignore,$insert_unique_fields)) { + return $xa_db->db_last_insert_id() || 1; + } + return undef; + +} + +sub buildrecords_fromrows { + + my ($rows,$load_recursive) = @_; + + my @records = (); + my $record; + + if (defined $rows and ref $rows eq 'ARRAY') { + foreach my $row (@$rows) { + $record = __PACKAGE__->new($row); + + # transformations go here ... + + push @records,$record; + } + } + + return \@records; + +} + +sub gettablename { + + return $tablename; + +} + +sub settablename { + + $tablename = shift; + +} + +sub check_table { + + return checktableinfo($get_db, + __PACKAGE__,$tablename, + $expected_fieldnames, + $indexes); + +} + +1; diff --git a/lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_provider.pm b/lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_provider.pm index 8b607fb..4158d0e 100644 --- a/lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_provider.pm +++ b/lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_provider.pm @@ -19,6 +19,8 @@ use NGCP::BulkProcessor::SqlProcessor qw( ); use NGCP::BulkProcessor::SqlRecord qw(); +use NGCP::BulkProcessor::Array qw(array_to_map); + require Exporter; our @ISA = qw(Exporter NGCP::BulkProcessor::SqlRecord); our @EXPORT_OK = qw( @@ -26,6 +28,8 @@ our @EXPORT_OK = qw( check_table findall + findby_id + findby_id_cached $CUSTOMER $RESELLER @@ -78,6 +82,36 @@ sub findall { } + +my $cdr_provider_map; + +sub findby_id_cached { + my ($id,$load_recursive) = @_; + unless ($cdr_provider_map) { + ($cdr_provider_map, my $providers, my $ids) = array_to_map(findall($load_recursive), + sub { return shift->{id}; }, sub { return shift; }, 'last'); + } + return __PACKAGE__->new($cdr_provider_map->{$id}) if defined $id; + return; +} + +sub findby_id { + + my ($id,$load_recursive) = @_; + + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + + my $stmt = 'SELECT * FROM ' . $table . ' WHERE ' . + $db->columnidentifier('id') . ' = ?'; + my @params = ($id); + my $rows = $db->db_get_all_arrayref($stmt,@params); + + return buildrecords_fromrows($rows,$load_recursive); + +} + sub buildrecords_fromrows { my ($rows,$load_recursive) = @_; diff --git a/lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_relation.pm b/lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_relation.pm new file mode 100644 index 0000000..80cc2f8 --- /dev/null +++ b/lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_relation.pm @@ -0,0 +1,143 @@ +package NGCP::BulkProcessor::Dao::Trunk::accounting::cdr_relation; +use strict; + +## no critic + +use NGCP::BulkProcessor::Logging qw( + getlogger +); + +use NGCP::BulkProcessor::ConnectorPool qw( + get_accounting_db + destroy_dbs +); + +use NGCP::BulkProcessor::SqlProcessor qw( + checktableinfo + copy_row + +); +use NGCP::BulkProcessor::SqlRecord qw(); + +use NGCP::BulkProcessor::Array qw(array_to_map); + +require Exporter; +our @ISA = qw(Exporter NGCP::BulkProcessor::SqlRecord); +our @EXPORT_OK = qw( + gettablename + check_table + + findall + findby_id + findby_id_cached + +); + + +my $tablename = 'cdr_relation'; +my $get_db = \&get_accounting_db; + +my $expected_fieldnames = [ + "id", + "type", +]; + + +my $indexes = {}; + +my $insert_unique_fields = []; + +sub new { + + my $class = shift; + my $self = NGCP::BulkProcessor::SqlRecord->new($class,$get_db, + $tablename,$expected_fieldnames,$indexes); + + copy_row($self,shift,$expected_fieldnames); + + return $self; + +} + +sub findall { + + my ($load_recursive) = @_; + + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + + my $stmt = 'SELECT * FROM ' . $table; + my @params = (); + my $rows = $db->db_get_all_arrayref($stmt,@params); + + return buildrecords_fromrows($rows,$load_recursive); + +} + +my $cdr_relation_map; + +sub findby_id_cached { + my ($id,$load_recursive) = @_; + unless ($cdr_relation_map) { + ($cdr_relation_map, my $relations, my $ids) = array_to_map(findall($load_recursive), + sub { return shift->{id}; }, sub { return shift; }, 'last'); + } + return __PACKAGE__->new($cdr_relation_map->{$id}) if defined $id; + return; +} + +sub findby_id { + + my ($id,$load_recursive) = @_; + + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + + my $stmt = 'SELECT * FROM ' . $table . ' WHERE ' . + $db->columnidentifier('id') . ' = ?'; + my @params = ($id); + my $rows = $db->db_get_all_arrayref($stmt,@params); + + return buildrecords_fromrows($rows,$load_recursive); + +} + +sub buildrecords_fromrows { + + my ($rows,$load_recursive) = @_; + + my @records = (); + my $record; + + if (defined $rows and ref $rows eq 'ARRAY') { + foreach my $row (@$rows) { + $record = __PACKAGE__->new($row); + + # transformations go here ... + + push @records,$record; + } + } + + return \@records; + +} + +sub gettablename { + + return $tablename; + +} + +sub check_table { + + return checktableinfo($get_db, + __PACKAGE__,$tablename, + $expected_fieldnames, + $indexes); + +} + +1; diff --git a/lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_relation_data.pm b/lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_relation_data.pm new file mode 100644 index 0000000..88f49ba --- /dev/null +++ b/lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_relation_data.pm @@ -0,0 +1,184 @@ +package NGCP::BulkProcessor::Dao::Trunk::accounting::cdr_relation_data; +use strict; + +## no critic + +use NGCP::BulkProcessor::Logging qw( + getlogger +); + +use NGCP::BulkProcessor::ConnectorPool qw( + get_accounting_db + destroy_dbs +); + +use NGCP::BulkProcessor::SqlProcessor qw( + checktableinfo + copy_row + + insert_record +); +use NGCP::BulkProcessor::SqlRecord qw(); + +use NGCP::BulkProcessor::Dao::Trunk::accounting::cdr_relation qw(); +use NGCP::BulkProcessor::Dao::Trunk::accounting::cdr_provider qw(); +use NGCP::BulkProcessor::Dao::Trunk::accounting::cdr_direction qw(); + +require Exporter; +our @ISA = qw(Exporter NGCP::BulkProcessor::SqlRecord); +our @EXPORT_OK = qw( + gettablename + settablename + check_table + + findby_cdrproviderdirectiontag + findby_cdrid + + insert_row +); + +my $tablename = 'cdr_relation_data'; +my $get_db = \&get_accounting_db; + +my $expected_fieldnames = [ + "cdr_id", + "provider_id", + "direction_id", + "relation_id", + "val", + "cdr_start_time", +]; + +my $indexes = {}; + +my $insert_unique_fields = []; + +sub new { + + my $class = shift; + my $self = NGCP::BulkProcessor::SqlRecord->new($class,$get_db, + $tablename,$expected_fieldnames,$indexes); + + copy_row($self,shift,$expected_fieldnames); + + return $self; + +} + +sub findby_cdrproviderdirectiontag { + + my ($xa_db,$cdrid,$providerid,$directionid,$tagid,$load_recursive) = @_; + + check_table(); + my $db = &$get_db(); + $xa_db //= $db; + my $table = $db->tableidentifier($tablename); + + my $stmt = 'SELECT * FROM ' . $table . ' WHERE ' . + $db->columnidentifier('cdr_id') . ' = ?' . + ' AND ' . $db->columnidentifier('provider_id') . ' = ?' . + ' AND ' . $db->columnidentifier('direction_id') . ' = ?' . + ' AND ' . $db->columnidentifier('tag_id') . ' = ?'; + my @params = ($cdrid,$providerid,$directionid,$tagid); + my $rows = $xa_db->db_get_all_arrayref($stmt,@params); + + return buildrecords_fromrows($rows,$load_recursive); + +} + +sub findby_cdrid { + + my ($xa_db,$cdrid,$load_recursive) = @_; + + check_table(); + my $db = &$get_db(); + $xa_db //= $db; + my $table = $db->tableidentifier($tablename); + + my $stmt = 'SELECT * FROM ' . $table . ' WHERE ' . + $db->columnidentifier('cdr_id') . ' = ?'; + my @params = ($cdrid); + my $rows = $xa_db->db_get_all_arrayref($stmt,@params); + + return buildrecords_fromrows($rows,$load_recursive); + +} + +sub findby_callid { + + my ($xa_db,$callid,$load_recursive) = @_; + + check_table(); + my $db = &$get_db(); + $xa_db //= $db; + my $table = $db->tableidentifier($tablename); + + my $stmt = 'SELECT * FROM ' . $table . ' WHERE ' . + $db->columnidentifier('call_id') . ' = ?'; + my @params = ($callid); + my $rows = $xa_db->db_get_all_arrayref($stmt,@params); + + return buildrecords_fromrows($rows,$load_recursive); + +} + +sub insert_row { + + my $db = &$get_db(); + my $xa_db = shift // $db; + + my ($data,$insert_ignore) = @_; + check_table(); + if (insert_record($db,$xa_db,__PACKAGE__,$data,$insert_ignore,$insert_unique_fields)) { + return $xa_db->db_last_insert_id() || 1; + } + return undef; + +} + +sub buildrecords_fromrows { + + my ($rows,$load_recursive) = @_; + + my @records = (); + my $record; + + if (defined $rows and ref $rows eq 'ARRAY') { + foreach my $row (@$rows) { + $record = __PACKAGE__->new($row); + + # transformations go here ... + $record->load_relation($load_recursive,'relation','NGCP::BulkProcessor::Dao::Trunk::accounting::cdr_relation::findby_id_cached',$record->{relation_id},$load_recursive); + $record->load_relation($load_recursive,'direction','NGCP::BulkProcessor::Dao::Trunk::accounting::cdr_direction::findby_id_cached',$record->{direction_id},$load_recursive); + $record->load_relation($load_recursive,'provider','NGCP::BulkProcessor::Dao::Trunk::accounting::cdr_provider::findby_id_cached',$record->{provider_id},$load_recursive); + + push @records,$record; + } + } + + return \@records; + +} + +sub gettablename { + + return $tablename; + +} + +sub settablename { + + $tablename = shift; + +} + +sub check_table { + + return checktableinfo($get_db, + __PACKAGE__,$tablename, + $expected_fieldnames, + $indexes); + +} + +1; diff --git a/lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_tag.pm b/lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_tag.pm index 849163c..518ae14 100644 --- a/lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_tag.pm +++ b/lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_tag.pm @@ -19,6 +19,8 @@ use NGCP::BulkProcessor::SqlProcessor qw( ); use NGCP::BulkProcessor::SqlRecord qw(); +use NGCP::BulkProcessor::Array qw(array_to_map); + require Exporter; our @ISA = qw(Exporter NGCP::BulkProcessor::SqlRecord); our @EXPORT_OK = qw( @@ -26,6 +28,8 @@ our @EXPORT_OK = qw( check_table findall + findby_id + findby_id_cached $CALLING_PARTY_CATEGORY $FURNISHED_CHARGING_INFO @@ -82,6 +86,35 @@ sub findall { } +my $cdr_tag_map; + +sub findby_id_cached { + my ($id,$load_recursive) = @_; + unless ($cdr_tag_map) { + ($cdr_tag_map, my $types, my $ids) = array_to_map(findall($load_recursive), + sub { return shift->{id}; }, sub { return shift; }, 'last'); + } + return __PACKAGE__->new($cdr_tag_map->{$id}) if defined $id; + return; +} + +sub findby_id { + + my ($id,$load_recursive) = @_; + + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + + my $stmt = 'SELECT * FROM ' . $table . ' WHERE ' . + $db->columnidentifier('id') . ' = ?'; + my @params = ($id); + my $rows = $db->db_get_all_arrayref($stmt,@params); + + return buildrecords_fromrows($rows,$load_recursive); + +} + sub buildrecords_fromrows { my ($rows,$load_recursive) = @_; diff --git a/lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_tag_data.pm b/lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_tag_data.pm index efe5d68..186ac39 100644 --- a/lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_tag_data.pm +++ b/lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_tag_data.pm @@ -20,6 +20,10 @@ use NGCP::BulkProcessor::SqlProcessor qw( ); use NGCP::BulkProcessor::SqlRecord qw(); +use NGCP::BulkProcessor::Dao::Trunk::accounting::cdr_tag qw(); +use NGCP::BulkProcessor::Dao::Trunk::accounting::cdr_provider qw(); +use NGCP::BulkProcessor::Dao::Trunk::accounting::cdr_direction qw(); + require Exporter; our @ISA = qw(Exporter NGCP::BulkProcessor::SqlRecord); our @EXPORT_OK = qw( @@ -100,24 +104,6 @@ sub findby_cdrid { } -sub findby_callid { - - my ($xa_db,$callid,$load_recursive) = @_; - - check_table(); - my $db = &$get_db(); - $xa_db //= $db; - my $table = $db->tableidentifier($tablename); - - my $stmt = 'SELECT * FROM ' . $table . ' WHERE ' . - $db->columnidentifier('call_id') . ' = ?'; - my @params = ($callid); - my $rows = $xa_db->db_get_all_arrayref($stmt,@params); - - return buildrecords_fromrows($rows,$load_recursive); - -} - sub insert_row { my $db = &$get_db(); @@ -144,6 +130,9 @@ sub buildrecords_fromrows { $record = __PACKAGE__->new($row); # transformations go here ... + $record->load_relation($load_recursive,'tag','NGCP::BulkProcessor::Dao::Trunk::accounting::cdr_tag::findby_id_cached',$record->{tag_id},$load_recursive); + $record->load_relation($load_recursive,'direction','NGCP::BulkProcessor::Dao::Trunk::accounting::cdr_direction::findby_id_cached',$record->{direction_id},$load_recursive); + $record->load_relation($load_recursive,'provider','NGCP::BulkProcessor::Dao::Trunk::accounting::cdr_provider::findby_id_cached',$record->{provider_id},$load_recursive); push @records,$record; } diff --git a/lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_time_balance_data.pm b/lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_time_balance_data.pm new file mode 100644 index 0000000..4554681 --- /dev/null +++ b/lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_time_balance_data.pm @@ -0,0 +1,145 @@ +package NGCP::BulkProcessor::Dao::Trunk::accounting::cdr_time_balance_data; +use strict; + +## no critic + +use NGCP::BulkProcessor::Logging qw( + getlogger +); + +use NGCP::BulkProcessor::ConnectorPool qw( + get_accounting_db + destroy_dbs +); + +use NGCP::BulkProcessor::SqlProcessor qw( + checktableinfo + copy_row + + insert_record +); +use NGCP::BulkProcessor::SqlRecord qw(); + +use NGCP::BulkProcessor::Dao::Trunk::billing::contract_balances qw(); +use NGCP::BulkProcessor::Dao::Trunk::accounting::cdr_provider qw(); +use NGCP::BulkProcessor::Dao::Trunk::accounting::cdr_direction qw(); + +require Exporter; +our @ISA = qw(Exporter NGCP::BulkProcessor::SqlRecord); +our @EXPORT_OK = qw( + gettablename + settablename + check_table + + findby_cdrid + + insert_row +); + +my $tablename = 'cdr_time_balance_data'; +my $get_db = \&get_accounting_db; + +my $expected_fieldnames = [ + "cdr_id", + "provider_id", + "direction_id", + "time_balance_id", + "val_before", + "val_after", + "cdr_start_time", +]; + +my $indexes = {}; + +my $insert_unique_fields = []; + +sub new { + + my $class = shift; + my $self = NGCP::BulkProcessor::SqlRecord->new($class,$get_db, + $tablename,$expected_fieldnames,$indexes); + + copy_row($self,shift,$expected_fieldnames); + + return $self; + +} + +sub findby_cdrid { + + my ($xa_db,$cdrid,$load_recursive) = @_; + + check_table(); + my $db = &$get_db(); + $xa_db //= $db; + my $table = $db->tableidentifier($tablename); + + my $stmt = 'SELECT * FROM ' . $table . ' WHERE ' . + $db->columnidentifier('cdr_id') . ' = ?'; + my @params = ($cdrid); + my $rows = $xa_db->db_get_all_arrayref($stmt,@params); + + return buildrecords_fromrows($rows,$load_recursive); + +} + +sub insert_row { + + my $db = &$get_db(); + my $xa_db = shift // $db; + + my ($data,$insert_ignore) = @_; + check_table(); + if (insert_record($db,$xa_db,__PACKAGE__,$data,$insert_ignore,$insert_unique_fields)) { + return $xa_db->db_last_insert_id() || 1; + } + return undef; + +} + +sub buildrecords_fromrows { + + my ($rows,$load_recursive) = @_; + + my @records = (); + my $record; + + if (defined $rows and ref $rows eq 'ARRAY') { + foreach my $row (@$rows) { + $record = __PACKAGE__->new($row); + + # transformations go here ... + $record->load_relation($load_recursive,'contract_balance','NGCP::BulkProcessor::Dao::Trunk::billing::contract_balances::findby_id',undef,$record->{time_balance_id},$load_recursive); + $record->load_relation($load_recursive,'direction','NGCP::BulkProcessor::Dao::Trunk::accounting::cdr_direction::findby_id_cached',$record->{direction_id},$load_recursive); + $record->load_relation($load_recursive,'provider','NGCP::BulkProcessor::Dao::Trunk::accounting::cdr_provider::findby_id_cached',$record->{provider_id},$load_recursive); + + push @records,$record; + } + } + + return \@records; + +} + +sub gettablename { + + return $tablename; + +} + +sub settablename { + + $tablename = shift; + +} + +sub check_table { + + return checktableinfo($get_db, + __PACKAGE__,$tablename, + $expected_fieldnames, + $indexes); + +} + +1; diff --git a/lib/NGCP/BulkProcessor/Dao/Trunk/billing/billing_fees.pm b/lib/NGCP/BulkProcessor/Dao/Trunk/billing/billing_fees.pm index 972e3ea..d63aa63 100644 --- a/lib/NGCP/BulkProcessor/Dao/Trunk/billing/billing_fees.pm +++ b/lib/NGCP/BulkProcessor/Dao/Trunk/billing/billing_fees.pm @@ -55,8 +55,14 @@ my $expected_fieldnames = [ 'offpeak_init_interval', 'offpeak_follow_rate', 'offpeak_follow_interval', - 'use_free_time', + 'onpeak_use_free_time', 'match_mode', + 'onpeak_extra_rate', + 'onpeak_extra_second', + 'offpeak_extra_rate', + 'offpeak_extra_second', + 'offpeak_use_free_time', + 'aoc_pulse_amount_per_message', ]; my $indexes = {}; diff --git a/lib/NGCP/BulkProcessor/Dao/Trunk/billing/billing_fees_history.pm b/lib/NGCP/BulkProcessor/Dao/Trunk/billing/billing_fees_history.pm new file mode 100644 index 0000000..12c9106 --- /dev/null +++ b/lib/NGCP/BulkProcessor/Dao/Trunk/billing/billing_fees_history.pm @@ -0,0 +1,131 @@ +package NGCP::BulkProcessor::Dao::Trunk::billing::billing_fees_history; +use strict; + +## no critic + +use NGCP::BulkProcessor::Logging qw( + getlogger +); + +use NGCP::BulkProcessor::ConnectorPool qw( + get_billing_db + destroy_dbs +); + +use NGCP::BulkProcessor::SqlProcessor qw( + checktableinfo + copy_row +); +use NGCP::BulkProcessor::SqlRecord qw(); + +require Exporter; +our @ISA = qw(Exporter NGCP::BulkProcessor::SqlRecord); +our @EXPORT_OK = qw( + gettablename + check_table + + findby_id +); + +my $tablename = 'billing_fees_history'; +my $get_db = \&get_billing_db; + +my $expected_fieldnames = [ + 'id', + 'bf_id', + 'billing_profile_id', + 'billing_zones_history_id', + 'source', + 'destination', + 'direction', + 'type', + 'onpeak_init_rate', + 'onpeak_init_interval', + 'onpeak_follow_rate', + 'onpeak_follow_interval', + 'offpeak_init_rate', + 'offpeak_init_interval', + 'offpeak_follow_rate', + 'offpeak_follow_interval', + 'onpeak_use_free_time', + 'match_mode', + 'onpeak_extra_rate', + 'onpeak_extra_second', + 'offpeak_extra_rate', + 'offpeak_extra_second', + 'offpeak_use_free_time', + 'aoc_pulse_amount_per_message', +]; + +my $indexes = {}; + +my $insert_unique_fields = []; + +#enum('regex_longest_pattern','regex_longest_match','prefix','exact_destination') + +sub new { + + my $class = shift; + my $self = NGCP::BulkProcessor::SqlRecord->new($class,$get_db, + $tablename,$expected_fieldnames,$indexes); + + copy_row($self,shift,$expected_fieldnames); + + return $self; + +} + +sub findby_id { + + my ($id,$load_recursive) = @_; + + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + + my $stmt = 'SELECT * FROM ' . $table . ' WHERE ' . + $db->columnidentifier('id') . ' = ?'; + my @params = ($id); + my $rows = $db->db_get_all_arrayref($stmt,@params); + + return buildrecords_fromrows($rows,$load_recursive)->[0]; + +} + +sub buildrecords_fromrows { + + my ($rows,$load_recursive) = @_; + + my @records = (); + my $record; + + if (defined $rows and ref $rows eq 'ARRAY') { + foreach my $row (@$rows) { + $record = __PACKAGE__->new($row); + + # transformations go here ... + + push @records,$record; + } + } + + return \@records; + +} + +sub gettablename { + + return $tablename; + +} + +sub check_table { + + return checktableinfo($get_db, + __PACKAGE__,$tablename, + $expected_fieldnames, + $indexes); + +} + +1; diff --git a/lib/NGCP/BulkProcessor/Dao/Trunk/billing/billing_zones.pm b/lib/NGCP/BulkProcessor/Dao/Trunk/billing/billing_zones.pm index d71120c..3ca31e2 100644 --- a/lib/NGCP/BulkProcessor/Dao/Trunk/billing/billing_zones.pm +++ b/lib/NGCP/BulkProcessor/Dao/Trunk/billing/billing_zones.pm @@ -22,9 +22,6 @@ use NGCP::BulkProcessor::SqlProcessor qw( ); use NGCP::BulkProcessor::SqlRecord qw(); -#use NGCP::BulkProcessor::Dao::Trunk::billing::billing_mappings qw(); -#use NGCP::BulkProcessor::Dao::Trunk::billing::billing_profiles qw(); - require Exporter; our @ISA = qw(Exporter NGCP::BulkProcessor::SqlRecord); our @EXPORT_OK = qw( @@ -33,7 +30,6 @@ our @EXPORT_OK = qw( insert_row process_records - ); my $tablename = 'billing_zones'; @@ -62,7 +58,6 @@ sub new { } - sub insert_row { my $db = &$get_db(); diff --git a/lib/NGCP/BulkProcessor/Dao/Trunk/billing/billing_zones_history.pm b/lib/NGCP/BulkProcessor/Dao/Trunk/billing/billing_zones_history.pm new file mode 100644 index 0000000..a82b4da --- /dev/null +++ b/lib/NGCP/BulkProcessor/Dao/Trunk/billing/billing_zones_history.pm @@ -0,0 +1,110 @@ +package NGCP::BulkProcessor::Dao::Trunk::billing::billing_zones_history; +use strict; + +## no critic + +use NGCP::BulkProcessor::Logging qw( + getlogger +); + +use NGCP::BulkProcessor::ConnectorPool qw( + get_billing_db + destroy_dbs +); + +use NGCP::BulkProcessor::SqlProcessor qw( + checktableinfo + copy_row +); +use NGCP::BulkProcessor::SqlRecord qw(); + +require Exporter; +our @ISA = qw(Exporter NGCP::BulkProcessor::SqlRecord); +our @EXPORT_OK = qw( + gettablename + check_table + + findby_id +); + +my $tablename = 'billing_zones_history'; +my $get_db = \&get_billing_db; + +my $expected_fieldnames = [ + 'id', + 'bz_id', + 'billing_profile_id', + 'zone', + 'detail', +]; + +my $indexes = {}; + +my $insert_unique_fields = []; + +sub new { + + my $class = shift; + my $self = NGCP::BulkProcessor::SqlRecord->new($class,$get_db, + $tablename,$expected_fieldnames,$indexes); + + copy_row($self,shift,$expected_fieldnames); + + return $self; + +} + +sub findby_id { + + my ($id,$load_recursive) = @_; + + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + + my $stmt = 'SELECT * FROM ' . $table . ' WHERE ' . + $db->columnidentifier('id') . ' = ?'; + my @params = ($id); + my $rows = $db->db_get_all_arrayref($stmt,@params); + + return buildrecords_fromrows($rows,$load_recursive)->[0]; + +} + +sub buildrecords_fromrows { + + my ($rows,$load_recursive) = @_; + + my @records = (); + my $record; + + if (defined $rows and ref $rows eq 'ARRAY') { + foreach my $row (@$rows) { + $record = __PACKAGE__->new($row); + + # transformations go here ... + + push @records,$record; + } + } + + return \@records; + +} + +sub gettablename { + + return $tablename; + +} + +sub check_table { + + return checktableinfo($get_db, + __PACKAGE__,$tablename, + $expected_fieldnames, + $indexes); + +} + +1; diff --git a/lib/NGCP/BulkProcessor/Dao/Trunk/billing/contract_balances.pm b/lib/NGCP/BulkProcessor/Dao/Trunk/billing/contract_balances.pm index 2fa57fb..d255785 100644 --- a/lib/NGCP/BulkProcessor/Dao/Trunk/billing/contract_balances.pm +++ b/lib/NGCP/BulkProcessor/Dao/Trunk/billing/contract_balances.pm @@ -38,6 +38,7 @@ our @EXPORT_OK = qw( sort_by_end_asc get_new_balance_values get_free_ratio + findby_id ); my $tablename = 'contract_balances'; @@ -77,6 +78,25 @@ sub new { } +sub findby_id { + + my ($xa_db,$id,$load_recursive) = @_; + + check_table(); + my $db = &$get_db(); + $xa_db //= $db; + my $table = $db->tableidentifier($tablename); + + my $stmt = 'SELECT * FROM ' . $table . ' WHERE ' . + $db->columnidentifier('id') . ' = ?'; + my @params = ($id); + + my $rows = $xa_db->db_get_all_arrayref($stmt,@params); + + return buildrecords_fromrows($rows,$load_recursive); + +} + sub findby_contractid { my ($xa_db,$contract_id,$load_recursive) = @_; diff --git a/lib/NGCP/BulkProcessor/Dao/Trunk/billing/contracts.pm b/lib/NGCP/BulkProcessor/Dao/Trunk/billing/contracts.pm index 0b7f152..01582d5 100644 --- a/lib/NGCP/BulkProcessor/Dao/Trunk/billing/contracts.pm +++ b/lib/NGCP/BulkProcessor/Dao/Trunk/billing/contracts.pm @@ -343,6 +343,9 @@ sub buildrecords_fromrows { # transformations go here ... $record->load_relation($load_recursive,'voip_subscribers','NGCP::BulkProcessor::Dao::Trunk::billing::voip_subscribers::findby_contractid',$record->{id},$load_recursive); $record->load_relation($load_recursive,'contact','NGCP::BulkProcessor::Dao::Trunk::billing::contacts::findby_id',$record->{contact_id},$load_recursive); + $record->load_relation($load_recursive,'product','NGCP::BulkProcessor::Dao::Trunk::billing::products::findby_id_cached',$record->{product_id},$load_recursive); + $record->load_relation($load_recursive,'voip_peer_groups','NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_peer_groups::findby_peeringcontractid',$record->{id},$load_recursive); + $record->load_relation($load_recursive,'resellers','NGCP::BulkProcessor::Dao::Trunk::billing::resellers::findby_contractid',$record->{id},$load_recursive); push @records,$record; } diff --git a/lib/NGCP/BulkProcessor/Dao/Trunk/billing/products.pm b/lib/NGCP/BulkProcessor/Dao/Trunk/billing/products.pm index cf780fb..891c927 100644 --- a/lib/NGCP/BulkProcessor/Dao/Trunk/billing/products.pm +++ b/lib/NGCP/BulkProcessor/Dao/Trunk/billing/products.pm @@ -13,12 +13,17 @@ use NGCP::BulkProcessor::SqlProcessor qw( ); use NGCP::BulkProcessor::SqlRecord qw(); +use NGCP::BulkProcessor::Array qw(array_to_map); + require Exporter; our @ISA = qw(Exporter NGCP::BulkProcessor::SqlRecord); our @EXPORT_OK = qw( gettablename check_table + findby_id + findall + findby_id_cached findby_resellerid_handle $PSTN_PEERING_ACCOUNT_HANDLE @@ -63,6 +68,51 @@ sub new { } +sub findall { + + my ($load_recursive) = @_; + + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + + my $stmt = 'SELECT * FROM ' . $table; + my @params = (); + my $rows = $db->db_get_all_arrayref($stmt,@params); + + return buildrecords_fromrows($rows,$load_recursive); + +} + +my $product_map; + +sub findby_id_cached { + my ($id,$load_recursive) = @_; + unless ($product_map) { + ($product_map, my $types, my $ids) = array_to_map(findall($load_recursive), + sub { return shift->{id}; }, sub { return shift; }, 'last'); + } + return __PACKAGE__->new($product_map->{$id}) if defined $id; + return; +} + +sub findby_id { + + my ($id,$load_recursive) = @_; + + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + + my $stmt = 'SELECT * FROM ' . $table . ' WHERE ' . + $db->columnidentifier('id') . ' = ?'; + my @params = ($id); + my $rows = $db->db_get_all_arrayref($stmt,@params); + + return buildrecords_fromrows($rows,$load_recursive)->[0]; + +} + sub findby_resellerid_handle { my ($reseller_id,$handle,$load_recursive) = @_; diff --git a/lib/NGCP/BulkProcessor/Dao/Trunk/billing/resellers.pm b/lib/NGCP/BulkProcessor/Dao/Trunk/billing/resellers.pm index 895ccd2..a653422 100644 --- a/lib/NGCP/BulkProcessor/Dao/Trunk/billing/resellers.pm +++ b/lib/NGCP/BulkProcessor/Dao/Trunk/billing/resellers.pm @@ -28,6 +28,7 @@ our @EXPORT_OK = qw( findby_name findby_name_states findby_id + findby_contractid findall insert_row @@ -80,6 +81,23 @@ sub findall { } +sub findby_contractid { + + my ($contract_id,$load_recursive) = @_; + + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + + my $stmt = 'SELECT * FROM ' . $table . ' WHERE ' . + $db->columnidentifier('contract_id') . ' = ?'; + my @params = ($contract_id); + my $rows = $db->db_get_all_arrayref($stmt,@params); + + return buildrecords_fromrows($rows,$load_recursive); + +} + sub findby_name { my ($name,$load_recursive) = @_; diff --git a/lib/NGCP/BulkProcessor/Dao/Trunk/billing/voip_subscribers.pm b/lib/NGCP/BulkProcessor/Dao/Trunk/billing/voip_subscribers.pm index d359b5b..cac0d07 100644 --- a/lib/NGCP/BulkProcessor/Dao/Trunk/billing/voip_subscribers.pm +++ b/lib/NGCP/BulkProcessor/Dao/Trunk/billing/voip_subscribers.pm @@ -36,6 +36,7 @@ our @EXPORT_OK = qw( delete_row findby_id + findby_uuid findby_contractid findby_domainid_username_states countby_status_resellerid @@ -101,6 +102,23 @@ sub findby_id { } +sub findby_uuid { + + my ($uuid,$load_recursive) = @_; + + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + + my $stmt = 'SELECT * FROM ' . $table . ' WHERE ' . + $db->columnidentifier('uuid') . ' = ?'; + my @params = ($uuid); + my $rows = $db->db_get_all_arrayref($stmt,@params); + + return buildrecords_fromrows($rows,$load_recursive)->[0]; + +} + sub findby_domainid_usernames { my ($xa_db,$domain_id,$usernames,$load_recursive) = @_; diff --git a/lib/NGCP/BulkProcessor/Dao/Trunk/provisioning/voip_peer_groups.pm b/lib/NGCP/BulkProcessor/Dao/Trunk/provisioning/voip_peer_groups.pm index c3fb8f6..2c913eb 100644 --- a/lib/NGCP/BulkProcessor/Dao/Trunk/provisioning/voip_peer_groups.pm +++ b/lib/NGCP/BulkProcessor/Dao/Trunk/provisioning/voip_peer_groups.pm @@ -26,6 +26,7 @@ our @EXPORT_OK = qw( gettablename check_table + findby_peeringcontractid insert_row findby_name @@ -60,7 +61,22 @@ sub new { } +sub findby_peeringcontractid { + my ($contract_id,$load_recursive) = @_; + + check_table(); + my $db = &$get_db(); + my $table = $db->tableidentifier($tablename); + + my $stmt = 'SELECT * FROM ' . $table . ' WHERE ' . + $db->columnidentifier('peering_contract_id') . ' = ?'; + my @params = ($contract_id); + my $rows = $db->db_get_all_arrayref($stmt,@params); + + return buildrecords_fromrows($rows,$load_recursive); + +} sub insert_row { diff --git a/lib/NGCP/BulkProcessor/FileProcessor.pm b/lib/NGCP/BulkProcessor/FileProcessor.pm index c0c22c2..3b8cd90 100644 --- a/lib/NGCP/BulkProcessor/FileProcessor.pm +++ b/lib/NGCP/BulkProcessor/FileProcessor.pm @@ -13,6 +13,7 @@ use File::Basename qw(basename); use NGCP::BulkProcessor::Globals qw( $enablemultithreading $cpucount + get_threadqueuelength ); use NGCP::BulkProcessor::Logging qw( getlogger @@ -21,6 +22,7 @@ use NGCP::BulkProcessor::Logging qw( fileprocessingdone lines_read processing_lines + enable_threading_info ); use NGCP::BulkProcessor::LogError qw( @@ -42,6 +44,7 @@ my $thread_sleep_secs = 0.1; my $RUNNING = 1; my $COMPLETED = 2; my $ERROR = 4; +my $STOP = 8; sub new { @@ -155,6 +158,23 @@ sub process { $processors{$processor->tid()} = $processor; } + my $signal_handler = sub { + my $tid = threadid(); + $errorstate = $STOP; + enable_threading_info(1); + filethreadingdebug("[$tid] interrupt signal received",getlogger(__PACKAGE__)); + #print("[$tid] interrupt signal received"); + #_info($context,"interrupt signal received"); + #$result = 0; + my $errorstates = \%errorstates; + lock $errorstates; + $errorstates->{$tid} = $STOP; + }; + local $SIG{TERM} = $signal_handler; + local $SIG{INT} = $signal_handler; + local $SIG{QUIT} = $signal_handler; + local $SIG{HUP} = $signal_handler; + $reader->join(); filethreadingdebug('reader thread joined',getlogger(__PACKAGE__)); while ((scalar keys %processors) > 0) { @@ -168,7 +188,8 @@ sub process { sleep($thread_sleep_secs); } - $errorstate = (_get_other_threads_state(\%errorstates,$tid) & ~$RUNNING); + $errorstate = $COMPLETED if $errorstate == $RUNNING; + $errorstate |= (_get_other_threads_state(\%errorstates,$tid) & ~$RUNNING); } else { @@ -382,7 +403,7 @@ sub _reader { my $i = 0; my $state = $RUNNING; #start at first - while (($state & $RUNNING) == $RUNNING and ($state & $ERROR) == 0) { #as long there is one running consumer and no defunct consumer + while (($state & $RUNNING) == $RUNNING and ($state & $ERROR) == 0 and ($state & $STOP) == 0) { #as long there is one running consumer and no defunct consumer #fetching_lines($context->{filename},$i,$context->{instance}->{blocksize},undef,getlogger(__PACKAGE__)); my $block_n = 0; my @lines = (); @@ -429,7 +450,7 @@ sub _reader { $context->{queue}->enqueue(\%packet); #$packet); $blockcount++; #wait if thequeue is full and there there is one running consumer - while (((($state = _get_other_threads_state($context->{errorstates},$tid)) & $RUNNING) == $RUNNING) and $context->{queue}->pending() >= $context->{instance}->{threadqueuelength}) { + while (((($state = _get_other_threads_state($context->{errorstates},$tid)) & $RUNNING) == $RUNNING) and $context->{queue}->pending() >= get_threadqueuelength($context->{instance}->{threadqueuelength})) { #yield(); sleep($thread_sleep_secs); } @@ -445,10 +466,11 @@ sub _reader { } } } - if (not (($state & $RUNNING) == $RUNNING and ($state & $ERROR) == 0)) { + if (not (($state & $RUNNING) == $RUNNING and ($state & $ERROR) == 0 and ($state & $STOP) == 0)) { filethreadingdebug('[' . $tid . '] reader thread is shutting down (' . - (($state & $RUNNING) == $RUNNING ? 'still running consumer threads' : 'no running consumer threads') . ', ' . - (($state & $ERROR) == 0 ? 'no defunct thread(s)' : 'defunct thread(s)') . ') ...' + (($state & $RUNNING) == $RUNNING ? 'still running consumer threads' : uc('no running consumer threads')) . ', ' . + (($state & $ERROR) == 0 ? 'no defunct thread(s)' : uc('defunct thread(s)')) . ', ' . + (($state & $STOP) == 0 ? 'no thread(s) stopping by signal' : uc('thread(s) stopping by signal')) . ') ...' ,getlogger(__PACKAGE__)); } close(INPUTFILE_READER); @@ -463,7 +485,7 @@ sub _reader { if ($@) { $context->{errorstates}->{$tid} = $ERROR; fileprocessingfailed($filename,getlogger(__PACKAGE__)); - } else { + } elsif ($context->{errorstates}->{$tid} != $STOP) { $context->{errorstates}->{$tid} = $COMPLETED; } return $context->{errorstates}->{$tid}; @@ -527,7 +549,7 @@ sub _process { lock $context->{errorstates}; if ($err) { $context->{errorstates}->{$tid} = $ERROR; - } else { + } elsif ($context->{errorstates}->{$tid} != $STOP) { $context->{errorstates}->{$tid} = $COMPLETED; #(not $rowblock_result) ? $ERROR : $COMPLETED; } return $context->{errorstates}->{$tid}; @@ -563,16 +585,17 @@ sub _get_stop_consumer_thread { $reader_state = $errorstates->{$context->{readertid}}; } $queuesize = $context->{queue}->pending(); - if (($other_threads_state & $ERROR) == 0 and ($queuesize > 0 or $reader_state == $RUNNING)) { + if (($other_threads_state & $ERROR) == 0 and ($other_threads_state & $STOP) == 0 and ($queuesize > 0 or $reader_state == $RUNNING)) { $result = 0; #keep the consumer thread running if there is no defunct thread and queue is not empty or reader is still running } if ($result) { filethreadingdebug('[' . $tid . '] consumer thread is shutting down (' . - (($other_threads_state & $ERROR) == 0 ? 'no defunct thread(s)' : 'defunct thread(s)') . ', ' . - ($queuesize > 0 ? 'blocks pending' : 'no blocks pending') . ', ' . - ($reader_state == $RUNNING ? 'reader thread running' : 'reader thread not running') . ') ...' + (($other_threads_state & $ERROR) == 0 ? 'no defunct thread(s)' : uc('defunct thread(s)')) . ', ' . + (($other_threads_state & $STOP) == 0 ? 'no thread(s) stopping by signal' : uc('thread(s) stopping by signal')) . ', ' . + ($queuesize > 0 ? 'blocks pending' : uc('no blocks pending')) . ', ' . + ($reader_state == $RUNNING ? 'reader thread running' : uc('reader thread not running')) . ') ...' ,getlogger(__PACKAGE__)); } diff --git a/lib/NGCP/BulkProcessor/FileProcessors/CSVFileSimple.pm b/lib/NGCP/BulkProcessor/FileProcessors/CSVFileSimple.pm index 83e7131..71ad11d 100644 --- a/lib/NGCP/BulkProcessor/FileProcessors/CSVFileSimple.pm +++ b/lib/NGCP/BulkProcessor/FileProcessors/CSVFileSimple.pm @@ -3,6 +3,10 @@ use strict; ## no critic +use NGCP::BulkProcessor::Globals qw( + $cpucount +); + use NGCP::BulkProcessor::Logging qw( getlogger ); diff --git a/lib/NGCP/BulkProcessor/Globals.pm b/lib/NGCP/BulkProcessor/Globals.pm index 5248fd7..0fdeddc 100644 --- a/lib/NGCP/BulkProcessor/Globals.pm +++ b/lib/NGCP/BulkProcessor/Globals.pm @@ -47,6 +47,7 @@ our @EXPORT_OK = qw( $enablemultithreading $root_threadid $cpucount + get_threadqueuelength $cells_transfer_memory_limit $LongReadLen_limit @@ -130,6 +131,10 @@ our @EXPORT_OK = qw( $jobnamespace @config_search_paths + + $provisioning_conf_data + $constants_yml_data + $config_yml_data ); #set process umask for open and mkdir calls: @@ -164,6 +169,14 @@ if ($is_perl_debug) { our $cpucount = get_cpucount(); +sub get_threadqueuelength { + my $length = shift; + if ($length < 2 * $cpucount) { + $length = 2 * $cpucount; + } + return $length; +} + our $root_threadid = 0; #threadid() . ''; #0 our $cells_transfer_memory_limit = 10000000; #db fields our $transfer_defer_indexes = 1; @@ -222,8 +235,11 @@ our $working_path = tempdir(CLEANUP => 0) . '/'; #'/var/sipwise/'; our $provisioning_conf = undef; +our $provisioning_conf_data = undef; our $constants_yml = undef; - +our $constants_yml_data = undef; +our $config_yml = undef; +our $config_yml_data = undef; # csv our $csv_path = $working_path . 'csv/'; @@ -368,6 +384,14 @@ sub update_masterconfig { $screenloglevel = $data->{screenloglevel} if exists $data->{screenloglevel}; $emailloglevel = $data->{emailloglevel} if exists $data->{emailloglevel}; + eval { + if ('debug' eq lc($fileloglevel) + or 'debug' eq lc($screenloglevel) + or 'debug' eq lc($emailloglevel)) { + $NGCP::BulkProcessor::SqlConnector::log_db_operations = 1; + } + }; + if (exists $data->{working_path}) { $result &= _prepare_working_paths($data->{working_path},1,$fileerrorcode,$configlogger); } else { @@ -386,6 +410,16 @@ sub update_masterconfig { { force_plugins => [ 'Config::Any::XML' ] } ]); } + + $config_yml = $data->{config_yml} if exists $data->{config_yml}; + + if (defined $config_yml and length($config_yml) > 0) { + push(@loadconfig_args,[ + $config_yml, + \&_update_config_yml, + $yamlconfigtype, + ]); + } $constants_yml = $data->{constants_yml} if exists $data->{constants_yml}; @@ -408,6 +442,8 @@ sub _update_provisioning_conf { my ($data,$configfile) = @_; + $provisioning_conf_data = $data; + if (defined $data) { my $result = 1; @@ -425,6 +461,8 @@ sub _update_constants_yml { my ($data,$configfile) = @_; + $constants_yml_data = $data; + if (defined $data) { my $result = 1; @@ -477,6 +515,23 @@ sub _update_constants_yml { } +sub _update_config_yml { + + my ($data,$configfile) = @_; + + $config_yml_data = $data; + + if (defined $data) { + + my $result = 1; + + return $result; + + } + return 0; + +} + sub _postprocess_masterconfig { my %params = @_; diff --git a/lib/NGCP/BulkProcessor/LoadConfig.pm b/lib/NGCP/BulkProcessor/LoadConfig.pm index 31d5676..26163b6 100644 --- a/lib/NGCP/BulkProcessor/LoadConfig.pm +++ b/lib/NGCP/BulkProcessor/LoadConfig.pm @@ -3,6 +3,8 @@ use strict; ## no critic +use Cwd 'abs_path'; + use NGCP::BulkProcessor::Globals qw( $system_name $system_instance_label @@ -31,6 +33,7 @@ use NGCP::BulkProcessor::LogError qw( use YAML qw(); $YAML::UseCode = 1; + use Config::Any qw(); use NGCP::BulkProcessor::Utils qw(format_number trim); @@ -131,7 +134,9 @@ sub load_config { return $result; } else { my $result = &$process_code($data,$variant); - configurationinfo('config file ' . $variant . ' loaded',getlogger(__PACKAGE__)); + my $msg = 'config file ' . $variant . ' loaded'; + $msg .= ' (' . abs_path($variant) . ')' if $variant ne abs_path($variant); + configurationinfo($msg,getlogger(__PACKAGE__)); return $result; } @@ -177,7 +182,9 @@ sub _splashinfo { configurationinfo('application path: ' . $application_path,getlogger(__PACKAGE__)); configurationinfo('working path: ' . $working_path,getlogger(__PACKAGE__)); configurationinfo($cpucount . ' cpu(s), multithreading ' . ($enablemultithreading ? 'enabled' : 'disabled'),getlogger(__PACKAGE__)); - configurationinfo('master config file ' . $configfile . ' loaded',getlogger(__PACKAGE__)); + my $msg = 'master config file ' . $configfile . ' loaded'; + $msg .= ' (' . abs_path($configfile) . ')' if $configfile ne abs_path($configfile); + configurationinfo($msg,getlogger(__PACKAGE__)); configurationinfo('WARNING: running perl debug',getlogger(__PACKAGE__)) if $is_perl_debug; } @@ -302,9 +309,14 @@ sub _parse_yaml_config { my ($file,$configparser_args) = @_; - my $config = undef; + my $config; + unless (-e $file and -f _ and -r _) { + filewarn('parsing yaml format - cannot open file ' . $file,getlogger(__PACKAGE__)); + return $config; + } + eval { - $config = YAML::LoadFile($file); + $config = YAML::LoadFile($file) // {}; }; if ($@) { configurationerror($file,'parsing yaml format - error: ' . $@,getlogger(__PACKAGE__)); @@ -318,9 +330,15 @@ sub _parse_any_config { my ($file,$configparser_args) = @_; - my $config = undef; + my $config; + + unless (-e $file and -f _ and -r _) { + filewarn('parsing any format - cannot open file ' . $file,getlogger(__PACKAGE__)); + return $config; + } + eval { - $config = Config::Any->load_files( { files => [ $file ], (defined $configparser_args ? %$configparser_args : ()) } ); + $config = Config::Any->load_files( { files => [ $file ], (defined $configparser_args ? %$configparser_args : ()) } ) // {}; }; if ($@) { configurationerror($file,'parsing any format - error: ' . $@,getlogger(__PACKAGE__)); diff --git a/lib/NGCP/BulkProcessor/LogError.pm b/lib/NGCP/BulkProcessor/LogError.pm index 661cacf..1cb48ce 100644 --- a/lib/NGCP/BulkProcessor/LogError.pm +++ b/lib/NGCP/BulkProcessor/LogError.pm @@ -27,6 +27,7 @@ use NGCP::BulkProcessor::Utils qw( getscriptpath timestamp secs_to_years + is_in_eval ); use POSIX qw(ceil); # locale_h); @@ -202,7 +203,7 @@ sub terminate { my ($message,$logger) = @_; - if (threadid() == $root_threadid) { + if (threadid() == $root_threadid and not is_in_eval()) { my $appexitsecs = Time::HiRes::time(); #$message .= "\n\n" . sprintf("%.2f",$appexitsecs - $appstartsecs) . ' seconds'; @@ -477,11 +478,28 @@ sub processzerorowcount { } +sub _processing_prefix { + my $context = shift; + $context = { tid => $context, } unless ref $context; + my $name = ''; + $name = $context->{name} if exists $context->{name}; + if (length($name) > 0) { + if ($context->{tid} != $root_threadid) { + return '[' . $context->{tid} . ' ' . $name . '] '; + } else { + return '[' . $name . '] '; + } + } elsif ($context->{tid} != $root_threadid) { + return '[' . $context->{tid} . '] '; + } + return ''; +} + sub rowprocessingerror { - my ($tid, $message, $logger) = @_; + my ($context, $message, $logger) = @_; if (defined $logger) { - $logger->error(($enablemultithreading ? '[' . $tid . '] ' : '') . $message); + $logger->error(_processing_prefix($context) . $message); } terminate($message, $logger); @@ -489,9 +507,9 @@ sub rowprocessingerror { sub rowprocessingwarn { - my ($tid, $message, $logger) = @_; + my ($context, $message, $logger) = @_; if (defined $logger) { - $logger->warn(($enablemultithreading ? '[' . $tid . '] ' : '') . $message); + $logger->warn(_processing_prefix($context) . $message); } warning($message, $logger); @@ -770,7 +788,7 @@ sub _getsqlconnectorinstanceprefix { my $instancestring = $db->instanceidentifier(); if (length($instancestring) > 0) { if ($db->{tid} != $root_threadid) { - return '[' . $db->{tid} . '/' . $instancestring . '] '; + return '[' . $db->{tid} . ' ' . $instancestring . '] '; } else { return '[' . $instancestring . '] '; } @@ -822,7 +840,7 @@ sub _getrestconnectorinstanceprefix { my $instancestring = $restapi->instanceidentifier(); if (length($instancestring) > 0) { if ($restapi->{tid} != $root_threadid) { - return '[' . $restapi->{tid} . '/' . $instancestring . '] '; + return '[' . $restapi->{tid} . ' ' . $instancestring . '] '; } else { return '[' . $instancestring . '] '; } @@ -847,7 +865,7 @@ sub _getnosqlconnectorinstanceprefix { my $instancestring = $connector->instanceidentifier(); if (length($instancestring) > 0) { if ($connector->{tid} != $root_threadid) { - return '[' . $connector->{tid} . '/' . $instancestring . '] '; + return '[' . $connector->{tid} . ' ' . $instancestring . '] '; } else { return '[' . $instancestring . '] '; } diff --git a/lib/NGCP/BulkProcessor/Logging.pm b/lib/NGCP/BulkProcessor/Logging.pm index dd8d84e..bc97c67 100644 --- a/lib/NGCP/BulkProcessor/Logging.pm +++ b/lib/NGCP/BulkProcessor/Logging.pm @@ -3,6 +3,9 @@ use strict; ## no critic +use threads; # as early as possible... +use threads::shared; + use NGCP::BulkProcessor::Globals qw( $root_threadid $logfile_path @@ -14,6 +17,9 @@ use NGCP::BulkProcessor::Globals qw( use Log::Log4perl qw(get_logger); +*Log::Log4perl::Logger::notice = *Log::Log4perl::Logger::info; +*Log::Log4perl::Logger::warning = *Log::Log4perl::Logger::warn; + use File::Basename qw(basename); use NGCP::BulkProcessor::Utils qw(timestampdigits datestampdigits changemod chopstring trim humanize_bytes); @@ -57,6 +63,7 @@ our @EXPORT_OK = qw( rowinserted rowupserted + rowsupserted rowupdated rowsdeleted rowsupdated @@ -107,6 +114,8 @@ our @EXPORT_OK = qw( tablefixed servicedebug serviceinfo + + enable_threading_info ); #rowskipped @@ -154,17 +163,29 @@ sub init_log_default { sub init_log { - $currentlogfile = $logfile_path . timestampdigits() . $logfileextension; - createlogfile($currentlogfile); - $attachmentlogfile = $logfile_path . 'email_' . timestampdigits() . $logfileextension; - createlogfile($attachmentlogfile); - #$weblogfile = $logfile_path . 'web_' . datestampdigits() . $logfileextension; - #createlogfile($weblogfile); + my ($ts,$name,$daemon_logfile) = @_; + + $name //= ''; + $name = '_' . $name if $name; - # log configuration - my $conf = "log4perl.logger = DEBUG, FileApp, ScreenApp, MailAttApp\n" . + undef $currentlogfile; + undef $attachmentlogfile; - "log4perl.appender.FileApp = Log::Log4perl::Appender::File\n" . + if ($daemon_logfile) { + $currentlogfile = sprintf($daemon_logfile,$name); + } else { + $ts //= time; + $ts = timestampdigits($ts); + $currentlogfile = $logfile_path . $ts . $name . $logfileextension; + $attachmentlogfile = $logfile_path . 'email_' . $ts . $name . $logfileextension; + } + + # log configuration + my @loggers = ( 'DEBUG' ); + my $conf = ''; + if (length($fileloglevel) and 'off' ne lc($fileloglevel) and $currentlogfile) { + createlogfile($currentlogfile); + $conf .= "log4perl.appender.FileApp = Log::Log4perl::Appender::File\n" . "log4perl.appender.FileApp.umask = 0\n" . "log4perl.appender.FileApp.syswite = 1\n" . 'log4perl.appender.FileApp.Threshold = ' . $fileloglevel . "\n" . @@ -172,9 +193,12 @@ sub init_log { 'log4perl.appender.FileApp.filename = ' . $currentlogfile . "\n" . "log4perl.appender.FileApp.create_at_logtime = 1\n" . "log4perl.appender.FileApp.layout = PatternLayout\n" . - 'log4perl.appender.FileApp.layout.ConversionPattern = %d> %m%n' . "\n\n" . - - "log4perl.appender.MailAttApp = Log::Log4perl::Appender::File\n" . + 'log4perl.appender.FileApp.layout.ConversionPattern = %d> %m%n' . "\n\n"; + push(@loggers,'FileApp'); + } + if (length($emailloglevel) and 'off' ne lc($emailloglevel) and $attachmentlogfile) { + createlogfile($attachmentlogfile); + $conf .= "log4perl.appender.MailAttApp = Log::Log4perl::Appender::File\n" . "log4perl.appender.MailApp.umask = 0\n" . "log4perl.appender.MailApp.syswite = 1\n" . 'log4perl.appender.MailAttApp.Threshold = ' . $emailloglevel . "\n" . @@ -182,14 +206,21 @@ sub init_log { 'log4perl.appender.MailAttApp.filename = ' . $attachmentlogfile . "\n" . "log4perl.appender.MailAttApp.create_at_logtime = 1\n" . "log4perl.appender.MailAttApp.layout = Log::Log4perl::Layout::SimpleLayout\n" . - 'log4perl.appender.MailAttApp.layout.ConversionPattern = %d> %m%n' . "\n\n" . + 'log4perl.appender.MailAttApp.layout.ConversionPattern = %d> %m%n' . "\n\n"; + push(@loggers,'MailAttApp'); + } - "log4perl.appender.ScreenApp = Log::Log4perl::Appender::Screen\n" . + if (length($screenloglevel) and 'off' ne lc($screenloglevel)) { + $conf .= "log4perl.appender.ScreenApp = Log::Log4perl::Appender::Screen\n" . #"log4perl.appender.ScreenApp = Log::Log4perl::Appender::ScreenColoredLevels\n" . 'log4perl.appender.ScreenApp.Threshold = ' . $screenloglevel . "\n" . "log4perl.appender.ScreenApp.stderr = 0\n" . "log4perl.appender.ScreenApp.layout = Log::Log4perl::Layout::SimpleLayout\n" . 'log4perl.appender.ScreenApp.layout.ConversionPattern = %d> %m%n'; + push(@loggers,'ScreenApp'); + } + + $conf = "log4perl.logger = " . join(',',@loggers) . "\n" . $conf; # Initialize logging behaviour Log::Log4perl->init( \$conf ); @@ -223,7 +254,7 @@ sub getlogger { #if (defined $loglogger and defined $newlogger) { # $loglogger->debug('logger for category ' . $package . ' created'); #} - return get_logger($package); + return eval { get_logger($package) }; #_get_loglogger()->debug('logger for category ' . $package . ' created'); #return $newlogger; @@ -420,11 +451,24 @@ sub tableprocessingstarted { } +my $threading_info : shared = 0; + +sub enable_threading_info { + my $info = shift; + lock $threading_info; + $threading_info = 1; +} + sub tablethreadingdebug { my ($message,$logger) = @_; if (defined $logger) { - $logger->debug($message); + lock $threading_info; + if ($threading_info) { + $logger->info($message); + } else { + $logger->debug($message); + } } } @@ -528,6 +572,15 @@ sub rowupserted { } +sub rowsupserted { + + my ($db,$tablename,$logger) = @_; + if (defined $logger) { + $logger->debug(_getsqlconnectorinstanceprefix($db) . 'row(s) upserted'); + } + +} + sub rowupdated { my ($db,$tablename,$logger) = @_; @@ -644,13 +697,30 @@ sub writing_rows { sub processing_rows { - my ($tid, $start,$blocksize,$totalnumofrows,$logger) = @_; + my ($context, $start,$blocksize,$totalnumofrows,$logger) = @_; if (defined $logger) { - $logger->info(($enablemultithreading ? '[' . $tid . '] ' : '') . 'processing rows: ' . ($start + 1) . '-' . ($start + $blocksize) . ' of ' . $totalnumofrows); + $logger->info(_processing_prefix($context) . 'processing rows: ' . ($start + 1) . '-' . ($start + $blocksize) . ' of ' . $totalnumofrows); } } +sub _processing_prefix { + my $context = shift; + $context = { tid => $context, } unless ref $context; + my $name = ''; + $name = $context->{name} if exists $context->{name}; + if (length($name) > 0) { + if ($context->{tid} != $root_threadid) { + return '[' . $context->{tid} . ' ' . $name . '] '; + } else { + return '[' . $name . '] '; + } + } elsif ($context->{tid} != $root_threadid) { + return '[' . $context->{tid} . '] '; + } + return ''; +} + sub filethreadingdebug { @@ -710,14 +780,14 @@ sub lines_read { sub processing_lines { - my ($tid, $start,$blocksize,$block_n,$logger) = @_; + my ($context, $start,$blocksize,$block_n,$logger) = @_; if (defined $logger) { if (defined $block_n) { if ($block_n > 0) { - $logger->info(($enablemultithreading ? '[' . $tid . '] ' : '') . 'processing lines: ' . humanize_bytes($block_n)); + $logger->info(_processing_prefix($context) . 'processing lines: ' . humanize_bytes($block_n)); } } else { - $logger->info(($enablemultithreading ? '[' . $tid . '] ' : '') . 'processing lines: ' . ($start + 1) . '-' . ($start + $blocksize)); + $logger->info(_processing_prefix($context) . 'processing lines: ' . ($start + 1) . '-' . ($start + $blocksize)); } } @@ -725,18 +795,18 @@ sub processing_lines { sub processing_info { - my ($tid, $message, $logger) = @_; + my ($context, $message, $logger) = @_; if (defined $logger) { - $logger->info(($enablemultithreading ? '[' . $tid . '] ' : '') . $message); + $logger->info(_processing_prefix($context) . $message); } } sub processing_debug { - my ($tid, $message, $logger) = @_; + my ($context, $message, $logger) = @_; if (defined $logger) { - $logger->debug(($enablemultithreading ? '[' . $tid . '] ' : '') . $message); + $logger->debug(_processing_prefix($context) . $message); } } @@ -762,9 +832,9 @@ sub fetching_items { sub processing_items { - my ($tid, $start,$blocksize,$logger) = @_; + my ($context, $start,$blocksize,$logger) = @_; if (defined $logger) { - $logger->info(($enablemultithreading ? '[' . $tid . '] ' : '') . 'processing items: ' . ($start + 1) . '-' . ($start + $blocksize)); + $logger->info(_processing_prefix($context) . 'processing items: ' . ($start + 1) . '-' . ($start + $blocksize)); } } @@ -808,12 +878,12 @@ sub fetching_entries { sub processing_entries { - my ($tid, $start, $blocksize, $logger) = @_; + my ($context, $start, $blocksize, $logger) = @_; if (defined $logger) { if ($blocksize) { - $logger->info(($enablemultithreading ? '[' . $tid . '] ' : '') . 'processing entries: ' . ($start + 1) . '-' . ($start + $blocksize)); + $logger->info(_processing_prefix($context) . 'processing entries: ' . ($start + 1) . '-' . ($start + $blocksize)); } else { - $logger->info(($enablemultithreading ? '[' . $tid . '] ' : '') . 'processing entries: (none)'); + $logger->info(_processing_prefix($context) . 'processing entries: (none)'); } } @@ -927,7 +997,7 @@ sub _getsqlconnectorinstanceprefix { my $instancestring = $db->instanceidentifier(); if (length($instancestring) > 0) { if ($db->{tid} != $root_threadid) { - return '[' . $db->{tid} . '/' . $instancestring . '] '; + return '[' . $db->{tid} . ' ' . $instancestring . '] '; } else { return '[' . $instancestring . '] '; } @@ -955,7 +1025,7 @@ sub _getnosqlconnectorinstanceprefix { my $instancestring = $connector->instanceidentifier(); if (length($instancestring) > 0) { if ($connector->{tid} != $root_threadid) { - return '[' . $connector->{tid} . '/' . $instancestring . '] '; + return '[' . $connector->{tid} . ' ' . $instancestring . '] '; } else { return '[' . $instancestring . '] '; } @@ -979,7 +1049,7 @@ sub _getrestconnectorinstanceprefix { my $instancestring = $restapi->instanceidentifier(); if (length($instancestring) > 0) { if ($restapi->{tid} != $root_threadid) { - return '[' . $restapi->{tid} . '/' . $instancestring . '] '; + return '[' . $restapi->{tid} . ' ' . $instancestring . '] '; } else { return '[' . $instancestring . '] '; } diff --git a/lib/NGCP/BulkProcessor/NoSqlConnector.pm b/lib/NGCP/BulkProcessor/NoSqlConnector.pm index b5b6fdf..4c35b44 100644 --- a/lib/NGCP/BulkProcessor/NoSqlConnector.pm +++ b/lib/NGCP/BulkProcessor/NoSqlConnector.pm @@ -109,9 +109,9 @@ sub DESTROY { if ($self->{tid} == threadid()) { $self->disconnect(); - eval { + #eval { nosqldebug($self,(ref $self) . ' connector destroyed',getlogger(__PACKAGE__)); - }; + #}; } } diff --git a/lib/NGCP/BulkProcessor/NoSqlConnectors/RedisEntry.pm b/lib/NGCP/BulkProcessor/NoSqlConnectors/RedisEntry.pm index 192c77e..2d582d7 100644 --- a/lib/NGCP/BulkProcessor/NoSqlConnectors/RedisEntry.pm +++ b/lib/NGCP/BulkProcessor/NoSqlConnectors/RedisEntry.pm @@ -7,6 +7,8 @@ use Tie::IxHash; use NGCP::BulkProcessor::Utils qw(load_module); +use NGCP::BulkProcessor::Closure qw(is_code); + require Exporter; our @ISA = qw(Exporter); our @EXPORT_OK = qw( @@ -225,24 +227,25 @@ sub load_relation { $transform = $include->{transform}; if (exists $include->{include}) { $include = $include->{include}; + } elsif (exists $include->{load}) { + $include = $include->{load}; } elsif ($transform or $filter) { $include = 1; } } - if (('CODE' eq ref $include and $include->($self)) + if ((is_code($include) and $self->_calc_closure($relation_path,'load',$include,$load_recursive->{_context},$load_recursive->{_cache},$self)) or (not ref $include and $include)) { load_module($findby); no strict "refs"; ## no critic (ProhibitNoStrict) $load_recursive->{_relation_path} = $relation_path; $self->{$relation} = $findby->(@findby_args); if ('ARRAY' eq ref $self->{$relation} - and 'CODE' eq ref $filter) { - my $closure = _closure($filter,$load_recursive->{_context}); - $self->{$relation} = [ grep { $closure->($_); } @{$self->{$relation}}]; + and is_code($filter)) { + my $cache = $load_recursive->{_cache} // {}; + $self->{$relation} = [ grep { $self->_calc_closure($relation_path,'filter',$filter,$load_recursive->{_context},$cache,$_,$self); } @{$self->{$relation}} ]; } - if ('CODE' eq ref $transform) { - my $closure = _closure($transform,$load_recursive->{_context}); - $self->{$relation} = $closure->($self->{$relation}); + if (is_code($transform)) { + $self->{$relation} = $self->_calc_closure($relation_path,'transform',$transform,$load_recursive->{_context},$load_recursive->{_cache},$self->{$relation},$self); } $load_recursive->{_relation_path} = $relation_path_backup; return 1; @@ -251,15 +254,15 @@ sub load_relation { return 0; } -sub _closure { - my ($sub,$context) = @_; - return sub { - foreach my $key (keys %$context) { - no strict "refs"; ## no critic (ProhibitNoStrict) - *{"main::$key"} = $context->{$key} if 'CODE' eq ref $context->{$key}; - } - return $sub->(@_,$context); - }; +sub _calc_closure { + + my $self = shift; + my ($relation_path,$func,$code,$context,$cache,@args) = @_; + my $id = '_relations_' . $func . '_' . $relation_path; + $cache //= {}; + $cache->{$id} = NGCP::BulkProcessor::Closure->new($code,$context,"relations '$relation_path' $func'") unless exists $cache->{$id}; + return $cache->{$id}->calc($context,@args); + } 1; diff --git a/lib/NGCP/BulkProcessor/NoSqlConnectors/RedisProcessor.pm b/lib/NGCP/BulkProcessor/NoSqlConnectors/RedisProcessor.pm index 043effc..45bc683 100644 --- a/lib/NGCP/BulkProcessor/NoSqlConnectors/RedisProcessor.pm +++ b/lib/NGCP/BulkProcessor/NoSqlConnectors/RedisProcessor.pm @@ -15,6 +15,7 @@ use Time::HiRes qw(sleep); use NGCP::BulkProcessor::Globals qw( $enablemultithreading $cpucount + get_threadqueuelength ); use NGCP::BulkProcessor::Logging qw( getlogger @@ -23,6 +24,7 @@ use NGCP::BulkProcessor::Logging qw( nosqlprocessingdone fetching_entries processing_entries + enable_threading_info ); use NGCP::BulkProcessor::LogError qw( @@ -50,6 +52,7 @@ my $thread_sleep_secs = 0.1; my $RUNNING = 1; my $COMPLETED = 2; my $ERROR = 4; +my $STOP = 8; sub process_entries { @@ -132,6 +135,23 @@ sub process_entries { $processors{$processor->tid()} = $processor; } + my $signal_handler = sub { + my $tid = threadid(); + $errorstate = $STOP; + enable_threading_info(1); + nosqlthreadingdebug("[$tid] interrupt signal received",getlogger(__PACKAGE__)); + #print("[$tid] interrupt signal received"); + #_info($context,"interrupt signal received"); + #$result = 0; + my $errorstates = \%errorstates; + lock $errorstates; + $errorstates->{$tid} = $STOP; + }; + local $SIG{TERM} = $signal_handler; + local $SIG{INT} = $signal_handler; + local $SIG{QUIT} = $signal_handler; + local $SIG{HUP} = $signal_handler; + $reader->join(); nosqlthreadingdebug('reader thread joined',getlogger(__PACKAGE__)); while ((scalar keys %processors) > 0) { @@ -145,7 +165,8 @@ sub process_entries { sleep($thread_sleep_secs); } - $errorstate = (_get_other_threads_state(\%errorstates,$tid) & ~$RUNNING); + $errorstate = $COMPLETED if $errorstate == $RUNNING; + $errorstate |= (_get_other_threads_state(\%errorstates,$tid) & ~$RUNNING); nosqlthreadingdebug('restoring connections ...',getlogger(__PACKAGE__)); @@ -238,7 +259,7 @@ sub _reader { my $i = 0; my $cursor = 0; my $state = $RUNNING; #start at first - while (($state & $RUNNING) == $RUNNING and ($state & $ERROR) == 0) { #as long there is one running consumer and no defunct consumer + while (($state & $RUNNING) == $RUNNING and ($state & $ERROR) == 0 and ($state & $STOP) == 0) { #as long there is one running consumer and no defunct consumer fetching_entries($store,$context->{scan_pattern},$i,$blocksize,getlogger(__PACKAGE__)); ($cursor, my $rowblock) = $store->scan_shared($cursor,get_scan_args($context->{scan_pattern},$blocksize,$context->{type})); my $realblocksize = scalar @$rowblock; @@ -249,7 +270,7 @@ sub _reader { $context->{queue}->enqueue(\%packet); #$packet); $blockcount++; #wait if thequeue is full and there there is one running consumer - while (((($state = _get_other_threads_state($context->{errorstates},$tid)) & $RUNNING) == $RUNNING) and $context->{queue}->pending() >= $context->{threadqueuelength}) { + while (((($state = _get_other_threads_state($context->{errorstates},$tid)) & $RUNNING) == $RUNNING) and $context->{queue}->pending() >= get_threadqueuelength($context->{threadqueuelength})) { #yield(); sleep($thread_sleep_secs); } @@ -259,10 +280,11 @@ sub _reader { last; } } - if (not (($state & $RUNNING) == $RUNNING and ($state & $ERROR) == 0)) { + if (not (($state & $RUNNING) == $RUNNING and ($state & $ERROR) == 0 and ($state & $ERROR) == 0)) { nosqlthreadingdebug('[' . $tid . '] reader thread is shutting down (' . - (($state & $RUNNING) == $RUNNING ? 'still running consumer threads' : 'no running consumer threads') . ', ' . - (($state & $ERROR) == 0 ? 'no defunct thread(s)' : 'defunct thread(s)') . ') ...' + (($state & $RUNNING) == $RUNNING ? 'still running consumer threads' : uc('no running consumer threads')) . ', ' . + (($state & $ERROR) == 0 ? 'no defunct thread(s)' : uc('defunct thread(s)')) . ', ' . + (($state & $STOP) == 0 ? 'no thread(s) stopping by signal' : uc('thread(s) stopping by signal')) . ') ...' ,getlogger(__PACKAGE__)); } }; @@ -287,7 +309,7 @@ sub _reader { lock $context->{errorstates}; if ($@) { $context->{errorstates}->{$tid} = $ERROR; - } else { + } elsif ($context->{errorstates}->{$tid} != $STOP) { $context->{errorstates}->{$tid} = $COMPLETED; } return $context->{errorstates}->{$tid}; @@ -348,7 +370,7 @@ sub _process { lock $context->{errorstates}; if ($err) { $context->{errorstates}->{$tid} = $ERROR; - } else { + } elsif ($context->{errorstates}->{$tid} != $STOP) { $context->{errorstates}->{$tid} = $COMPLETED; #(not $rowblock_result) ? $ERROR : $COMPLETED; } return $context->{errorstates}->{$tid}; @@ -384,16 +406,17 @@ sub _get_stop_consumer_thread { $reader_state = $errorstates->{$context->{readertid}}; } $queuesize = $context->{queue}->pending(); - if (($other_threads_state & $ERROR) == 0 and ($queuesize > 0 or $reader_state == $RUNNING)) { + if (($other_threads_state & $ERROR) == 0 and ($other_threads_state & $STOP) == 0 and ($queuesize > 0 or $reader_state == $RUNNING)) { $result = 0; #keep the consumer thread running if there is no defunct thread and queue is not empty or reader is still running } if ($result) { nosqlthreadingdebug('[' . $tid . '] consumer thread is shutting down (' . - (($other_threads_state & $ERROR) == 0 ? 'no defunct thread(s)' : 'defunct thread(s)') . ', ' . - ($queuesize > 0 ? 'blocks pending' : 'no blocks pending') . ', ' . - ($reader_state == $RUNNING ? 'reader thread running' : 'reader thread not running') . ') ...' + (($other_threads_state & $ERROR) == 0 ? 'no defunct thread(s)' : uc('defunct thread(s)')) . ', ' . + (($other_threads_state & $STOP) == 0 ? 'no thread(s) stopping by signal' : uc('thread(s) stopping by signal')) . ', ' . + ($queuesize > 0 ? 'blocks pending' : uc('no blocks pending')) . ', ' . + ($reader_state == $RUNNING ? 'reader thread running' : uc('reader thread not running')) . ') ...' ,getlogger(__PACKAGE__)); } diff --git a/lib/NGCP/BulkProcessor/Projects/ETL/CDR/Dao/Tabular.pm b/lib/NGCP/BulkProcessor/Projects/ETL/CDR/Dao/Tabular.pm deleted file mode 100644 index ada8853..0000000 --- a/lib/NGCP/BulkProcessor/Projects/ETL/CDR/Dao/Tabular.pm +++ /dev/null @@ -1,279 +0,0 @@ -package NGCP::BulkProcessor::Projects::ETL::CDR::Dao::Tabular; -use strict; - -## no critic - -use NGCP::BulkProcessor::Projects::ETL::CDR::ProjectConnectorPool qw( - get_sqlite_db - destroy_all_dbs -); - -use NGCP::BulkProcessor::Projects::ETL::CDR::Settings qw( - $tabular_fields - $csv_all_expected_fields -); - -use NGCP::BulkProcessor::SqlProcessor qw( - registertableinfo - create_targettable - checktableinfo - copy_row - insert_stmt - transfer_table -); - -use NGCP::BulkProcessor::SqlRecord qw(); - -require Exporter; -our @ISA = qw(Exporter NGCP::BulkProcessor::SqlRecord); -our @EXPORT_OK = qw( - create_table - gettablename - check_table - getinsertstatement - getupsertstatement - - get_fieldnames - - update_delta - findby_delta - countby_delta - - $deleted_delta - $updated_delta - $added_delta - - copy_table -); - -my $tablename = 'tabular'; -my $get_db = \&get_sqlite_db; - -my $fieldnames; -my $expected_fieldnames; -sub get_fieldnames { - my $expected = shift; - unless (defined $fieldnames and defined $expected_fieldnames) { - $fieldnames = [ map { - local $_ = (ref $_ ? (exists $_->{colname} ? $_->{colname} : $_->{path}) : $_); - $_ =~ s/\./_/g; - $_ =~ s/\[(\d+)\]/_$1/g; - $_; - } @$tabular_fields ]; - $expected_fieldnames = [ @$fieldnames ]; - push(@$expected_fieldnames,'id') unless grep { 'id' eq $_; } @$expected_fieldnames; - push(@$expected_fieldnames,'delta'); - } - return $fieldnames unless $expected; - return $expected_fieldnames; -} - -my $primarykey_fieldnames = [ 'id' ]; -my $indexes = { - $tablename . '_delta' => [ 'delta(7)' ], -}; - -our $deleted_delta = 'DELETED'; -our $updated_delta = 'UPDATED'; -our $added_delta = 'ADDED'; - -sub new { - - my $class = shift; - my $self = NGCP::BulkProcessor::SqlRecord->new($class,$get_db, - $tablename,get_fieldnames(1),$indexes); - - copy_row($self,shift,get_fieldnames(1)); - - return $self; - -} - -sub create_table { - - my ($truncate) = @_; - - my $db = &$get_db(); - - registertableinfo($db,__PACKAGE__,$tablename,get_fieldnames(1),$indexes,$primarykey_fieldnames); - return create_targettable($db,__PACKAGE__,$db,__PACKAGE__,$tablename,$truncate,0,undef); - -} - -sub findby_delta { - - my ($delta,$load_recursive) = @_; - - check_table(); - my $db = &$get_db(); - my $table = $db->tableidentifier($tablename); - - return [] unless defined $delta; - - my $rows = $db->db_get_all_arrayref( - 'SELECT * FROM ' . - $table . - ' WHERE ' . - $db->columnidentifier('delta') . ' = ?' - , $delta); - - return buildrecords_fromrows($rows,$load_recursive); - -} - -sub findby_domainusername { - - my ($domain,$username,$load_recursive) = @_; - - check_table(); - my $db = &$get_db(); - my $table = $db->tableidentifier($tablename); - - return [] unless (defined $domain and defined $username); - - my $rows = $db->db_get_all_arrayref( - 'SELECT * FROM ' . $table . - ' WHERE ' . $db->columnidentifier('domain') . ' = ?' . - ' AND ' . $db->columnidentifier('username') . ' = ?' - , $domain, $username); - - return buildrecords_fromrows($rows,$load_recursive)->[0]; - -} - -sub update_delta { - - my ($id,$delta) = @_; - - check_table(); - my $db = &$get_db(); - my $table = $db->tableidentifier($tablename); - - my $stmt = 'UPDATE ' . $table . ' SET delta = ?'; - my @params = (); - push(@params,$delta); - if (defined $id) { - $stmt .= ' WHERE ' . - $db->columnidentifier('id') . ' = ?'; - push(@params, $id); - } - - return $db->db_do($stmt,@params); - -} - -sub countby_delta { - - my ($deltas) = @_; - - check_table(); - my $db = &$get_db(); - my $table = $db->tableidentifier($tablename); - - my $stmt = 'SELECT COUNT(*) FROM ' . $table . ' WHERE 1=1'; - my @params = (); - if (defined $deltas and 'HASH' eq ref $deltas) { - foreach my $in (keys %$deltas) { - my @values = (defined $deltas->{$in} and 'ARRAY' eq ref $deltas->{$in} ? @{$deltas->{$in}} : ($deltas->{$in})); - $stmt .= ' AND ' . $db->columnidentifier('delta') . ' ' . $in . ' (' . substr(',?' x scalar @values,1) . ')'; - push(@params,@values); - } - } elsif (defined $deltas and length($deltas) > 0) { - $stmt .= ' AND ' . $db->columnidentifier('delta') . ' = ?'; - push(@params,$deltas); - } - - return $db->db_get_value($stmt,@params); - -} - -sub copy_table { - - my ($get_target_db) = @_; - - if ($csv_all_expected_fields) { - check_table(); - } else { - checktableinfo($get_db, - __PACKAGE__,$tablename, - get_fieldnames(0), - $indexes); - } - - return transfer_table( - get_db => $get_db, - class => __PACKAGE__, - get_target_db => $get_target_db, - targetclass => __PACKAGE__, - targettablename => $tablename, - ); - -} - -sub buildrecords_fromrows { - - my ($rows,$load_recursive) = @_; - - my @records = (); - my $record; - - if (defined $rows and ref $rows eq 'ARRAY') { - foreach my $row (@$rows) { - $record = __PACKAGE__->new($row); - - # transformations go here ... - - push @records,$record; - } - } - - return \@records; - -} - -sub getinsertstatement { - - my ($insert_ignore) = @_; - check_table(); - return insert_stmt($get_db,__PACKAGE__,$insert_ignore); - -} - -sub getupsertstatement { - - check_table(); - my $db = &$get_db(); - my $table = $db->tableidentifier($tablename); - my $upsert_stmt = 'INSERT OR REPLACE INTO ' . $table . ' (' . - join(', ', map { local $_ = $_; $_ = $db->columnidentifier($_); $_; } @{get_fieldnames(1)}) . ')'; - my @values = (); - foreach my $fieldname (@{get_fieldnames(1)}) { - if ('delta' eq $fieldname) { - my $stmt = 'SELECT \'' . $updated_delta . '\' FROM ' . $table . ' WHERE ' . - $db->columnidentifier('id') . ' = ?'; - push(@values,'COALESCE((' . $stmt . '), \'' . $added_delta . '\')'); - } else { - push(@values,'?'); - } - } - $upsert_stmt .= ' VALUES (' . join(',',@values) . ')'; - return $upsert_stmt; - -} - -sub gettablename { - - return $tablename; - -} - -sub check_table { - - return checktableinfo($get_db, - __PACKAGE__,$tablename, - get_fieldnames(1), - $indexes); - -} - -1; diff --git a/lib/NGCP/BulkProcessor/Projects/ETL/CDR/ExportCDR.pm b/lib/NGCP/BulkProcessor/Projects/ETL/CDR/ExportCDR.pm deleted file mode 100644 index 6dc6fa1..0000000 --- a/lib/NGCP/BulkProcessor/Projects/ETL/CDR/ExportCDR.pm +++ /dev/null @@ -1,490 +0,0 @@ -package NGCP::BulkProcessor::Projects::ETL::CDR::ExportCDR; -use strict; - -## no critic - -use threads::shared qw(); - -use Tie::IxHash; - -use NGCP::BulkProcessor::Serialization qw(); -use Scalar::Util qw(blessed); -use MIME::Base64 qw(encode_base64); - -use NGCP::BulkProcessor::Projects::ETL::CDR::Settings qw( - $dry - $skip_errors - - $export_cdr_multithreading - $export_cdr_numofthreads - $export_cdr_blocksize - - run_dao_method - get_dao_var - get_export_filename - - write_export_file - $cdr_export_filename_format - - $tabular_fields - $load_recursive - $tabular_single_row_txn - $ignore_tabular_unique - $graph_fields - $graph_fields_mode -); - -use NGCP::BulkProcessor::Logging qw ( - getlogger - processing_info - processing_debug -); -use NGCP::BulkProcessor::LogError qw( - rowprocessingerror - rowprocessingwarn - fileerror -); - -use NGCP::BulkProcessor::Dao::Trunk::billing::contracts qw(); - -use NGCP::BulkProcessor::Projects::ETL::CDR::Dao::Tabular qw(); - -use NGCP::BulkProcessor::Projects::ETL::CDR::ProjectConnectorPool qw( - get_sqlite_db - destroy_all_dbs - ping_all_dbs -); - -use NGCP::BulkProcessor::Utils qw(create_uuid threadid timestamp stringtobool trim); #check_ipnet -#use NGCP::BulkProcessor::DSSorter qw(sort_by_configs); -#use NGCP::BulkProcessor::Table qw(get_rowhash); -use NGCP::BulkProcessor::Array qw(array_to_map); -use NGCP::BulkProcessor::DSPath qw(); - -require Exporter; -our @ISA = qw(Exporter); -our @EXPORT_OK = qw( - export_cdr_graph - export_cdr_tabular -); - -sub _init_graph_field_map { - my $context = shift; - my %graph_field_map = (); - my %graph_field_globs = (); - tie(%graph_field_globs, 'Tie::IxHash'); - foreach my $graph_field (@$graph_fields) { - my ($c,$a); - if ('HASH' eq ref $graph_field) { - $a = $graph_field->{path}; - $c = $graph_field; - } else { - $a = $graph_field; - $c = 1; - } - #my @a = (); - #my $b = ''; - #foreach my $c (split(/\./,$a)) { - # - # foreach my () { - # $b .= $_ - # push() - # } - #} - if ($a =~ /\*/) { - $a = quotemeta($a); - $a =~ s/(\\\*)+/[^.]+/g; - $a = '^' . $a . '$'; - $graph_field_globs{$a} = $c unless exists $graph_field_globs{$a}; - } else { - $graph_field_map{$a} = $c unless exists $graph_field_map{$a}; - } - } - $context->{graph_field_map} = \%graph_field_map; - $context->{graph_field_globs} = \%graph_field_globs; -} - -sub export_cdr_graph { - - my $static_context = { - - }; - _init_graph_field_map($static_context); - ($static_context->{export_filename},$static_context->{export_format}) = get_export_filename($cdr_export_filename_format); - - my $result = 1; #_copy_cdr_checks($static_context); - - destroy_all_dbs(); - my $warning_count :shared = 0; - return ($result && run_dao_method('accounting::cdr::process_fromto', - #source_dbs => $static_context->{source_dbs}, - static_context => $static_context, - process_code => sub { - my ($context,$records,$row_offset) = @_; - ping_all_dbs(); - my @data = (); - foreach my $record (@$records) { - next unless _export_cdr_graph_init_context($context,$record); - push(@data,_get_contract_graph($context)); - } - write_export_file(\@data,$context->{export_filename},$context->{export_format}); - return 1; - }, - init_process_context_code => sub { - my ($context)= @_; - $context->{error_count} = 0; - $context->{warning_count} = 0; - }, - uninit_process_context_code => sub { - my ($context)= @_; - destroy_all_dbs(); - { - lock $warning_count; - $warning_count += $context->{warning_count}; - } - }, - destroy_reader_dbs_code => \&destroy_all_dbs, - blocksize => $export_cdr_blocksize, - multithreading => $export_cdr_multithreading, - numofthreads => $export_cdr_numofthreads, - ),$warning_count,); - -} - -sub export_cdr_tabular { - - my $result = NGCP::BulkProcessor::Projects::ETL::CDR::Dao::Tabular::create_table(0); - - my $static_context = { - upsert => _tabular_rows_reset_delta(), - }; - - destroy_all_dbs(); - my $warning_count :shared = 0; - return ($result && run_dao_method('billing::contracts::process_records', - static_context => $static_context, - process_code => sub { - my ($context,$records,$row_offset) = @_; - ping_all_dbs(); - my @subscriber_rows = (); - foreach my $record (@$records) { - next unless _export_cdr_tabular_init_context($context,$record); - push(@subscriber_rows, _get_subscriber_rows($context)); - - if ($tabular_single_row_txn and (scalar @subscriber_rows) > 0) { - while (defined (my $subscriber_row = shift @subscriber_rows)) { - if ($skip_errors) { - eval { _insert_tabular_rows($context,[$subscriber_row]); }; - _warn($context,$@) if $@; - } else { - _insert_tabular_rows($context,[$subscriber_row]); - } - } - } - } - - if (not $tabular_single_row_txn and (scalar @subscriber_rows) > 0) { - if ($skip_errors) { - eval { insert_tabular_rows($context,\@subscriber_rows); }; - _warn($context,$@) if $@; - } else { - insert_tabular_rows($context,\@subscriber_rows); - } - } - - return 1; - }, - init_process_context_code => sub { - my ($context)= @_; - $context->{db} = &get_sqlite_db(); - $context->{error_count} = 0; - $context->{warning_count} = 0; - }, - uninit_process_context_code => sub { - my ($context)= @_; - undef $context->{db}; - destroy_all_dbs(); - { - lock $warning_count; - $warning_count += $context->{warning_count}; - } - }, - destroy_reader_dbs_code => \&destroy_all_dbs, - blocksize => $export_cdr_blocksize, - multithreading => $export_cdr_multithreading, - numofthreads => $export_cdr_numofthreads, - ),$warning_count,); - -} - -sub _tabular_rows_reset_delta { - my $upsert = 0; - if (NGCP::BulkProcessor::Projects::ETL::CDR::Dao::Tabular::countby_delta() > 0) { - processing_info(threadid(),'resetting delta of ' . - NGCP::BulkProcessor::Projects::ETL::CDR::Dao::Tabular::update_delta(undef, - $NGCP::BulkProcessor::Projects::ETL::CDR::Dao::Tabular::deleted_delta) . - ' records',getlogger(__PACKAGE__)); - $upsert |= 1; - } - return $upsert; -} - -sub _insert_tabular_rows { - my ($context,$subscriber_rows) = @_; - $context->{db}->db_do_begin( - ($context->{upsert} ? - NGCP::BulkProcessor::Projects::ETL::CDR::Dao::Tabular::getupsertstatement() - : NGCP::BulkProcessor::Projects::ETL::CDR::Dao::Tabular::getinsertstatement($ignore_tabular_unique)), - ); - eval { - $context->{db}->db_do_rowblock($subscriber_rows); - $context->{db}->db_finish(); - }; - my $err = $@; - if ($err) { - eval { - $context->{db}->db_finish(1); - }; - die($err); - } - -} - -sub _export_cdr_graph_init_context { - - my ($context,$record) = @_; - - my $result = 1; - - return 0 unless _load_contract($context,$record); - - return $result; - -} - -sub _get_contract_graph { - my ($context) = @_; - - my $dp = NGCP::BulkProcessor::DSPath->new($context->{contract}, { - filter => sub { - my $path = shift; - if ('whitelist' eq $graph_fields_mode) { - my $include; - if (exists $context->{graph_field_map}->{$path}) { - $include = $context->{graph_field_map}->{$path}; - } else { - foreach my $glob (keys %{$context->{graph_field_globs}}) { - if ($path =~ /$glob/) { - $include = $context->{graph_field_globs}->{$glob}; - last; - } - } - } - if ('HASH' eq ref $include) { - if (exists $include->{include}) { - return $include->{include}; - } - return 1; - } else { - return $include; - } - } elsif ('blacklist' eq $graph_fields_mode) { - my $exclude; - if (exists $context->{graph_field_map}->{$path}) { - $exclude = $context->{graph_field_map}->{$path}; - } else { - foreach my $glob (keys %{$context->{graph_field_globs}}) { - if ($path =~ /$glob/) { - $exclude = $context->{graph_field_globs}->{$glob}; - last; - } - } - } - if ('HASH' eq ref $exclude) { - if (exists $exclude->{exclude}) { - return not $exclude->{exclude}; - } elsif ($exclude->{transform}) { - return 1; - } - return 0; - } else { - return not $exclude; - } - } - }, - transform => sub { - return shift; - # ($bill_subs->{provisioning_voip_subscriber}->{voip_usr_preferences}, my $as, my $vs) = - # array_to_map($bill_subs->{provisioning_voip_subscriber}->{voip_usr_preferences}, - # sub { return shift->{attribute}; }, sub { my $p = shift; }, 'group' ); - # if (my $prov_subscriber = $bill_subs->{provisioning_voip_subscriber}) { - # foreach my $voicemail_user (@{$prov_subscriber->{voicemail_users}}) { - # foreach my $voicemail (@{$voicemail_user->{voicemail_spool}}) { - # $voicemail->{recording} = encode_base64($voicemail->{recording},''); - # } - # } - # } - }, - }); - - $dp->filter()->transform(); - - return $context->{contract}; - -} - -sub _export_cdr_tabular_init_context { - - my ($context,$record) = @_; - - my $result = 1; - - return 0 unless _load_contract($context,$record); - - if (defined $context->{contract}->{voip_subscribers} - and not scalar @{$context->{contract}->{voip_subscribers}}) { - _info($context,"contract ID $record->{id} has no subscribers, skipping",1); - $result = 0; - } - - return $result; - -} - -sub _get_subscriber_rows { - - my ($context) = @_; - - my @rows = (); - foreach my $bill_subs (@{$context->{contract}->{voip_subscribers}}) { - my @row = (); - $bill_subs->{contract} = NGCP::BulkProcessor::Dao::Trunk::billing::contracts->new($context->{contract}); #no circular ref - ($bill_subs->{provisioning_voip_subscriber}->{voip_usr_preferences}, my $as, my $vs) = - array_to_map($bill_subs->{provisioning_voip_subscriber}->{voip_usr_preferences}, - sub { return shift->{_attribute}; }, sub { my $p = shift; }, 'group' ); - if (my $prov_subscriber = $bill_subs->{provisioning_voip_subscriber}) { - foreach my $voicemail_user (@{$prov_subscriber->{voicemail_users}}) { - foreach my $voicemail (@{$voicemail_user->{voicemail_spool}}) { - $voicemail->{recording} = encode_base64($voicemail->{recording},''); - } - } - } - my $dp = NGCP::BulkProcessor::DSPath->new($bill_subs, { - retrieve_key_from_non_hash => sub {}, - key_does_not_exist => sub {}, - index_does_not_exist => sub {}, - }); - foreach my $tabular_field (@$tabular_fields) { - my $a; - my $sep = ','; - my $transform; - if ('HASH' eq ref $tabular_field) { - $a = $tabular_field->{path}; - $sep = $tabular_field->{sep}; - $transform = $tabular_field->{transform}; - } else { - $a = $tabular_field; - } - #eval {'' . ($dp->get('.' . $a) // '');}; if($@){ - # my $x=5; - #} - my $v = $dp->get('.' . $a); - if ('CODE' eq ref $transform) { - my $closure = _closure($transform,_get_closure_context($context)); - $v = $closure->($v,$bill_subs); - } - if ('ARRAY' eq ref $v) { - if ('HASH' eq ref $v->[0] - or (blessed($v->[0]) and $v->[0]->isa('NGCP::BulkProcessor::SqlRecord'))) { - $v = join($sep, sort map { $_->{$tabular_field->{field}}; } @$v); - } else { - $v = join($sep, sort @$v); - } - } else { - $v = '' . ($v // ''); - } - push(@row,$v); - } - push(@row,$bill_subs->{uuid}) unless grep { 'uuid' eq $_; } @{NGCP::BulkProcessor::Projects::ETL::CDR::Dao::Tabular::get_fieldnames()}; - if ($context->{upsert}) { - push(@row,$bill_subs->{uuid}); - } else { - push(@row,$NGCP::BulkProcessor::Projects::ETL::CDR::Dao::Tabular::added_delta); - } - - push(@rows,\@row); - } - - return @rows; - -} - -sub _load_contract { - - my ($context,$record) = @_; - $context->{contract} = run_dao_method('billing::contracts::findby_id', $record->{id}, { %$load_recursive, - #'contracts.voip_subscribers.domain' => 1, - _context => _get_closure_context($context), - }); - - return 1 if $context->{contract}; - return 0; - -} - -sub _get_closure_context { - my $context = shift; - return { - _info => \&_info, - _error => \&_error, - _debug => \&_debug, - _warn => \&_warn, - context => $context, - }; -} - -sub _closure { - my ($sub,$context) = @_; - return sub { - foreach my $key (keys %$context) { - no strict "refs"; ## no critic (ProhibitNoStrict) - *{"main::$key"} = $context->{$key} if 'CODE' eq ref $context->{$key}; - } - return $sub->(@_,$context); - }; -} - -sub _error { - - my ($context,$message) = @_; - $context->{error_count} = $context->{error_count} + 1; - rowprocessingerror($context->{tid} // threadid(),$message,getlogger(__PACKAGE__)); - -} - -sub _warn { - - my ($context,$message) = @_; - $context->{warning_count} = $context->{warning_count} + 1; - rowprocessingwarn($context->{tid} // threadid(),$message,getlogger(__PACKAGE__)); - -} - -sub _info { - - my ($context,$message,$debug) = @_; - if ($debug) { - processing_debug($context->{tid} // threadid(),$message,getlogger(__PACKAGE__)); - } else { - processing_info($context->{tid} // threadid(),$message,getlogger(__PACKAGE__)); - } -} - -sub _debug { - - my ($context,$message,$debug) = @_; - processing_debug($context->{tid} // threadid(),$message,getlogger(__PACKAGE__)); - -} - -1; diff --git a/lib/NGCP/BulkProcessor/Projects/ETL/CDR/ProjectConnectorPool.pm b/lib/NGCP/BulkProcessor/Projects/ETL/CDR/ProjectConnectorPool.pm deleted file mode 100644 index 2986c3d..0000000 --- a/lib/NGCP/BulkProcessor/Projects/ETL/CDR/ProjectConnectorPool.pm +++ /dev/null @@ -1,120 +0,0 @@ -package NGCP::BulkProcessor::Projects::ETL::CDR::ProjectConnectorPool; -use strict; - -## no critic - -use File::Basename; -use Cwd; -use lib Cwd::abs_path(File::Basename::dirname(__FILE__) . '/../../../'); - -use NGCP::BulkProcessor::Projects::ETL::CDR::Settings qw( - $csv_dir - $sqlite_db_file -); - -use NGCP::BulkProcessor::ConnectorPool qw( - get_connectorinstancename -); - -use NGCP::BulkProcessor::SqlConnectors::CSVDB qw(); -use NGCP::BulkProcessor::SqlConnectors::SQLiteDB qw($staticdbfilemode); - -use NGCP::BulkProcessor::SqlProcessor qw(cleartableinfo); - -require Exporter; -our @ISA = qw(Exporter); -our @EXPORT_OK = qw( - - get_sqlite_db - sqlite_db_tableidentifier - - get_csv_db - csv_db_tableidentifier - - destroy_dbs - destroy_all_dbs - ping_all_dbs - -); - -my $sqlite_dbs = {}; -my $csv_dbs = {}; - -sub get_sqlite_db { - - my ($instance_name,$reconnect) = @_; - my $name = get_connectorinstancename($instance_name); - - if (not defined $sqlite_dbs->{$name}) { - $sqlite_dbs->{$name} = NGCP::BulkProcessor::SqlConnectors::SQLiteDB->new($instance_name); - if (not defined $reconnect) { - $reconnect = 1; - } - } - if ($reconnect) { - $sqlite_dbs->{$name}->db_connect($staticdbfilemode,$sqlite_db_file); - } - - return $sqlite_dbs->{$name}; - -} - -sub sqlite_db_tableidentifier { - - my ($get_target_db,$tablename) = @_; - my $target_db = (ref $get_target_db eq 'CODE') ? &$get_target_db() : $get_target_db; - return $target_db->getsafetablename(NGCP::BulkProcessor::SqlConnectors::SQLiteDB::get_tableidentifier($tablename,$staticdbfilemode,$sqlite_db_file)); - -} - -sub get_csv_db { - - my ($instance_name,$reconnect) = @_; - my $name = get_connectorinstancename($instance_name); - if (not defined $csv_dbs->{$name}) { - $csv_dbs->{$name} = NGCP::BulkProcessor::SqlConnectors::CSVDB->new($instance_name); - if (not defined $reconnect) { - $reconnect = 1; - } - } - if ($reconnect) { - $csv_dbs->{$name}->db_connect($csv_dir); - } - return $csv_dbs->{$name}; - -} - -sub csv_db_tableidentifier { - - my ($get_target_db,$tablename) = @_; - my $target_db = (ref $get_target_db eq 'CODE') ? &$get_target_db() : $get_target_db; - return $target_db->getsafetablename(NGCP::BulkProcessor::SqlConnectors::CSVDB::get_tableidentifier($tablename,$csv_dir)); - -} - -sub destroy_dbs { - - foreach my $name (keys %$sqlite_dbs) { - cleartableinfo($sqlite_dbs->{$name}); - undef $sqlite_dbs->{$name}; - delete $sqlite_dbs->{$name}; - } - - foreach my $name (keys %$csv_dbs) { - cleartableinfo($csv_dbs->{$name}); - undef $csv_dbs->{$name}; - delete $csv_dbs->{$name}; - } - -} - -sub destroy_all_dbs() { - destroy_dbs(); - NGCP::BulkProcessor::ConnectorPool::destroy_dbs(); -} - -sub ping_all_dbs() { - NGCP::BulkProcessor::ConnectorPool::ping_dbs(); -} - -1; diff --git a/lib/NGCP/BulkProcessor/Projects/ETL/CDR/Settings.pm b/lib/NGCP/BulkProcessor/Projects/ETL/CDR/Settings.pm deleted file mode 100644 index 3c770d4..0000000 --- a/lib/NGCP/BulkProcessor/Projects/ETL/CDR/Settings.pm +++ /dev/null @@ -1,484 +0,0 @@ -package NGCP::BulkProcessor::Projects::ETL::CDR::Settings; -use strict; - -## no critic - -use threads::shared qw(); - -use File::Basename qw(fileparse); -use NGCP::BulkProcessor::Serialization qw(); -use DateTime::TimeZone qw(); - -use JSON -support_by_pp, -no_export; -*NGCP::BulkProcessor::Serialization::serialize_json = sub { - my $input_ref = shift; - return JSON::to_json($input_ref, { allow_nonref => 1, allow_blessed => 1, convert_blessed => 1, pretty => 1, as_nonblessed => 1 }); -}; - -use NGCP::BulkProcessor::Globals qw( - $working_path - $enablemultithreading - $cpucount - create_path -); - -use NGCP::BulkProcessor::Logging qw( - getlogger - scriptinfo - configurationinfo -); - -use NGCP::BulkProcessor::LogError qw( - fileerror - filewarn - configurationwarn - configurationerror -); - -use NGCP::BulkProcessor::LoadConfig qw( - split_tuple - parse_regexp -); - -use NGCP::BulkProcessor::Utils qw(prompt timestampdigits threadid load_module); - -use NGCP::BulkProcessor::Array qw(contains); - -require Exporter; -our @ISA = qw(Exporter); -our @EXPORT_OK = qw( - update_settings - run_dao_method - get_dao_var - get_export_filename - write_export_file - write_sql_file - - update_load_recursive - $load_yml - $load_recursive - - update_tabular_fields - $tabular_yml - $tabular_fields - $ignore_tabular_unique - $tabular_single_row_txn - $graph_yml - $graph_fields - $graph_fields_mode - update_graph_fields - - $sqlite_db_file - $csv_dir - - check_dry - - $output_path - $input_path - - $cdr_export_filename_format - $cdr_import_filename - - $defaultsettings - $defaultconfig - - $dry - $skip_errors - $force - - $export_cdr_multithreading - $export_cdr_numofthreads - $export_cdr_blocksize - - $csv_all_expected_fields -); -#$cf_default_priority -#$cf_default_timeout -#$cft_default_ringtimeout - -our $defaultconfig = 'config.cfg'; -our $defaultsettings = 'settings.cfg'; - -our $tabular_yml = 'tabular.yml'; -our $tabular_fields = []; -our $ignore_tabular_unique = 0; -our $tabular_single_row_txn = 1; - -our $graph_yml = 'graph.yml'; -our $graph_fields = []; -our $graph_fields_mode = 'whitelist'; -my @graph_fields_modes = qw(whitelist blacklist); - -our $load_yml = 'load.yml'; -our $load_recursive; - -our $output_path = $working_path . 'output/'; -our $input_path = $working_path . 'input/'; -our $csv_dir = 'cdr'; - -our $cdr_export_filename_format = undef; - -our $csv_all_expected_fields = 1; - -#our $cdr_import_filename = undef; -#our $cdr_import_numofthreads = $cpucount; -#our $cdr_import_multithreading = 1; -#our $cdr_reseller_name = 'default'; -#our $cdr_billing_profile_name = 'Default Billing Profile'; -#our $cdr_domain = undef; -#our $cdr_contact_email_format = '%s@example.org'; -#our $subscriber_contact_email_format = '%s@example.org'; -#our $split_cdrs = 0; - -#our $subscriber_timezone = undef; -#our $contract_timezone = undef; - -#our $subscriber_profile_set_name = undef; -#our $subscriber_profile_name = undef; -#our $webusername_format = '%1$s'; -#our $subscriber_externalid_format = undef; - -our $force = 0; -our $dry = 0; -our $skip_errors = 0; - -my $mr = 'Trunk'; -my @supported_mr = ('Trunk'); - -our $sqlite_db_file = 'sqlite'; - -our $export_cdr_multithreading = $enablemultithreading; -our $export_cdr_numofthreads = $cpucount; -our $export_cdr_blocksize = 1000; - -#our $cf_default_priority = 1; -#our $cf_default_timeout = 300; -#our $cft_default_ringtimeout = 20; - -#our $rollback_sql_export_filename_format = undef; -#our $rollback_sql_stmt_format = undef; - -my $file_lock :shared = undef; - -sub update_settings { - - my ($data,$configfile) = @_; - - if (defined $data) { - - my $result = 1; - my $regexp_result; - - #&$configurationinfocode("testinfomessage",$configlogger); - - $result &= _prepare_working_paths(1); - - $cdr_export_filename_format = $data->{cdr_export_filename} if exists $data->{cdr_export_filename}; - get_export_filename($data->{cdr_export_filename},$configfile); - - #$rollback_sql_export_filename_format = $data->{rollback_sql_export_filename_format} if exists $data->{rollback_sql_export_filename_format}; - #get_export_filename($data->{rollback_sql_export_filename_format},$configfile); - #$rollback_sql_stmt_format = $data->{rollback_sql_stmt_format} if exists $data->{rollback_sql_stmt_format}; - - $sqlite_db_file = $data->{sqlite_db_file} if exists $data->{sqlite_db_file}; - $csv_dir = $data->{csv_dir} if exists $data->{csv_dir}; - - #$cdr_import_filename = _get_import_filename($cdr_import_filename,$data,'cdr_import_filename'); - #$cdr_import_multithreading = $data->{cdr_import_multithreading} if exists $data->{cdr_import_multithreading}; - #$cdr_import_numofthreads = _get_numofthreads($cpucount,$data,'cdr_import_numofthreads'); - #$cdr_reseller_name = $data->{cdr_reseller_name} if exists $data->{cdr_reseller_name}; - #$cdr_billing_profile_name = $data->{cdr_billing_profile_name} if exists $data->{cdr_billing_profile_name}; - #$cdr_domain = $data->{cdr_domain} if exists $data->{cdr_domain}; - #$cdr_contact_email_format = $data->{cdr_contact_email_format} if exists $data->{cdr_contact_email_format}; - #$subscriber_contact_email_format = $data->{subscriber_contact_email_format} if exists $data->{subscriber_contact_email_format}; - #$split_cdrs = $data->{split_cdrs} if exists $data->{split_cdrs}; - - #$contract_timezone = $data->{cdr_timezone} if exists $data->{cdr_timezone}; - #if ($contract_timezone and not DateTime::TimeZone->is_valid_name($contract_timezone)) { - # configurationerror($configfile,"invalid cdr_timezone '$contract_timezone'"); - # $result = 0; - #} - - #$subscriber_timezone = $data->{subscriber_timezone} if exists $data->{subscriber_timezone}; - #if ($subscriber_timezone and not DateTime::TimeZone->is_valid_name($subscriber_timezone)) { - # configurationerror($configfile,"invalid subscriber_timezone '$subscriber_timezone'"); - # $result = 0; - #} - - #$subscriber_profile_set_name = $data->{subscriber_profile_set_name} if exists $data->{subscriber_profile_set_name}; - #$subscriber_profile_name = $data->{subscriber_profile_name} if exists $data->{subscriber_profile_name}; - #if ($subscriber_profile_set_name and not $subscriber_profile_name - # or not $subscriber_profile_set_name and $subscriber_profile_name) { - # configurationerror($configfile,"both subscriber_profile_set_name and subscriber_profile_name required"); - # $result = 0; - #} - #$webusername_format = $data->{webusername_format} if exists $data->{webusername_format}; - #$subscriber_externalid_format = $data->{subscriber_externalid_format} if exists $data->{subscriber_externalid_format}; - - $dry = $data->{dry} if exists $data->{dry}; - $skip_errors = $data->{skip_errors} if exists $data->{skip_errors}; - - $export_cdr_multithreading = $data->{export_cdr_multithreading} if exists $data->{export_cdr_multithreading}; - $export_cdr_numofthreads = _get_numofthreads($cpucount,$data,'export_cdr_numofthreads'); - $export_cdr_blocksize = $data->{export_cdr_blocksize} if exists $data->{export_cdr_blocksize}; - - $tabular_yml = $data->{tabular_yml} if exists $data->{tabular_yml}; - $graph_yml = $data->{graph_yml} if exists $data->{graph_yml}; - $graph_fields_mode = $data->{graph_fields_mode} if exists $data->{graph_fields_mode}; - if (not $graph_fields_mode or not contains($graph_fields_mode,\@graph_fields_modes)) { - configurationerror($configfile,'graph_fields_mode must be one of ' . join(', ', @graph_fields_modes)); - $result = 0; - } - $load_yml = $data->{load_yml} if exists $data->{load_yml}; - $tabular_single_row_txn = $data->{tabular_single_row_txn} if exists $data->{tabular_single_row_txn}; - $ignore_tabular_unique = $data->{ignore_tabular_unique} if exists $data->{ignore_tabular_unique}; - - #$cf_default_priority = $data->{cf_default_priority} if exists $data->{cf_default_priority}; - #$cf_default_timeout = $data->{cf_default_timeout} if exists $data->{cf_default_timeout}; - #$cft_default_ringtimeout = $data->{cft_default_ringtimeout} if exists $data->{cft_default_ringtimeout}; - - $csv_all_expected_fields = $data->{csv_all_expected_fields} if exists $data->{csv_all_expected_fields}; - - $mr = $data->{schema_version}; - if (not defined $mr or not contains($mr,\@supported_mr)) { - configurationerror($configfile,'schema_version must be one of ' . join(', ', @supported_mr)); - $result = 0; - } - - return $result; - } - return 0; - -} - -sub run_dao_method { - my $method_name = 'NGCP::BulkProcessor::Dao::' . $mr . '::' . shift; - load_module($method_name); - no strict 'refs'; - return $method_name->(@_); -} - -sub get_dao_var { - my $var_name = 'NGCP::BulkProcessor::Dao::' . $mr . '::' . shift; - load_module($var_name); - no strict 'refs'; - return @{$var_name} if wantarray; - return ${$var_name}; -} - -sub _prepare_working_paths { - - my ($create) = @_; - my $result = 1; - my $path_result; - - ($path_result,$input_path) = create_path($working_path . 'input',$input_path,$create,\&fileerror,getlogger(__PACKAGE__)); - $result &= $path_result; - ($path_result,$output_path) = create_path($working_path . 'output',$output_path,$create,\&fileerror,getlogger(__PACKAGE__)); - $result &= $path_result; - - return $result; - -} - -sub _get_numofthreads { - my ($default_value,$data,$key) = @_; - my $numofthreads = $default_value; - $numofthreads = $data->{$key} if exists $data->{$key}; - $numofthreads = $cpucount if $numofthreads > $cpucount; - return $numofthreads; -} - -sub get_export_filename { - my ($filename_format,$configfile) = @_; - my $export_filename; - my $export_format; - if ($filename_format) { - $export_filename = sprintf($filename_format,timestampdigits(),threadid()); - unless ($export_filename =~ /^\//) { - $export_filename = $output_path . $export_filename; - } - if (-e $export_filename and (unlink $export_filename) == 0) { - filewarn('cannot remove ' . $export_filename . ': ' . $!,getlogger(__PACKAGE__)); - $export_filename = undef; - } - my ($name,$path,$suffix) = fileparse($export_filename,".json",".yml",".yaml",".xml",".php",".pl",".db",".csv"); - if ($suffix eq '.json') { - $export_format = $NGCP::BulkProcessor::Serialization::format_json; - } elsif ($suffix eq '.yml' or $suffix eq '.yaml') { - $export_format = $NGCP::BulkProcessor::Serialization::format_yaml; - } elsif ($suffix eq '.xml') { - $export_format = $NGCP::BulkProcessor::Serialization::format_xml; - } elsif ($suffix eq '.php') { - $export_format = $NGCP::BulkProcessor::Serialization::format_php; - } elsif ($suffix eq '.pl') { - $export_format = $NGCP::BulkProcessor::Serialization::format_perl; - } elsif ($suffix eq '.db') { - $export_format = 'sqlite'; - } elsif ($suffix eq '.csv') { - $export_format = 'csv'; - } else { - configurationerror($configfile,"$filename_format: either .json/.yaml/.xml/.php/.pl or .db/.csv export file format required"); - } - } - return ($export_filename,$export_format); -} - -sub write_export_file { - - my ($data,$export_filename,$export_format) = @_; - if (defined $export_filename) { - fileerror("invalid extension for output filename $export_filename",getlogger(__PACKAGE__)) - unless contains($export_format,\@NGCP::BulkProcessor::Serialization::formats); - # "concatenated json" https://en.wikipedia.org/wiki/JSON_streaming - my $str = ''; - if (ref $data eq 'ARRAY') { - foreach my $obj (@$data) { - #$str .= "\n" if length($str); - $str .= NGCP::BulkProcessor::Serialization::serialize($obj,$export_format); - } - } else { - $str = NGCP::BulkProcessor::Serialization::serialize($data,$export_format); - } - _write_file($str,$export_filename); - } - -} - -sub write_sql_file { - - my ($data,$export_filename,$stmt_format) = @_; - if (defined $export_filename and $stmt_format) { - my $str = ''; - if (ref $data eq 'ARRAY') { - foreach my $obj (@$data) { - $str .= "\n" if length($str); - if (ref $obj eq 'ARRAY') { - $str .= sprintf($stmt_format,@$obj); - } else { - $str .= sprintf($stmt_format,$str); - } - } - } else { - $str = sprintf($stmt_format,$data); - } - $str .= "\n"; - _write_file($str,$export_filename); - } - -} - -sub _write_file { - - my ($str,$export_filename) = @_; - if (defined $export_filename) { - lock $file_lock; - open(my $fh, '>>', $export_filename) or fileerror('cannot open file ' . $export_filename . ': ' . $!,getlogger(__PACKAGE__)); - binmode($fh); - print $fh $str; - close $fh; - } - -} - -sub update_tabular_fields { - - my ($data,$configfile) = @_; - - if (defined $data) { - - my $result = 1; - - eval { - $tabular_fields = $data; - }; - if ($@ or 'ARRAY' ne ref $tabular_fields) { - $tabular_fields //= []; - configurationerror($configfile,'invalid tabular fields',getlogger(__PACKAGE__)); - $result = 0; - } - - return $result; - } - return 0; - -} - -sub update_graph_fields { - - my ($data,$configfile) = @_; - - if (defined $data) { - - my $result = 1; - - eval { - $graph_fields = $data; - }; - if ($@ or 'ARRAY' ne ref $graph_fields) { - $graph_fields //= []; - configurationerror($configfile,'invalid graph fields',getlogger(__PACKAGE__)); - $result = 0; - } - - return $result; - } - return 0; - -} - -sub update_load_recursive { - - my ($data,$configfile) = @_; - - if (defined $data) { - - my $result = 1; - - eval { - $load_recursive = $data; - }; - if ($@ or 'HASH' ne ref $load_recursive) { - undef $load_recursive; - configurationerror($configfile,'invalid load recursive def',getlogger(__PACKAGE__)); - $result = 0; - } - - return $result; - } - return 0; - -} - -sub _get_import_filename { - my ($old_value,$data,$key) = @_; - my $import_filename = $old_value; - $import_filename = $data->{$key} if exists $data->{$key}; - if (defined $import_filename and length($import_filename) > 0) { - $import_filename = $input_path . $import_filename unless -e $import_filename; - } - return $import_filename; -} - -sub check_dry { - - if ($dry) { - scriptinfo('running in dry mode - NGCP databases will not be modified',getlogger(__PACKAGE__)); - return 1; - } else { - scriptinfo('NO DRY MODE - NGCP DATABASES WILL BE MODIFIED!',getlogger(__PACKAGE__)); - if (!$force) { - if ('yes' eq lc(prompt("Type 'yes' to proceed: "))) { - return 1; - } else { - return 0; - } - } else { - scriptinfo('force option applied',getlogger(__PACKAGE__)); - return 1; - } - } - -} - -1; diff --git a/lib/NGCP/BulkProcessor/Projects/ETL/CDR/config.cfg b/lib/NGCP/BulkProcessor/Projects/ETL/CDR/config.cfg deleted file mode 100644 index 442b428..0000000 --- a/lib/NGCP/BulkProcessor/Projects/ETL/CDR/config.cfg +++ /dev/null @@ -1,61 +0,0 @@ -##general settings: -working_path = /var/sipwise -cpucount = 4 -enablemultithreading = 1 - -##gearman/service listener config: -jobservers = 127.0.0.1:4730 - -##NGCP MySQL connectivity - "accounting" db: -accounting_host = db01 -accounting_port = 3306 -accounting_databasename = accounting -accounting_username = root -accounting_password = - -##NGCP MySQL connectivity - "billing" db: -billing_host = db01 -billing_port = 3306 -billing_databasename = billing -billing_username = root -billing_password = - -##NGCP MySQL connectivity - "provisioning" db: -provisioning_host = db01 -provisioning_port = 3306 -provisioning_databasename = provisioning -provisioning_username = root -provisioning_password = - -##NGCP MySQL connectivity - "kamailio" db: -kamailio_host = db01 -kamailio_port = 3306 -kamailio_databasename = kamailio -kamailio_username = root -kamailio_password = - -##NGCP MySQL connectivity - default db for distributed transactions (XA) to connect to: -xa_host = db01 -xa_port = 3306 -xa_databasename = ngcp -xa_username = root -xa_password = - -##NGCP REST-API connectivity: -ngcprestapi_uri = https://127.0.0.1:1443 -ngcprestapi_username = administrator -ngcprestapi_password = administrator -ngcprestapi_realm = api_admin_http - -##sending email: -emailenable = 0 -erroremailrecipient = -warnemailrecipient = -completionemailrecipient = rkrenn@sipwise.com -doneemailrecipient = - -##logging: -fileloglevel = INFO -#DEBUG -screenloglevel = INFO -emailloglevel = OFF diff --git a/lib/NGCP/BulkProcessor/Projects/ETL/CDR/config.debug.cfg b/lib/NGCP/BulkProcessor/Projects/ETL/CDR/config.debug.cfg deleted file mode 100644 index 200aede..0000000 --- a/lib/NGCP/BulkProcessor/Projects/ETL/CDR/config.debug.cfg +++ /dev/null @@ -1,61 +0,0 @@ -##general settings: -working_path = /home/rkrenn/temp/customer_exporter -cpucount = 4 -enablemultithreading = 1 - -##gearman/service listener config: -jobservers = 127.0.0.1:4730 - -##NGCP MySQL connectivity - "accounting" db: -accounting_host = 192.168.0.178 -accounting_port = 3306 -accounting_databasename = accounting -accounting_username = root -accounting_password = - -##NGCP MySQL connectivity - "billing" db: -billing_host = 192.168.0.178 -billing_port = 3306 -billing_databasename = billing -billing_username = root -billing_password = - -##NGCP MySQL connectivity - "provisioning" db: -provisioning_host = 192.168.0.178 -provisioning_port = 3306 -provisioning_databasename = provisioning -provisioning_username = root -provisioning_password = - -##NGCP MySQL connectivity - "kamailio" db: -kamailio_host = 192.168.0.178 -kamailio_port = 3306 -kamailio_databasename = kamailio -kamailio_username = root -kamailio_password = - -##NGCP MySQL connectivity - default db for distributed transactions (XA) to connect to: -xa_host = 192.168.0.178 -xa_port = 3306 -xa_databasename = ngcp -xa_username = root -xa_password = - -##NGCP REST-API connectivity: -ngcprestapi_uri = https://127.0.0.1:1443 -ngcprestapi_username = administrator -ngcprestapi_password = administrator -ngcprestapi_realm = api_admin_http - -##sending email: -emailenable = 0 -erroremailrecipient = -warnemailrecipient = -completionemailrecipient = rkrenn@sipwise.com -doneemailrecipient = - -##logging: -fileloglevel = INFO -#DEBUG -screenloglevel = INFO -emailloglevel = OFF diff --git a/lib/NGCP/BulkProcessor/Projects/ETL/CDR/graph.yml b/lib/NGCP/BulkProcessor/Projects/ETL/CDR/graph.yml deleted file mode 100644 index 01f088a..0000000 --- a/lib/NGCP/BulkProcessor/Projects/ETL/CDR/graph.yml +++ /dev/null @@ -1,5 +0,0 @@ -# graph.yml: whitelist/blacklist of *contract* fields to export to .json/.yaml/.xml/... - -- id -- voip_subscribers*.provisioning_voip_subscriber.voip_usr_preferences*.attribute.attribute -- voip_subscribers*.provisioning_voip_subscriber.voip_usr_preferences*.value diff --git a/lib/NGCP/BulkProcessor/Projects/ETL/CDR/load.yml b/lib/NGCP/BulkProcessor/Projects/ETL/CDR/load.yml deleted file mode 100644 index 8c14467..0000000 --- a/lib/NGCP/BulkProcessor/Projects/ETL/CDR/load.yml +++ /dev/null @@ -1,44 +0,0 @@ -# load.yml: define which *contract* relations to fetch from db. - -#contracts.voip_subscribers: 1 -contracts.voip_subscribers: - include: !!perl/code | - { - my ($contract,$context) = @_; - #return 0 if $contract->{status} eq 'terminated'; - return 1; - } - - filter: !!perl/code | - { - my ($bill_subs,$context) = @_; - #_debug($context,"skipping terminated subscriber $bill_subs->{username}") if $bill_subs->{status} eq 'terminated'; - #return 0 if $bill_subs->{status} eq 'terminated'; - return 1; - } - - transform: !!perl/code | - { - my ($bill_subs,$context) = @_; - return $bill_subs; - } - -contracts.contact: 1 -contracts.voip_subscribers.primary_number: 1 -contracts.voip_subscribers.provisioning_voip_subscriber: 1 -contracts.voip_subscribers.provisioning_voip_subscriber.voip_dbaliases: 1 -contracts.voip_subscribers.provisioning_voip_subscriber.voip_usr_preferences: 1 -contracts.voip_subscribers.provisioning_voip_subscriber.voip_usr_preferences.attribute: 1 -contracts.voip_subscribers.provisioning_voip_subscriber.voip_usr_preferences.allowed_ips: 1 -contracts.voip_subscribers.provisioning_voip_subscriber.voip_usr_preferences.ncos: 1 -contracts.voip_subscribers.provisioning_voip_subscriber.voip_usr_preferences.cf_mapping: 1 -contracts.voip_subscribers.provisioning_voip_subscriber.voip_usr_preferences.cf_mapping.destinations: 1 -contracts.voip_subscribers.provisioning_voip_subscriber.voicemail_users: 1 -#contracts.voip_subscribers.provisioning_voip_subscriber.voicemail_users.voicemail_spool: 1 -contracts.voip_subscribers.provisioning_voip_subscriber.voip_fax_preferences: 1 -contracts.voip_subscribers.provisioning_voip_subscriber.voip_fax_destinations: - transform: !!perl/code | - { - my ($fax_destinations,$context) = @_; - return [ map { $_->{destination} . ' (' . $_->{filetype} . ')'; } @$fax_destinations ]; - } \ No newline at end of file diff --git a/lib/NGCP/BulkProcessor/Projects/ETL/CDR/process.pl b/lib/NGCP/BulkProcessor/Projects/ETL/CDR/process.pl deleted file mode 100644 index aea2662..0000000 --- a/lib/NGCP/BulkProcessor/Projects/ETL/CDR/process.pl +++ /dev/null @@ -1,319 +0,0 @@ -use strict; - -## no critic - -our $VERSION = "0.0"; - -use File::Basename; -use Cwd; -use lib Cwd::abs_path(File::Basename::dirname(__FILE__) . '/../../../../../'); - -use Getopt::Long qw(GetOptions); -use Fcntl qw(LOCK_EX LOCK_NB); - -use NGCP::BulkProcessor::Globals qw(); -use NGCP::BulkProcessor::Projects::ETL::CDR::Settings qw( - update_settings - update_tabular_fields - update_graph_fields - $tabular_yml - $graph_yml - - update_load_recursive - get_export_filename - $cdr_export_filename_format - $load_yml - - check_dry - $output_path - $defaultsettings - $defaultconfig - $dry - $skip_errors - $force -); - -use NGCP::BulkProcessor::Logging qw( - init_log - getlogger - $attachmentlogfile - scriptinfo - cleanuplogfiles - $currentlogfile -); -use NGCP::BulkProcessor::LogError qw ( - completion - done - scriptwarn - scripterror - filewarn - fileerror -); -use NGCP::BulkProcessor::LoadConfig qw( - load_config - $SIMPLE_CONFIG_TYPE - $YAML_CONFIG_TYPE - $ANY_CONFIG_TYPE -); -use NGCP::BulkProcessor::Array qw(removeduplicates); -use NGCP::BulkProcessor::Utils qw(getscriptpath prompt cleanupdir); -use NGCP::BulkProcessor::Mail qw( - cleanupmsgfiles -); - -use NGCP::BulkProcessor::SqlConnectors::CSVDB qw(cleanupcvsdirs); -use NGCP::BulkProcessor::SqlConnectors::SQLiteDB qw(cleanupdbfiles); - -use NGCP::BulkProcessor::Projects::ETL::CDR::ProjectConnectorPool qw(destroy_all_dbs get_csv_db get_sqlite_db); - -use NGCP::BulkProcessor::Projects::ETL::CDR::Dao::Tabular qw(); - -use NGCP::BulkProcessor::Projects::ETL::CDR::ExportCDR qw( - export_cdr_graph - export_cdr_tabular -); -#use NGCP::BulkProcessor::Projects::ETL::Cdr::ImportCdr qw( -# import_cdr_json -#); - -scripterror(getscriptpath() . ' already running',getlogger(getscriptpath())) unless flock DATA, LOCK_EX | LOCK_NB; - -my @TASK_OPTS = (); - -my $tasks = []; - -my $cleanup_task_opt = 'cleanup'; -push(@TASK_OPTS,$cleanup_task_opt); - -my $cleanup_all_task_opt = 'cleanup_all'; -push(@TASK_OPTS,$cleanup_all_task_opt); - -my $export_cdr_graph_task_opt = 'export_cdr_graph'; -push(@TASK_OPTS,$export_cdr_graph_task_opt); - -my $export_cdr_tabular_task_opt = 'export_cdr_tabular'; -push(@TASK_OPTS,$export_cdr_tabular_task_opt); - -#my $import_cdr_json_task_opt = 'import_cdr_json'; -#push(@TASK_OPTS,$import_cdr_json_task_opt); - -if (init()) { - main(); - exit(0); -} else { - exit(1); -} - -sub init { - - my $configfile = $defaultconfig; - my $settingsfile = $defaultsettings; - - return 0 unless GetOptions( - "config=s" => \$configfile, - "settings=s" => \$settingsfile, - "task=s" => $tasks, - "dry" => \$dry, - "skip-errors" => \$skip_errors, - "force" => \$force, - ); - - $tasks = removeduplicates($tasks,1); - - my $result = load_config($configfile); - init_log(); - $result &= load_config($settingsfile,\&update_settings,$SIMPLE_CONFIG_TYPE); - $result &= load_config($tabular_yml,\&update_tabular_fields,$YAML_CONFIG_TYPE); - $result &= load_config($graph_yml,\&update_graph_fields,$YAML_CONFIG_TYPE); - $result &= load_config($load_yml,\&update_load_recursive,$YAML_CONFIG_TYPE); - return $result; - -} - -sub main() { - - my @messages = (); - my @attachmentfiles = (); - my $result = 1; - my $completion = 0; - - if (defined $tasks and 'ARRAY' eq ref $tasks and (scalar @$tasks) > 0) { - scriptinfo('skip-errors: processing won\'t stop upon errors',getlogger(__PACKAGE__)) if $skip_errors; - foreach my $task (@$tasks) { - - if (lc($cleanup_task_opt) eq lc($task)) { - $result &= cleanup_task(\@messages,0) if taskinfo($cleanup_task_opt,$result); - } elsif (lc($cleanup_all_task_opt) eq lc($task)) { - $result &= cleanup_task(\@messages,1) if taskinfo($cleanup_all_task_opt,$result); - - } elsif (lc($export_cdr_graph_task_opt) eq lc($task)) { - $result &= export_cdr_graph_task(\@messages) if taskinfo($export_cdr_graph_task_opt,$result); - $completion |= 1; - } elsif (lc($export_cdr_tabular_task_opt) eq lc($task)) { - $result &= export_cdr_tabular_task(\@messages) if taskinfo($export_cdr_tabular_task_opt,$result); - $completion |= 1; - #} elsif (lc($import_cdr_json_task_opt) eq lc($task)) { - # if (taskinfo($import_cdr_json_task_opt,$result,1)) { - # next unless check_dry(); - # $result &= import_cdr_json_task(\@messages); - # $completion |= 1; - # } - - } else { - $result = 0; - scripterror("unknown task option '" . $task . "', must be one of " . join(', ',@TASK_OPTS),getlogger(getscriptpath())); - last; - } - } - } else { - $result = 0; - scripterror('at least one task option is required. supported tasks: ' . join(', ',@TASK_OPTS),getlogger(getscriptpath())); - } - - push(@attachmentfiles,$attachmentlogfile); - if ($completion) { - completion(join("\n\n",@messages),\@attachmentfiles,getlogger(getscriptpath())); - } else { - done(join("\n\n",@messages),\@attachmentfiles,getlogger(getscriptpath())); - } - - return $result; -} - -sub taskinfo { - my ($task,$result) = @_; - scriptinfo($result ? "starting task: '$task'" : "skipping task '$task' due to previous problems",getlogger(getscriptpath())); - return $result; -} - -sub cleanup_task { - my ($messages,$clean_generated) = @_; - my $result = 0; - if (!$clean_generated or $force or 'yes' eq lc(prompt("Type 'yes' to proceed: "))) { - eval { - cleanupcvsdirs(); - cleanupdbfiles(); - cleanuplogfiles(\&fileerror,\&filewarn,($currentlogfile,$attachmentlogfile)); - cleanupmsgfiles(\&fileerror,\&filewarn); - cleanupdir($output_path,1,\&filewarn,getlogger(getscriptpath())) if $clean_generated; - $result = 1; - }; - } - if ($@ or !$result) { - push(@$messages,'working directory cleanup INCOMPLETE'); - return 0; - } else { - push(@$messages,'working directory folders cleaned up'); - return 1; - } -} - -sub export_cdr_graph_task { - - my ($messages) = @_; - my ($result,$warning_count) = (0,0); - eval { - ($result,$warning_count) = export_cdr_graph(); - }; - my $err = $@; - my $stats = ": $warning_count warnings"; - eval { - #$stats .= "\n total mta subscriber records: " . - # NGCP::BulkProcessor::Projects::Migration::UPCAT::Dao::import::MtaSubscriber::countby_ccacsn() . ' rows'; - #my $added_count = NGCP::BulkProcessor::Projects::Migration::UPCAT::Dao::import::MtaSubscriber::countby_delta( - # $NGCP::BulkProcessor::Projects::Migration::UPCAT::Dao::import::MtaSubscriber::added_delta - #); - #$stats .= "\n new: $added_count rows"; - #my $existing_count = NGCP::BulkProcessor::Projects::Migration::UPCAT::Dao::import::MtaSubscriber::countby_delta( - # $NGCP::BulkProcessor::Projects::Migration::UPCAT::Dao::import::MtaSubscriber::updated_delta - #); - #$stats .= "\n existing: $existing_count rows"; - #my $deleted_count = NGCP::BulkProcessor::Projects::Migration::UPCAT::Dao::import::MtaSubscriber::countby_delta( - # $NGCP::BulkProcessor::Projects::Migration::UPCAT::Dao::import::MtaSubscriber::deleted_delta - #); - #$stats .= "\n removed: $deleted_count rows"; - }; - if ($err or !$result) { - push(@$messages,"exporting cdr (graph) INCOMPLETE$stats"); - } else { - push(@$messages,"exporting cdr (graph) completed$stats"); - } - destroy_all_dbs(); - return $result; - -} - -sub export_cdr_tabular_task { - - my ($messages) = @_; - my ($result,$warning_count) = (0,0); - eval { - ($result,$warning_count) = export_cdr_tabular(); - }; - my $err = $@; - my $stats = ": $warning_count warnings"; - eval { - $stats .= "\n total subscriber records: " . - NGCP::BulkProcessor::Projects::ETL::CDR::Dao::Tabular::countby_delta() . ' rows'; - my $added_count = NGCP::BulkProcessor::Projects::ETL::CDR::Dao::Tabular::countby_delta( - $NGCP::BulkProcessor::Projects::ETL::CDR::Dao::Tabular::added_delta - ); - $stats .= "\n new: $added_count rows"; - my $existing_count = NGCP::BulkProcessor::Projects::ETL::CDR::Dao::Tabular::countby_delta( - $NGCP::BulkProcessor::Projects::ETL::CDR::Dao::Tabular::updated_delta - ); - $stats .= "\n existing: $existing_count rows"; - my $deleted_count = NGCP::BulkProcessor::Projects::ETL::CDR::Dao::Tabular::countby_delta( - $NGCP::BulkProcessor::Projects::ETL::CDR::Dao::Tabular::deleted_delta - ); - $stats .= "\n removed: $deleted_count rows"; - my ($export_filename,$export_format) = get_export_filename($cdr_export_filename_format); - if ('sqlite' eq $export_format) { - &get_sqlite_db()->copydbfile($export_filename); - } elsif ('csv' eq $export_format) { - NGCP::BulkProcessor::Projects::ETL::CDR::Dao::Tabular::copy_table(\&get_csv_db); - &get_csv_db()->copytablefile(NGCP::BulkProcessor::Projects::ETL::CDR::Dao::Tabular::gettablename(),$export_filename); - } else { - push(@$messages,'invalid extension for output filename $export_filename'); - } - }; - if ($err or !$result) { - push(@$messages,"exporting cdr (tabular) INCOMPLETE$stats"); - } else { - push(@$messages,"exporting cdr (tabular) completed$stats"); - } - destroy_all_dbs(); - return $result; - -} - -#sub import_cdr_json_task { -# -# my ($messages) = @_; -# my ($result,$warning_count,$contract_read_count,$subscriber_read_count,$contract_created_count,$subscriber_created_count,$contract_failed_count,$subscriber_failed_count) = (0,0,0,0,0,0,0,0); -# eval { -# ($result,$warning_count,$contract_read_count,$subscriber_read_count,$contract_created_count,$subscriber_created_count,$contract_failed_count,$subscriber_failed_count) = import_cdr_json(); -# }; -# my $err = $@; -# my $stats = ": $warning_count warnings"; -# eval { -# $stats .= "\n contracts read: " . $contract_read_count; -# $stats .= "\n contracts created: " . $contract_created_count; -# $stats .= "\n contracts failed: " . $contract_failed_count; -# $stats .= "\n subscribers read: " . $subscriber_read_count; -# $stats .= "\n subscribers created: " . $subscriber_created_count; -# $stats .= "\n subscribers failed: " . $subscriber_failed_count; -# }; -# if ($err or !$result) { -# push(@$messages,"importing cdr (json) INCOMPLETE$stats"); -# } else { -# push(@$messages,"importing cdr (json) completed$stats"); -# } -# destroy_all_dbs(); -# return $result; -# -#} - -__DATA__ -This exists to allow the locking code at the beginning of the file to work. -DO NOT REMOVE THESE LINES! diff --git a/lib/NGCP/BulkProcessor/Projects/ETL/CDR/settings.cfg b/lib/NGCP/BulkProcessor/Projects/ETL/CDR/settings.cfg deleted file mode 100644 index 1f381df..0000000 --- a/lib/NGCP/BulkProcessor/Projects/ETL/CDR/settings.cfg +++ /dev/null @@ -1,58 +0,0 @@ -#dry=0 -#skip_errors=0 - -schema_version = Trunk - -export_cdr_multithreading = 1 -export_cdr_numofthreads = 4 -export_cdr_blocksize = 1000 - -cdr_export_filename=cdr_%s.csv - -load_yml = load.yml -tabular_yml = tabular.yml -graph_yml = graph.yml -graph_fields_mode = whitelist - -csv_all_expected_fields = 0 - -sqlite_db_file = sqlite -csv_dir = cdr -tabular_single_row_txn = 1 -ignore_tabular_unique = 0 - -#cdr_import_filename=cdr_20210216173615.json -#split_cdr = 1 -#cdr_import_multithreading = 1 -#cdr_import_numofthreads = 4 -#cdr_reseller_name = default -#cdr_billing_profile_name = Default Billing Profile -#cdr_domain = test1610072315.example.org -#cdr_contact_email_format = DN0%2$s%3$s@example.org -#cdr_timezone = Europe/Vienna -#subscriber_profile_set_name = subscriber_profile_1_set_65261 -#subscriber_profile_name = subscriber_profile_1_65261 -## sip username as webusername: -##webusername_format = %1$s -## webusername = cc+ac+sn: -##webusername_format = %2$s%3$s%4$s -## webusername = 0+ac+sn: -#webusername_format = 0%3$s%4$s -## sip username as external_id: -##subscriber_externalid_format = %1$s -## external_id = cc+ac+sn: -##subscriber_externalid_format = %2$s%3$s%4$s -## external_id = 0+ac+sn: -#subscriber_externalid_format = 0%3$s%4$s -## subscriber contact will be created, only if one of below is set. -#subscriber_contact_email_format = DN0%2$s%3$s@domain.org -#subscriber_timezone = Europe/Vienna - -#cf_default_priority: 1 -#cf_default_timeout: 300 -#cft_default_ringtimeout: 20 - -##write sql files for legacy db to set/unset the is_external pref of migrated subscribers: -# -#rollback_sql_export_filename_format = delete_subscribers_%s.sql -#rollback_sql_stmt_format = start transaction;call billing.remove_subscriber("%1$s",%2$s);commit; diff --git a/lib/NGCP/BulkProcessor/Projects/ETL/CDR/tabular.yml b/lib/NGCP/BulkProcessor/Projects/ETL/CDR/tabular.yml deleted file mode 100644 index 7ea0796..0000000 --- a/lib/NGCP/BulkProcessor/Projects/ETL/CDR/tabular.yml +++ /dev/null @@ -1,70 +0,0 @@ -# tabular.yml: define which *subscriber* columns to add tabular (.db/.csv) exports. - -- path: contract.id - transform: !!perl/code | - { - my ($id,$bill_subs) = @_; - return $id; - } - -- path: primary_number.cc -- path: primary_number.ac -- path: primary_number.sn -- path: provisioning_voip_subscriber.voicemail_users[0].attach -- path: provisioning_voip_subscriber.voicemail_users[0].delete -- path: provisioning_voip_subscriber.voicemail_users[0].email -- path: provisioning_voip_subscriber.voicemail_users[0].password -- path: provisioning_voip_subscriber.voip_usr_preferences.allowed_clis - sep: ',' - field: 'value' -- path: provisioning_voip_subscriber.voip_usr_preferences.allowed_ips_grp[0].allowed_ips - sep: ',' - field: 'ipnet' -- path: provisioning_voip_subscriber.voip_usr_preferences.block_out_list - sep: ',' - field: 'value' -- path: provisioning_voip_subscriber.voip_usr_preferences.block_out_mode[0].value -- path: provisioning_voip_subscriber.voip_usr_preferences.block_in_list - sep: ',' - field: 'value' -- path: provisioning_voip_subscriber.voip_usr_preferences.block_in_mode[0].value -- path: provisioning_voip_subscriber.voip_usr_preferences.adm_block_in_list - sep: ',' - field: 'value' -- path: provisioning_voip_subscriber.voip_usr_preferences.adm_block_in_mode[0].value -- path: provisioning_voip_subscriber.voip_usr_preferences.adm_block_out_list - sep: ',' - field: 'value' -- path: provisioning_voip_subscriber.voip_usr_preferences.adm_block_out_mode[0].value -- path: provisioning_voip_subscriber.voip_usr_preferences.ncos_id[0].ncos.level -- path: provisioning_voip_subscriber.voip_usr_preferences.adm_ncos_id[0].ncos.level -- path: provisioning_voip_subscriber.voip_usr_preferences.cfb[0].cf_mapping.destinations - sep: ',' - field: 'destination' -- path: provisioning_voip_subscriber.voip_usr_preferences.cfna[0].cf_mapping.destinations - sep: ',' - field: 'destination' -- path: provisioning_voip_subscriber.voip_usr_preferences.cfo[0].cf_mapping.destinations - sep: ',' - field: 'destination' -- path: provisioning_voip_subscriber.voip_usr_preferences.cfr[0].cf_mapping.destinations - sep: ',' - field: 'destination' -- path: provisioning_voip_subscriber.voip_usr_preferences.cfs[0].cf_mapping.destinations - sep: ',' - field: 'destination' -- path: provisioning_voip_subscriber.voip_usr_preferences.cft[0].cf_mapping.destinations - sep: ',' - field: 'destination' -- path: provisioning_voip_subscriber.voip_usr_preferences.cfu[0].cf_mapping.destinations - sep: ',' - field: 'destination' -- path: provisioning_voip_subscriber.voip_fax_preferences.active -- path: provisioning_voip_subscriber.voip_fax_preferences.ecm -- path: provisioning_voip_subscriber.voip_fax_preferences.name -- path: provisioning_voip_subscriber.voip_fax_preferences.t38 -- path: provisioning_voip_subscriber.voip_fax_destinations - sep: ',' -- path: provisioning_voip_subscriber.voip_usr_preferences.force_inbound_calls_to_peer[0].value -- path: provisioning_voip_subscriber.voip_usr_preferences.lnp_for_local_sub[0].value - diff --git a/lib/NGCP/BulkProcessor/Projects/ETL/EDR/Dao/PeriodEvents.pm b/lib/NGCP/BulkProcessor/Projects/ETL/EDR/Dao/PeriodEvents.pm deleted file mode 100644 index 3edaaca..0000000 --- a/lib/NGCP/BulkProcessor/Projects/ETL/EDR/Dao/PeriodEvents.pm +++ /dev/null @@ -1,157 +0,0 @@ -package NGCP::BulkProcessor::Projects::ETL::EDR::Dao::PeriodEvents; -use strict; - -## no critic - -use NGCP::BulkProcessor::Projects::ETL::EDR::ProjectConnectorPool qw( - get_sqlite_db - destroy_all_dbs -); - -use NGCP::BulkProcessor::SqlProcessor qw( - registertableinfo - create_targettable - checktableinfo - copy_row - insert_stmt - transfer_table -); - -use NGCP::BulkProcessor::SqlRecord qw(); - -require Exporter; -our @ISA = qw(Exporter NGCP::BulkProcessor::SqlRecord); -our @EXPORT_OK = qw( - create_table - gettablename - check_table - getinsertstatement - - copy_table -); - -my $tablename = 'period_events'; -my $get_db = \&get_sqlite_db; - -my $fieldnames; -my $expected_fieldnames = [ - 'subscriber_id', - 'profile_id', - 'start_profile', - 'update_profile', - 'stop_profile', -]; - -my $primarykey_fieldnames = []; -my $indexes = { - $tablename . '_suscriber_id' => [ 'subscriber_id(11)' ], -}; - -sub new { - - my $class = shift; - my $self = NGCP::BulkProcessor::SqlRecord->new($class,$get_db, - $tablename,$expected_fieldnames,$indexes); - - copy_row($self,shift,$expected_fieldnames); - - return $self; - -} - -sub create_table { - - my ($truncate) = @_; - - my $db = &$get_db(); - - registertableinfo($db,__PACKAGE__,$tablename,$expected_fieldnames,$indexes,$primarykey_fieldnames); - return create_targettable($db,__PACKAGE__,$db,__PACKAGE__,$tablename,$truncate,1,undef); - -} - -sub findby_domainusername { - - my ($domain,$username,$load_recursive) = @_; - - check_table(); - my $db = &$get_db(); - my $table = $db->tableidentifier($tablename); - - return [] unless (defined $domain and defined $username); - - my $rows = $db->db_get_all_arrayref( - 'SELECT * FROM ' . $table . - ' WHERE ' . $db->columnidentifier('domain') . ' = ?' . - ' AND ' . $db->columnidentifier('username') . ' = ?' - , $domain, $username); - - return buildrecords_fromrows($rows,$load_recursive)->[0]; - -} - -sub copy_table { - - my ($get_target_db) = @_; - - check_table(); - #checktableinfo($get_target_db, - # __PACKAGE__,$tablename, - # get_fieldnames(1), - # $indexes); - - return transfer_table( - get_db => $get_db, - class => __PACKAGE__, - get_target_db => $get_target_db, - targetclass => __PACKAGE__, - targettablename => $tablename, - ); - -} - -sub buildrecords_fromrows { - - my ($rows,$load_recursive) = @_; - - my @records = (); - my $record; - - if (defined $rows and ref $rows eq 'ARRAY') { - foreach my $row (@$rows) { - $record = __PACKAGE__->new($row); - - # transformations go here ... - - push @records,$record; - } - } - - return \@records; - -} - -sub getinsertstatement { - - my ($insert_ignore) = @_; - check_table(); - return insert_stmt($get_db,__PACKAGE__,$insert_ignore); - -} - -sub gettablename { - - return $tablename; - -} - -sub check_table { - - return checktableinfo($get_db, - __PACKAGE__,$tablename, - $expected_fieldnames, - $indexes); - -} - -1; diff --git a/lib/NGCP/BulkProcessor/Projects/ETL/EDR/ExportEvents.pm b/lib/NGCP/BulkProcessor/Projects/ETL/EDR/ExportEvents.pm deleted file mode 100644 index f13f4c7..0000000 --- a/lib/NGCP/BulkProcessor/Projects/ETL/EDR/ExportEvents.pm +++ /dev/null @@ -1,271 +0,0 @@ -package NGCP::BulkProcessor::Projects::ETL::EDR::ExportEvents; -use strict; - -## no critic - -use threads::shared qw(); - -use Tie::IxHash; - -#use NGCP::BulkProcessor::Serialization qw(); -#use Scalar::Util qw(blessed); -#use MIME::Base64 qw(encode_base64); - -use NGCP::BulkProcessor::Projects::ETL::EDR::Settings qw( - $dry - $skip_errors - - $export_subscriber_profiles_multithreading - $export_subscriber_profiles_numofthreads - $export_subscriber_profiles_blocksize - $export_subscriber_profiles_joins - $export_subscriber_profiles_conditions - $export_subscriber_profiles_limit - - $period_events_single_row_txn - $ignore_period_events_unique - -); - -use NGCP::BulkProcessor::Logging qw ( - getlogger - processing_info - processing_debug -); -use NGCP::BulkProcessor::LogError qw( - rowprocessingerror - rowprocessingwarn - fileerror -); - -use NGCP::BulkProcessor::Dao::Trunk::accounting::events qw(); - -use NGCP::BulkProcessor::Projects::ETL::EDR::Dao::PeriodEvents qw(); - -use NGCP::BulkProcessor::Projects::ETL::EDR::ProjectConnectorPool qw( - get_sqlite_db - destroy_all_dbs - ping_all_dbs -); - -use NGCP::BulkProcessor::Utils qw(create_uuid threadid timestamp stringtobool trim); #check_ipnet -use NGCP::BulkProcessor::DSSorter qw(sort_by_configs); -use NGCP::BulkProcessor::Array qw(contains); -use NGCP::BulkProcessor::Calendar qw(from_epoch datetime_to_string); -#use NGCP::BulkProcessor::DSPath qw(); - -require Exporter; -our @ISA = qw(Exporter); -our @EXPORT_OK = qw( - export_subscriber_profiles -); - -sub export_subscriber_profiles { - - my $result = NGCP::BulkProcessor::Projects::ETL::EDR::Dao::PeriodEvents::create_table(1); - - my $static_context = {}; - - destroy_all_dbs(); - my $warning_count :shared = 0; - return ($result && NGCP::BulkProcessor::Dao::Trunk::accounting::events::process_subscribers( - static_context => $static_context, - process_code => sub { - my ($context,$records,$row_offset) = @_; - ping_all_dbs(); - my @period_event_rows = (); - foreach my $subscriber_id (map { $_->[0]; } @$records) { - if ($subscriber_id == 202) { - my $x=1; - print "blah"; - } - next unless _export_subscriber_profiles_init_context($context,$subscriber_id); - push(@period_event_rows, _get_period_event_rows($context)); - - - - if ($period_events_single_row_txn and (scalar @period_event_rows) > 0) { - while (defined (my $period_event_row = shift @period_event_rows)) { - if ($skip_errors) { - eval { _insert_period_events_rows($context,[$period_event_row]); }; - _warn($context,$@) if $@; - } else { - _insert_period_events_rows($context,[$period_event_row]); - } - } - } - } - - if (not $period_events_single_row_txn and (scalar @period_event_rows) > 0) { - if ($skip_errors) { - eval { insert_period_events_rows($context,\@period_event_rows); }; - _warn($context,$@) if $@; - } else { - insert_period_events_rows($context,\@period_event_rows); - } - } - - return 1; - }, - init_process_context_code => sub { - my ($context)= @_; - $context->{db} = &get_sqlite_db(); - $context->{error_count} = 0; - $context->{warning_count} = 0; - }, - uninit_process_context_code => sub { - my ($context)= @_; - undef $context->{db}; - destroy_all_dbs(); - { - lock $warning_count; - $warning_count += $context->{warning_count}; - } - }, - destroy_reader_dbs_code => \&destroy_all_dbs, - blocksize => $export_subscriber_profiles_blocksize, - multithreading => $export_subscriber_profiles_multithreading, - numofthreads => $export_subscriber_profiles_numofthreads, - joins => $export_subscriber_profiles_joins, - conditions => $export_subscriber_profiles_conditions, - #sort => [{ column => 'id', numeric => 1, dir => 1 }], - #limit => $export_subscriber_profiles_limit, - ),$warning_count,); - -} - -sub _export_subscriber_profiles_init_context { - - my ($context,$subscriber_id) = @_; - - my $result = 1; - - $context->{events} = NGCP::BulkProcessor::Dao::Trunk::accounting::events::findby_subscriberid( - undef,$subscriber_id,$export_subscriber_profiles_joins,$export_subscriber_profiles_conditions); - - $context->{subscriber_id} = $subscriber_id; - - return $result; - -} - -sub _get_period_event_rows { - - my ($context) = @_; - - my $profile_events = { - start => undef, - update => [], - stop => undef, - }; - my $last_event; - - my %subscriber_profiles = (); - tie(%subscriber_profiles, 'Tie::IxHash'); - - foreach my $event (@{sort_by_configs([ grep { contains($_->{type},[ qw(start_profile update_profile end_profile) ]); } @{$context->{events}} ],[ - { numeric => 1, - dir => 1, #-1, - memberchain => [ 'id' ], - } - ])}) { - if ($event->{type} eq 'start_profile') { - if (not defined $last_event or $last_event->{type} eq 'end_profile') { - $profile_events->{start} = $event; - $last_event = $event; - $subscriber_profiles{$event->{new_status}} = $profile_events; - } else { - - } - } elsif ($event->{type} eq 'update_profile') { - if (defined $last_event and contains($last_event->{type},[ qw(start_profile update_profile) ])) { - push(@{$profile_events->{update}},$event); - $last_event = $event; - } else { - - } - } elsif ($event->{type} eq 'end_profile') { - if (defined $last_event and contains($last_event->{type},[ qw(start_profile update_profile) ])) { - $profile_events->{stop} = $event; - $last_event = $event; - $profile_events = { - start => undef, - update => [], - stop => undef, - }; - } else { - - } - } - } - - my @period_event_rows = (); - foreach my $profile_id (keys %subscriber_profiles) { - $profile_events = $subscriber_profiles{$profile_id}; - push(@period_event_rows,[ - $context->{subscriber_id}, - $profile_id, - datetime_to_string(from_epoch($profile_events->{start}->{timestamp})), - join(",",map { datetime_to_string(from_epoch($_->{timestamp})); } @{$profile_events->{update}}), - (defined $profile_events->{stop} ? datetime_to_string(from_epoch($profile_events->{stop}->{timestamp})) : undef), - ]); - } - - return @period_event_rows; - -} - -sub _insert_period_events_rows { - my ($context,$subscriber_rows) = @_; - $context->{db}->db_do_begin( - NGCP::BulkProcessor::Projects::ETL::EDR::Dao::PeriodEvents::getinsertstatement($ignore_period_events_unique), - ); - eval { - $context->{db}->db_do_rowblock($subscriber_rows); - $context->{db}->db_finish(); - }; - my $err = $@; - if ($err) { - eval { - $context->{db}->db_finish(1); - }; - die($err); - } - -} - - -sub _error { - - my ($context,$message) = @_; - $context->{error_count} = $context->{error_count} + 1; - rowprocessingerror($context->{tid} // threadid(),$message,getlogger(__PACKAGE__)); - -} - -sub _warn { - - my ($context,$message) = @_; - $context->{warning_count} = $context->{warning_count} + 1; - rowprocessingwarn($context->{tid} // threadid(),$message,getlogger(__PACKAGE__)); - -} - -sub _info { - - my ($context,$message,$debug) = @_; - if ($debug) { - processing_debug($context->{tid} // threadid(),$message,getlogger(__PACKAGE__)); - } else { - processing_info($context->{tid} // threadid(),$message,getlogger(__PACKAGE__)); - } -} - -sub _debug { - - my ($context,$message,$debug) = @_; - processing_debug($context->{tid} // threadid(),$message,getlogger(__PACKAGE__)); - -} - -1; diff --git a/lib/NGCP/BulkProcessor/Projects/ETL/EDR/ProjectConnectorPool.pm b/lib/NGCP/BulkProcessor/Projects/ETL/EDR/ProjectConnectorPool.pm deleted file mode 100644 index 987d7d3..0000000 --- a/lib/NGCP/BulkProcessor/Projects/ETL/EDR/ProjectConnectorPool.pm +++ /dev/null @@ -1,120 +0,0 @@ -package NGCP::BulkProcessor::Projects::ETL::EDR::ProjectConnectorPool; -use strict; - -## no critic - -use File::Basename; -use Cwd; -use lib Cwd::abs_path(File::Basename::dirname(__FILE__) . '/../../../'); - -use NGCP::BulkProcessor::Projects::ETL::EDR::Settings qw( - $csv_dir - $sqlite_db_file -); - -use NGCP::BulkProcessor::ConnectorPool qw( - get_connectorinstancename -); - -use NGCP::BulkProcessor::SqlConnectors::CSVDB qw(); -use NGCP::BulkProcessor::SqlConnectors::SQLiteDB qw($staticdbfilemode); - -use NGCP::BulkProcessor::SqlProcessor qw(cleartableinfo); - -require Exporter; -our @ISA = qw(Exporter); -our @EXPORT_OK = qw( - - get_sqlite_db - sqlite_db_tableidentifier - - get_csv_db - csv_db_tableidentifier - - destroy_dbs - destroy_all_dbs - ping_all_dbs - -); - -my $sqlite_dbs = {}; -my $csv_dbs = {}; - -sub get_sqlite_db { - - my ($instance_name,$reconnect) = @_; - my $name = get_connectorinstancename($instance_name); - - if (not defined $sqlite_dbs->{$name}) { - $sqlite_dbs->{$name} = NGCP::BulkProcessor::SqlConnectors::SQLiteDB->new($instance_name); - if (not defined $reconnect) { - $reconnect = 1; - } - } - if ($reconnect) { - $sqlite_dbs->{$name}->db_connect($staticdbfilemode,$sqlite_db_file); - } - - return $sqlite_dbs->{$name}; - -} - -sub sqlite_db_tableidentifier { - - my ($get_target_db,$tablename) = @_; - my $target_db = (ref $get_target_db eq 'CODE') ? &$get_target_db() : $get_target_db; - return $target_db->getsafetablename(NGCP::BulkProcessor::SqlConnectors::SQLiteDB::get_tableidentifier($tablename,$staticdbfilemode,$sqlite_db_file)); - -} - -sub get_csv_db { - - my ($instance_name,$reconnect) = @_; - my $name = get_connectorinstancename($instance_name); - if (not defined $csv_dbs->{$name}) { - $csv_dbs->{$name} = NGCP::BulkProcessor::SqlConnectors::CSVDB->new($instance_name); - if (not defined $reconnect) { - $reconnect = 1; - } - } - if ($reconnect) { - $csv_dbs->{$name}->db_connect($csv_dir); - } - return $csv_dbs->{$name}; - -} - -sub csv_db_tableidentifier { - - my ($get_target_db,$tablename) = @_; - my $target_db = (ref $get_target_db eq 'CODE') ? &$get_target_db() : $get_target_db; - return $target_db->getsafetablename(NGCP::BulkProcessor::SqlConnectors::CSVDB::get_tableidentifier($tablename,$csv_dir)); - -} - -sub destroy_dbs { - - foreach my $name (keys %$sqlite_dbs) { - cleartableinfo($sqlite_dbs->{$name}); - undef $sqlite_dbs->{$name}; - delete $sqlite_dbs->{$name}; - } - - foreach my $name (keys %$csv_dbs) { - cleartableinfo($csv_dbs->{$name}); - undef $csv_dbs->{$name}; - delete $csv_dbs->{$name}; - } - -} - -sub destroy_all_dbs() { - destroy_dbs(); - NGCP::BulkProcessor::ConnectorPool::destroy_dbs(); -} - -sub ping_all_dbs() { - NGCP::BulkProcessor::ConnectorPool::ping_dbs(); -} - -1; diff --git a/lib/NGCP/BulkProcessor/Projects/ETL/EDR/Settings.pm b/lib/NGCP/BulkProcessor/Projects/ETL/EDR/Settings.pm deleted file mode 100644 index 809020e..0000000 --- a/lib/NGCP/BulkProcessor/Projects/ETL/EDR/Settings.pm +++ /dev/null @@ -1,235 +0,0 @@ -package NGCP::BulkProcessor::Projects::ETL::EDR::Settings; -use strict; - -## no critic - -use File::Basename qw(fileparse); - -use NGCP::BulkProcessor::Globals qw( - $working_path - $enablemultithreading - $cpucount - create_path -); - -use NGCP::BulkProcessor::Logging qw( - getlogger - scriptinfo - configurationinfo -); - -use NGCP::BulkProcessor::LogError qw( - fileerror - filewarn - configurationwarn - configurationerror -); - -use NGCP::BulkProcessor::LoadConfig qw( - split_tuple - parse_regexp -); - -use NGCP::BulkProcessor::Utils qw(prompt timestampdigits threadid load_module); - -use NGCP::BulkProcessor::Array qw(contains); - -require Exporter; -our @ISA = qw(Exporter); -our @EXPORT_OK = qw( - update_settings - - get_export_filename - - $ignore_period_events_unique - $period_events_single_row_txn - - $sqlite_db_file - $csv_dir - - check_dry - - $output_path - $input_path - - $subscriber_profiles_export_filename_format - - $defaultsettings - $defaultconfig - - $dry - $skip_errors - $force - - $export_subscriber_profiles_multithreading - $export_subscriber_profiles_numofthreads - $export_subscriber_profiles_blocksize - - $export_subscriber_profiles_joins - $export_subscriber_profiles_conditions - $export_subscriber_profiles_limit - -); - -our $defaultconfig = 'config.cfg'; -our $defaultsettings = 'settings.cfg'; - -our $ignore_period_events_unique = 0; -our $period_events_single_row_txn = 1; - -our $output_path = $working_path . 'output/'; -our $input_path = $working_path . 'input/'; -our $csv_dir = 'events'; - -our $subscriber_profiles_export_filename_format = undef; - -our $force = 0; -our $dry = 0; -our $skip_errors = 0; - -our $sqlite_db_file = 'sqlite'; - -our $export_subscriber_profiles_multithreading = $enablemultithreading; -our $export_subscriber_profiles_numofthreads = $cpucount; -our $export_subscriber_profiles_blocksize = 1000; - -our $export_subscriber_profiles_joins = []; -our $export_subscriber_profiles_conditions = []; -our $export_subscriber_profiles_limit = undef; - -sub update_settings { - - my ($data,$configfile) = @_; - - if (defined $data) { - - my $result = 1; - my $regexp_result; - - #&$configurationinfocode("testinfomessage",$configlogger); - - $result &= _prepare_working_paths(1); - - $subscriber_profiles_export_filename_format = $data->{subscriber_profiles_export_filename} if exists $data->{subscriber_profiles_export_filename}; - get_export_filename($data->{subscriber_profiles_export_filename},$configfile); - - $sqlite_db_file = $data->{sqlite_db_file} if exists $data->{sqlite_db_file}; - $csv_dir = $data->{csv_dir} if exists $data->{csv_dir}; - - $dry = $data->{dry} if exists $data->{dry}; - $skip_errors = $data->{skip_errors} if exists $data->{skip_errors}; - - my $parse_result; - ($parse_result,$export_subscriber_profiles_joins) = _parse_export_joins($data->{export_subscriber_profiles_joins},$configfile); - $result &= $parse_result; - ($parse_result,$export_subscriber_profiles_conditions) = _parse_export_conditions($data->{export_subscriber_profiles_conditions},$configfile); - $result &= $parse_result; - - $export_subscriber_profiles_limit = $data->{export_subscriber_profiles_limit} if exists $data->{export_subscriber_profiles_limit}; - - $export_subscriber_profiles_multithreading = $data->{export_subscriber_profiles_multithreading} if exists $data->{export_subscriber_profiles_multithreading}; - $export_subscriber_profiles_numofthreads = _get_numofthreads($cpucount,$data,'export_subscriber_profiles_numofthreads'); - $export_subscriber_profiles_blocksize = $data->{export_subscriber_profiles_blocksize} if exists $data->{export_subscriber_profiles_blocksize}; - - $period_events_single_row_txn = $data->{period_events_single_row_txn} if exists $data->{period_events_single_row_txn}; - $ignore_period_events_unique = $data->{ignore_period_events_unique} if exists $data->{ignore_period_events_unique}; - - return $result; - } - return 0; - -} - -sub _prepare_working_paths { - - my ($create) = @_; - my $result = 1; - my $path_result; - - ($path_result,$input_path) = create_path($working_path . 'input',$input_path,$create,\&fileerror,getlogger(__PACKAGE__)); - $result &= $path_result; - ($path_result,$output_path) = create_path($working_path . 'output',$output_path,$create,\&fileerror,getlogger(__PACKAGE__)); - $result &= $path_result; - - return $result; - -} - -sub _get_numofthreads { - my ($default_value,$data,$key) = @_; - my $numofthreads = $default_value; - $numofthreads = $data->{$key} if exists $data->{$key}; - $numofthreads = $cpucount if $numofthreads > $cpucount; - return $numofthreads; -} - -sub get_export_filename { - my ($filename_format,$configfile) = @_; - my $export_filename; - my $export_format; - if ($filename_format) { - $export_filename = sprintf($filename_format,timestampdigits(),threadid()); - unless ($export_filename =~ /^\//) { - $export_filename = $output_path . $export_filename; - } - if (-e $export_filename and (unlink $export_filename) == 0) { - filewarn('cannot remove ' . $export_filename . ': ' . $!,getlogger(__PACKAGE__)); - $export_filename = undef; - } - my ($name,$path,$suffix) = fileparse($export_filename,".csv"); - if ($suffix eq '.csv') { - $export_format = 'csv'; - } else { - configurationerror($configfile,"$filename_format: .csv export file format required"); - } - } - return ($export_filename,$export_format); -} - -sub _parse_export_joins { - my ($token,$file) = @_; - my @joins = (); - if (defined $token and length($token) > 0) { - foreach my $f (_split(\$token)) { - next unless($f); - $f =~ s/^\s*\{?\s*//; - $f =~ s/\}\s*\}\s*$/}/; - my ($a, $b) = split(/\s*=>\s*{\s*/, $f); - $a =~ s/^\s*\'//; - $a =~ s/\'$//g; - $b =~ s/\s*\}\s*$//; - my ($c, $d) = split(/\s*=>\s*/, $b); - $c =~ s/^\s*\'//g; - $c =~ s/\'\s*//; - $d =~ s/^\s*\'//g; - $d =~ s/\'\s*//; - push @joins, { $a => { $c => $d } }; - } - } - return (1,\@joins); -} - -sub _parse_export_conditions { - my ($token,$file) = @_; - my @conditions = (); - if (defined $token and length($token) > 0) { - foreach my $f (_split(\$token)) { - next unless($f); - $f =~ s/^\s*\{?\s*//; - $f =~ s/\}\s*\}\s*$/}/; - my ($a, $b) = split(/\s*=>\s*{\s*/, $f); - $a =~ s/^\s*\'//; - $a =~ s/\'$//g; - $b =~ s/\s*\}\s*$//; - my ($c, $d) = split(/\s*=>\s*/, $b); - $c =~ s/^\s*\'//g; - $c =~ s/\'\s*//; - $d =~ s/^\s*\'//g; - $d =~ s/\'\s*//; - push @conditions, { $a => { $c => $d } }; - } - } - return (1,\@conditions); -} - -1; diff --git a/lib/NGCP/BulkProcessor/Projects/ETL/EDR/config.cfg b/lib/NGCP/BulkProcessor/Projects/ETL/EDR/config.cfg deleted file mode 100644 index 442b428..0000000 --- a/lib/NGCP/BulkProcessor/Projects/ETL/EDR/config.cfg +++ /dev/null @@ -1,61 +0,0 @@ -##general settings: -working_path = /var/sipwise -cpucount = 4 -enablemultithreading = 1 - -##gearman/service listener config: -jobservers = 127.0.0.1:4730 - -##NGCP MySQL connectivity - "accounting" db: -accounting_host = db01 -accounting_port = 3306 -accounting_databasename = accounting -accounting_username = root -accounting_password = - -##NGCP MySQL connectivity - "billing" db: -billing_host = db01 -billing_port = 3306 -billing_databasename = billing -billing_username = root -billing_password = - -##NGCP MySQL connectivity - "provisioning" db: -provisioning_host = db01 -provisioning_port = 3306 -provisioning_databasename = provisioning -provisioning_username = root -provisioning_password = - -##NGCP MySQL connectivity - "kamailio" db: -kamailio_host = db01 -kamailio_port = 3306 -kamailio_databasename = kamailio -kamailio_username = root -kamailio_password = - -##NGCP MySQL connectivity - default db for distributed transactions (XA) to connect to: -xa_host = db01 -xa_port = 3306 -xa_databasename = ngcp -xa_username = root -xa_password = - -##NGCP REST-API connectivity: -ngcprestapi_uri = https://127.0.0.1:1443 -ngcprestapi_username = administrator -ngcprestapi_password = administrator -ngcprestapi_realm = api_admin_http - -##sending email: -emailenable = 0 -erroremailrecipient = -warnemailrecipient = -completionemailrecipient = rkrenn@sipwise.com -doneemailrecipient = - -##logging: -fileloglevel = INFO -#DEBUG -screenloglevel = INFO -emailloglevel = OFF diff --git a/lib/NGCP/BulkProcessor/Projects/ETL/EDR/config.debug.cfg b/lib/NGCP/BulkProcessor/Projects/ETL/EDR/config.debug.cfg deleted file mode 100644 index 504dc89..0000000 --- a/lib/NGCP/BulkProcessor/Projects/ETL/EDR/config.debug.cfg +++ /dev/null @@ -1,61 +0,0 @@ -##general settings: -working_path = /home/rkrenn/temp/customer_exporter -cpucount = 4 -enablemultithreading = 1 - -##gearman/service listener config: -jobservers = 127.0.0.1:4730 - -##NGCP MySQL connectivity - "accounting" db: -accounting_host = 192.168.0.96 -accounting_port = 3306 -accounting_databasename = accounting -accounting_username = root -accounting_password = - -##NGCP MySQL connectivity - "billing" db: -billing_host = 192.168.0.96 -billing_port = 3306 -billing_databasename = billing -billing_username = root -billing_password = - -##NGCP MySQL connectivity - "provisioning" db: -provisioning_host = 192.168.0.96 -provisioning_port = 3306 -provisioning_databasename = provisioning -provisioning_username = root -provisioning_password = - -##NGCP MySQL connectivity - "kamailio" db: -kamailio_host = 192.168.0.96 -kamailio_port = 3306 -kamailio_databasename = kamailio -kamailio_username = root -kamailio_password = - -##NGCP MySQL connectivity - default db for distributed transactions (XA) to connect to: -xa_host = 192.168.0.96 -xa_port = 3306 -xa_databasename = ngcp -xa_username = root -xa_password = - -##NGCP REST-API connectivity: -ngcprestapi_uri = https://127.0.0.1:1443 -ngcprestapi_username = administrator -ngcprestapi_password = administrator -ngcprestapi_realm = api_admin_http - -##sending email: -emailenable = 0 -erroremailrecipient = -warnemailrecipient = -completionemailrecipient = rkrenn@sipwise.com -doneemailrecipient = - -##logging: -fileloglevel = INFO -#DEBUG -screenloglevel = INFO -emailloglevel = OFF diff --git a/lib/NGCP/BulkProcessor/Projects/ETL/EDR/process.pl b/lib/NGCP/BulkProcessor/Projects/ETL/EDR/process.pl deleted file mode 100644 index 21c59da..0000000 --- a/lib/NGCP/BulkProcessor/Projects/ETL/EDR/process.pl +++ /dev/null @@ -1,214 +0,0 @@ -use strict; - -## no critic - -our $VERSION = "0.0"; - -use File::Basename; -use Cwd; -use lib Cwd::abs_path(File::Basename::dirname(__FILE__) . '/../../../../../'); - -use Getopt::Long qw(GetOptions); -use Fcntl qw(LOCK_EX LOCK_NB); - -use NGCP::BulkProcessor::Globals qw(); -use NGCP::BulkProcessor::Projects::ETL::EDR::Settings qw( - update_settings - - get_export_filename - $subscriber_profiles_export_filename_format - - check_dry - $output_path - $defaultsettings - $defaultconfig - $dry - $skip_errors - $force -); - -use NGCP::BulkProcessor::Logging qw( - init_log - getlogger - $attachmentlogfile - scriptinfo - cleanuplogfiles - $currentlogfile -); -use NGCP::BulkProcessor::LogError qw ( - completion - done - scriptwarn - scripterror - filewarn - fileerror -); -use NGCP::BulkProcessor::LoadConfig qw( - load_config - $SIMPLE_CONFIG_TYPE - $YAML_CONFIG_TYPE - $ANY_CONFIG_TYPE -); -use NGCP::BulkProcessor::Array qw(removeduplicates); -use NGCP::BulkProcessor::Utils qw(getscriptpath prompt cleanupdir); -use NGCP::BulkProcessor::Mail qw( - cleanupmsgfiles -); - -use NGCP::BulkProcessor::SqlConnectors::CSVDB qw(cleanupcvsdirs); -use NGCP::BulkProcessor::SqlConnectors::SQLiteDB qw(cleanupdbfiles); - -use NGCP::BulkProcessor::Projects::ETL::EDR::ProjectConnectorPool qw(destroy_all_dbs get_csv_db get_sqlite_db); - -use NGCP::BulkProcessor::Projects::ETL::EDR::Dao::PeriodEvents qw(); - -use NGCP::BulkProcessor::Projects::ETL::EDR::ExportEvents qw( - export_subscriber_profiles -); - -scripterror(getscriptpath() . ' already running',getlogger(getscriptpath())) unless flock DATA, LOCK_EX | LOCK_NB; - -my @TASK_OPTS = (); - -my $tasks = []; - -my $cleanup_task_opt = 'cleanup'; -push(@TASK_OPTS,$cleanup_task_opt); - -my $cleanup_all_task_opt = 'cleanup_all'; -push(@TASK_OPTS,$cleanup_all_task_opt); - -my $export_subscriber_profiles_task_opt = 'export_subscriber_profiles'; -push(@TASK_OPTS,$export_subscriber_profiles_task_opt); - -if (init()) { - main(); - exit(0); -} else { - exit(1); -} - -sub init { - - my $configfile = $defaultconfig; - my $settingsfile = $defaultsettings; - - return 0 unless GetOptions( - "config=s" => \$configfile, - "settings=s" => \$settingsfile, - "task=s" => $tasks, - "skip-errors" => \$skip_errors, - "force" => \$force, - ); - - $tasks = removeduplicates($tasks,1); - - my $result = load_config($configfile); - init_log(); - $result &= load_config($settingsfile,\&update_settings,$SIMPLE_CONFIG_TYPE); - return $result; - -} - -sub main() { - - my @messages = (); - my @attachmentfiles = (); - my $result = 1; - my $completion = 0; - - if (defined $tasks and 'ARRAY' eq ref $tasks and (scalar @$tasks) > 0) { - scriptinfo('skip-errors: processing won\'t stop upon errors',getlogger(__PACKAGE__)) if $skip_errors; - foreach my $task (@$tasks) { - - if (lc($cleanup_task_opt) eq lc($task)) { - $result &= cleanup_task(\@messages,0) if taskinfo($cleanup_task_opt,$result); - } elsif (lc($cleanup_all_task_opt) eq lc($task)) { - $result &= cleanup_task(\@messages,1) if taskinfo($cleanup_all_task_opt,$result); - - } elsif (lc($export_subscriber_profiles_task_opt) eq lc($task)) { - $result &= export_subscriber_profiles_task(\@messages) if taskinfo($export_subscriber_profiles_task_opt,$result); - $completion |= 1; - - } else { - $result = 0; - scripterror("unknown task option '" . $task . "', must be one of " . join(', ',@TASK_OPTS),getlogger(getscriptpath())); - last; - } - } - } else { - $result = 0; - scripterror('at least one task option is required. supported tasks: ' . join(', ',@TASK_OPTS),getlogger(getscriptpath())); - } - - push(@attachmentfiles,$attachmentlogfile); - if ($completion) { - completion(join("\n\n",@messages),\@attachmentfiles,getlogger(getscriptpath())); - } else { - done(join("\n\n",@messages),\@attachmentfiles,getlogger(getscriptpath())); - } - - return $result; -} - -sub taskinfo { - my ($task,$result) = @_; - scriptinfo($result ? "starting task: '$task'" : "skipping task '$task' due to previous problems",getlogger(getscriptpath())); - return $result; -} - -sub cleanup_task { - my ($messages,$clean_generated) = @_; - my $result = 0; - if (!$clean_generated or $force or 'yes' eq lc(prompt("Type 'yes' to proceed: "))) { - eval { - cleanupcvsdirs(); - cleanupdbfiles(); - cleanuplogfiles(\&fileerror,\&filewarn,($currentlogfile,$attachmentlogfile)); - cleanupmsgfiles(\&fileerror,\&filewarn); - cleanupdir($output_path,1,\&filewarn,getlogger(getscriptpath())) if $clean_generated; - $result = 1; - }; - } - if ($@ or !$result) { - push(@$messages,'working directory cleanup INCOMPLETE'); - return 0; - } else { - push(@$messages,'working directory folders cleaned up'); - return 1; - } -} - -sub export_subscriber_profiles_task { - - my ($messages) = @_; - my ($result,$warning_count) = (0,0); - eval { - ($result,$warning_count) = export_subscriber_profiles(); - }; - my $err = $@; - my $stats = ": $warning_count warnings"; - eval { - my ($export_filename,$export_format) = get_export_filename($subscriber_profiles_export_filename_format); - if ('sqlite' eq $export_format) { - &get_sqlite_db()->copydbfile($export_filename); - } elsif ('csv' eq $export_format) { - NGCP::BulkProcessor::Projects::ETL::EDR::Dao::PeriodEvents::copy_table(\&get_csv_db); - &get_csv_db()->copytablefile(NGCP::BulkProcessor::Projects::ETL::EDR::Dao::PeriodEvents::gettablename(),$export_filename); - } else { - push(@$messages,'invalid extension for output filename $export_filename'); - } - }; - if ($err or !$result) { - push(@$messages,"exporting subscriber profiles INCOMPLETE$stats"); - } else { - push(@$messages,"exporting subscriber profiles completed$stats"); - } - destroy_all_dbs(); - return $result; - -} - -__DATA__ -This exists to allow the locking code at the beginning of the file to work. -DO NOT REMOVE THESE LINES! diff --git a/lib/NGCP/BulkProcessor/Projects/ETL/EDR/settings.cfg b/lib/NGCP/BulkProcessor/Projects/ETL/EDR/settings.cfg deleted file mode 100644 index 2d7b235..0000000 --- a/lib/NGCP/BulkProcessor/Projects/ETL/EDR/settings.cfg +++ /dev/null @@ -1,21 +0,0 @@ -#dry=0 -#skip_errors=0 - -export_subscriber_profiles_multithreading = 1 -export_subscriber_profiles_numofthreads = 2 -export_subscriber_profiles_blocksize = 1000 -export_subscriber_profiles_limit = 10000 - -#export_cdr_conditions = { 'accounting.cdr.destination_domain' => { 'IN' => '("80.110.2.164","ccs.upc.at")' } } -#export_cdr_conditions = { 'accounting.cdr.destination_domain' => { '=' => '"ccs.upc.at"' } } -#, { 'accounting.cdr.rating_status' => { '=' => '"ok"' } } -#{ 'accounting.cdr.call_status' => { '=' => '"ok"' } } -#export_cdr_joins = { 'accounting.cdr_export_status_data esd' => { 'esd.cdr_id' => 'accounting.cdr.id' } }, { 'accounting.cdr_export_status es' => { 'es.id' => 'esd.status_id' } } -export_cdr_conditions = { 'accounting.cdr.id' => { 'IN' => '(51,53, 87,89, 55, 79, 65,67,69, 81,83,85, 111, 113)' } } - -subscriber_profiles_export_filename=subscriber_profiles_%s.csv - -sqlite_db_file = sqlite -csv_dir = events -period_events_single_row_txn = 1 -ignore_period_events_unique = 0 diff --git a/lib/NGCP/BulkProcessor/Projects/Massive/Generator/process.pl b/lib/NGCP/BulkProcessor/Projects/Massive/Generator/process.pl index 3d7e961..7b26e3f 100755 --- a/lib/NGCP/BulkProcessor/Projects/Massive/Generator/process.pl +++ b/lib/NGCP/BulkProcessor/Projects/Massive/Generator/process.pl @@ -58,7 +58,7 @@ use NGCP::BulkProcessor::Utils qw(getscriptpath prompt cleanupdir); use NGCP::BulkProcessor::Mail qw( cleanupmsgfiles ); -use NGCP::BulkProcessor::SqlConnectors::CSVDB qw(cleanupcvsdirs); +use NGCP::BulkProcessor::SqlConnectors::CSVDB qw(cleanupcsvdirs); use NGCP::BulkProcessor::SqlConnectors::SQLiteDB qw(cleanupdbfiles); use NGCP::BulkProcessor::RestConnectors::NGCPRestApi qw(cleanupcertfiles); @@ -204,7 +204,7 @@ sub cleanup_task { my $result = 0; if (!$clean_generated or $force or 'yes' eq lc(prompt("Type 'yes' to proceed: "))) { eval { - cleanupcvsdirs() if $clean_generated; + cleanupcsvdirs() if $clean_generated; cleanupdbfiles() if $clean_generated; cleanuplogfiles(\&fileerror,\&filewarn,($currentlogfile,$attachmentlogfile)); cleanupmsgfiles(\&fileerror,\&filewarn); diff --git a/lib/NGCP/BulkProcessor/Projects/Massive/RegistrationMonitoring/process.pl b/lib/NGCP/BulkProcessor/Projects/Massive/RegistrationMonitoring/process.pl index 930a653..adcd7d8 100644 --- a/lib/NGCP/BulkProcessor/Projects/Massive/RegistrationMonitoring/process.pl +++ b/lib/NGCP/BulkProcessor/Projects/Massive/RegistrationMonitoring/process.pl @@ -170,7 +170,7 @@ sub cleanup_task { my $result = 0; if (!$clean_generated or $force or 'yes' eq lc(prompt("Type 'yes' to proceed: "))) { eval { - #cleanupcvsdirs() if $clean_generated; + #cleanupcsvdirs() if $clean_generated; cleanupdbfiles() if $clean_generated; cleanuplogfiles(\&fileerror,\&filewarn,($currentlogfile,$attachmentlogfile)); cleanupmsgfiles(\&fileerror,\&filewarn); diff --git a/lib/NGCP/BulkProcessor/RestProcessor.pm b/lib/NGCP/BulkProcessor/RestProcessor.pm index 08cbb4a..7f01b40 100644 --- a/lib/NGCP/BulkProcessor/RestProcessor.pm +++ b/lib/NGCP/BulkProcessor/RestProcessor.pm @@ -13,6 +13,7 @@ use Time::HiRes qw(sleep); use NGCP::BulkProcessor::Globals qw( $enablemultithreading $cpucount + get_threadqueuelength ); use NGCP::BulkProcessor::Logging qw( getlogger @@ -21,6 +22,7 @@ use NGCP::BulkProcessor::Logging qw( restprocessingdone fetching_items processing_items + enable_threading_info ); use NGCP::BulkProcessor::LogError qw( @@ -46,6 +48,7 @@ my $thread_sleep_secs = 0.1; my $RUNNING = 1; my $COMPLETED = 2; my $ERROR = 4; +my $STOP = 8; sub get_query_string { my ($filters) = @_; @@ -195,6 +198,23 @@ sub process_collection { $processors{$processor->tid()} = $processor; } + my $signal_handler = sub { + my $tid = threadid(); + $errorstate = $STOP; + enable_threading_info(1); + restthreadingdebug("[$tid] interrupt signal received",getlogger(__PACKAGE__)); + #print("[$tid] interrupt signal received"); + #_info($context,"interrupt signal received"); + #$result = 0; + my $errorstates = \%errorstates; + lock $errorstates; + $errorstates->{$tid} = $STOP; + }; + local $SIG{TERM} = $signal_handler; + local $SIG{INT} = $signal_handler; + local $SIG{QUIT} = $signal_handler; + local $SIG{HUP} = $signal_handler; + $reader->join(); restthreadingdebug('reader thread joined',getlogger(__PACKAGE__)); while ((scalar keys %processors) > 0) { @@ -208,7 +228,8 @@ sub process_collection { sleep($thread_sleep_secs); } - $errorstate = (_get_other_threads_state(\%errorstates,$tid) & ~$RUNNING); + $errorstate = $COMPLETED if $errorstate == $RUNNING; + $errorstate |= (_get_other_threads_state(\%errorstates,$tid) & ~$RUNNING); } else { @@ -302,7 +323,7 @@ sub _reader { } my $i = 0; my $state = $RUNNING; #start at first - while (($state & $RUNNING) == $RUNNING and ($state & $ERROR) == 0) { #as long there is one running consumer and no defunct consumer + while (($state & $RUNNING) == $RUNNING and ($state & $ERROR) == 0 and ($state & $STOP) == 0) { #as long there is one running consumer and no defunct consumer fetching_items($restapi,$context->{path_query},$i,$blocksize,getlogger(__PACKAGE__)); my $collection_page; @@ -318,7 +339,7 @@ sub _reader { $context->{queue}->enqueue(\%packet); #$packet); $blockcount++; #wait if thequeue is full and there there is one running consumer - while (((($state = _get_other_threads_state($context->{errorstates},$tid)) & $RUNNING) == $RUNNING) and $context->{queue}->pending() >= $context->{threadqueuelength}) { + while (((($state = _get_other_threads_state($context->{errorstates},$tid)) & $RUNNING) == $RUNNING) and $context->{queue}->pending() >= get_threadqueuelength($context->{threadqueuelength})) { #yield(); sleep($thread_sleep_secs); } @@ -333,10 +354,11 @@ sub _reader { last; } } - if (not (($state & $RUNNING) == $RUNNING and ($state & $ERROR) == 0)) { + if (not (($state & $RUNNING) == $RUNNING and ($state & $ERROR) == 0 and ($state & $STOP) == 0)) { restthreadingdebug('[' . $tid . '] reader thread is shutting down (' . - (($state & $RUNNING) == $RUNNING ? 'still running consumer threads' : 'no running consumer threads') . ', ' . - (($state & $ERROR) == 0 ? 'no defunct thread(s)' : 'defunct thread(s)') . ') ...' + (($state & $RUNNING) == $RUNNING ? 'still running consumer threads' : uc('no running consumer threads')) . ', ' . + (($state & $ERROR) == 0 ? 'no defunct thread(s)' : uc('defunct thread(s)')) . ', ' . + (($state & $STOP) == 0 ? 'no thread(s) stopping by signal' : uc('thread(s) stopping by signal')) . ') ...' ,getlogger(__PACKAGE__)); } }; @@ -344,7 +366,7 @@ sub _reader { lock $context->{errorstates}; if ($@) { $context->{errorstates}->{$tid} = $ERROR; - } else { + } elsif ($context->{errorstates}->{$tid} != $STOP) { $context->{errorstates}->{$tid} = $COMPLETED; } return $context->{errorstates}->{$tid}; @@ -405,7 +427,7 @@ sub _process { lock $context->{errorstates}; if ($err) { $context->{errorstates}->{$tid} = $ERROR; - } else { + } elsif ($context->{errorstates}->{$tid} != $STOP) { $context->{errorstates}->{$tid} = $COMPLETED; #(not $rowblock_result) ? $ERROR : $COMPLETED; } return $context->{errorstates}->{$tid}; @@ -441,16 +463,17 @@ sub _get_stop_consumer_thread { $reader_state = $errorstates->{$context->{readertid}}; } $queuesize = $context->{queue}->pending(); - if (($other_threads_state & $ERROR) == 0 and ($queuesize > 0 or $reader_state == $RUNNING)) { + if (($other_threads_state & $ERROR) == 0 and ($other_threads_state & $STOP) == 0 and ($queuesize > 0 or $reader_state == $RUNNING)) { $result = 0; #keep the consumer thread running if there is no defunct thread and queue is not empty or reader is still running } if ($result) { restthreadingdebug('[' . $tid . '] consumer thread is shutting down (' . - (($other_threads_state & $ERROR) == 0 ? 'no defunct thread(s)' : 'defunct thread(s)') . ', ' . - ($queuesize > 0 ? 'blocks pending' : 'no blocks pending') . ', ' . - ($reader_state == $RUNNING ? 'reader thread running' : 'reader thread not running') . ') ...' + (($other_threads_state & $ERROR) == 0 ? 'no defunct thread(s)' : uc('defunct thread(s)')) . ', ' . + (($other_threads_state & $STOP) == 0 ? 'no thread(s) stopping by signal' : uc('thread(s) stopping by signal')) . ', ' . + ($queuesize > 0 ? 'blocks pending' : uc('no blocks pending')) . ', ' . + ($reader_state == $RUNNING ? 'reader thread running' : uc('reader thread not running')) . ') ...' ,getlogger(__PACKAGE__)); } diff --git a/lib/NGCP/BulkProcessor/SqlConnector.pm b/lib/NGCP/BulkProcessor/SqlConnector.pm index bdba25e..36f7faa 100644 --- a/lib/NGCP/BulkProcessor/SqlConnector.pm +++ b/lib/NGCP/BulkProcessor/SqlConnector.pm @@ -30,11 +30,11 @@ use NGCP::BulkProcessor::Calendar qw(); require Exporter; our @ISA = qw(Exporter); -our @EXPORT_OK = qw(get_tableidentifier); +our @EXPORT_OK = qw(get_tableidentifier $log_db_operations); #my $logger = getlogger(__PACKAGE__); -my $log_db_operations = 0; #0; +our $log_db_operations = 0; #0; my $temptable_randomstringlength = 4; @@ -988,6 +988,7 @@ sub db_finish { my $self = shift; my $transactional = shift; my $rollback = shift; + my $log = shift; # since this is also called from DESTROY, no die() here! @@ -1001,7 +1002,7 @@ sub db_finish { if ($transactional) { #$self->unlock_tables(); if ($rollback) { - $self->db_rollback(1); + $self->db_rollback($log); } else { $self->db_commit(); } diff --git a/lib/NGCP/BulkProcessor/SqlConnectors/CSVDB.pm b/lib/NGCP/BulkProcessor/SqlConnectors/CSVDB.pm index e1bfcdf..8b29a00 100644 --- a/lib/NGCP/BulkProcessor/SqlConnectors/CSVDB.pm +++ b/lib/NGCP/BulkProcessor/SqlConnectors/CSVDB.pm @@ -54,7 +54,7 @@ use File::Copy qw(); require Exporter; our @ISA = qw(Exporter NGCP::BulkProcessor::SqlConnector); our @EXPORT_OK = qw( - cleanupcvsdirs + cleanupcsvdirs xlsbin2csv xlsxbin2csv sanitize_column_name @@ -342,7 +342,7 @@ sub vacuum { } -sub cleanupcvsdirs { +sub cleanupcsvdirs { my (@remainingdbdirs) = @_; local *DBDIR; diff --git a/lib/NGCP/BulkProcessor/SqlConnectors/MySQLDB.pm b/lib/NGCP/BulkProcessor/SqlConnectors/MySQLDB.pm index f90853a..6aad2e8 100644 --- a/lib/NGCP/BulkProcessor/SqlConnectors/MySQLDB.pm +++ b/lib/NGCP/BulkProcessor/SqlConnectors/MySQLDB.pm @@ -176,7 +176,7 @@ sub db_connect { my $self = shift; - my ($databasename,$username,$password,$host,$port) = @_; + my ($databasename,$username,$password,$host,$port,$create_database) = @_; $self->SUPER::db_connect($databasename,$username,$password,$host,$port); @@ -192,7 +192,7 @@ sub db_connect { $self->{username} = $username; $self->{password} = $password; - if (not contains($databasename,$self->getdatabases(),0)) { + if ($create_database and not contains($databasename,$self->getdatabases(),0)) { $self->_createdatabase($databasename); } diff --git a/lib/NGCP/BulkProcessor/SqlConnectors/OracleDB.pm b/lib/NGCP/BulkProcessor/SqlConnectors/OracleDB.pm index 17e0c2e..ff620a4 100644 --- a/lib/NGCP/BulkProcessor/SqlConnectors/OracleDB.pm +++ b/lib/NGCP/BulkProcessor/SqlConnectors/OracleDB.pm @@ -198,7 +198,7 @@ sub db_connect { my $self = shift; - my ($servicename,$sid,$schema,$username,$password,$host,$port) = @_; + my ($servicename,$sid,$schema,$username,$password,$host,$port,$create_database) = @_; $self->SUPER::db_connect($servicename,$sid,$schema,$username,$password,$host,$port); @@ -240,7 +240,7 @@ sub db_connect { $self->{dbh} = $dbh; - if (not contains($schema,$self->getdatabases(),0)) { + if ($create_database and not contains($schema,$self->getdatabases(),0)) { $self->_createdatabase($schema); #notimplemented error... } @@ -253,7 +253,8 @@ sub db_connect { $self->db_do('ALTER SESSION SET NLS_TERRITORY = \'' . $connNLS_TERRITORY . '\''); #$self->db_do('ALTER SESSION SET NLS_CHARACTERSET = \'' . $connNLS_CHARACTERSET . '\''); $self->db_do('ALTER SESSION SET NLS_NUMERIC_CHARACTERS = \'.,\''); - $self->db_do('ALTER SESSION SET NLS_DATE_FORMAT = \'YYYY-MM-DD HH24:MI:SS\''); + $self->db_do('ALTER SESSION SET NLS_DATE_FORMAT = \'YYYY-MM-DD\''); + $self->db_do('ALTER SESSION SET NLS_TIMESTAMP_FORMAT = \'YYYY-MM-DD HH24:MI:SS\''); if (length($isolation_level) > 0) { $self->db_do('ALTER SESSION SET ISOLATION_LEVEL = ' . $isolation_level); diff --git a/lib/NGCP/BulkProcessor/SqlConnectors/PostgreSQLDB.pm b/lib/NGCP/BulkProcessor/SqlConnectors/PostgreSQLDB.pm index 636cce6..2ce17f0 100644 --- a/lib/NGCP/BulkProcessor/SqlConnectors/PostgreSQLDB.pm +++ b/lib/NGCP/BulkProcessor/SqlConnectors/PostgreSQLDB.pm @@ -208,7 +208,7 @@ sub db_connect { my $self = shift; - my ($schemaname,$username,$password,$host,$port) = @_; + my ($schemaname,$username,$password,$host,$port,$create_database) = @_; $self->SUPER::db_connect($schemaname,$username,$password,$host,$port); @@ -224,7 +224,7 @@ sub db_connect { $self->{username} = $username; $self->{password} = $password; - if (not contains($schemaname,$self->getdatabases(),0)) { + if ($create_database and not contains($schemaname,$self->getdatabases(),0)) { $self->_createdatabase($schemaname); } diff --git a/lib/NGCP/BulkProcessor/SqlConnectors/SQLServerDB.pm b/lib/NGCP/BulkProcessor/SqlConnectors/SQLServerDB.pm index ff7d9fa..5149f31 100644 --- a/lib/NGCP/BulkProcessor/SqlConnectors/SQLServerDB.pm +++ b/lib/NGCP/BulkProcessor/SqlConnectors/SQLServerDB.pm @@ -162,7 +162,8 @@ sub _dbd_connect { } else { #$connection_string = 'dbi:ODBC:driver=SQL Server Native Client 11.0;server=tcp:' . $self->{host} . ',' . $self->{port}; #$connection_string = 'dbi:ODBC:driver=ODBC Driver 17 for SQL Server;server=tcp:' . $self->{host} . ',' . $self->{port}; - $connection_string = 'dbi:ODBC:driver={FreeTDS};server=' . $self->{host} . ',' . $self->{port}; + #$connection_string = 'dbi:ODBC:driver={FreeTDS};server=' . $self->{host} . ',' . $self->{port}; + $connection_string = 'dbi:ODBC:driver={FreeTDS};server=' . $self->{host} . ';encryption=off;port=' . $self->{port}; } if (length($databasename) > 0) { $connection_string .= ';database=' . $databasename; @@ -212,7 +213,7 @@ sub db_connect { my $self = shift; - my ($databasename,$username,$password,$host,$port) = @_; + my ($databasename,$username,$password,$host,$port,$create_database) = @_; $self->SUPER::db_connect($databasename,$username,$password,$host,$port); @@ -232,7 +233,7 @@ sub db_connect { $self->{username} = $username; $self->{password} = $password; - if (not contains($databasename,$self->getdatabases(),0)) { + if ($create_database and not contains($databasename,$self->getdatabases(),0)) { $self->_createdatabase($databasename); } diff --git a/lib/NGCP/BulkProcessor/SqlProcessor.pm b/lib/NGCP/BulkProcessor/SqlProcessor.pm index d3bcb19..091e88e 100644 --- a/lib/NGCP/BulkProcessor/SqlProcessor.pm +++ b/lib/NGCP/BulkProcessor/SqlProcessor.pm @@ -13,6 +13,7 @@ use Time::HiRes qw(sleep); use NGCP::BulkProcessor::Globals qw( $enablemultithreading $cpucount + get_threadqueuelength $cells_transfer_memory_limit $transfer_defer_indexes ); @@ -45,6 +46,7 @@ use NGCP::BulkProcessor::Logging qw( processing_rows tablethreadingdebug + enable_threading_info ); use NGCP::BulkProcessor::LogError qw( @@ -97,14 +99,15 @@ my $minnumberofchunks = 10; my $tableprocessing_threadqueuelength = 10; #my $tableprocessing_threads = $cpucount; #3; -my $reader_connection_name = 'reader'; -#my $writer_connection_name = 'writer'; +my $reader_name = 'reader'; +my $writer_name = 'writer'; my $thread_sleep_secs = 0.1; my $RUNNING = 1; my $COMPLETED = 2; my $ERROR = 4; +my $STOP = 8; sub init_record { @@ -773,8 +776,8 @@ sub transfer_table { if (ref $get_db eq 'CODE' and ref $get_target_db eq 'CODE') { - my $db = &$get_db($reader_connection_name,1); - my $target_db = &$get_target_db(); #$writer_connection_name); + my $db = &$get_db($reader_name,1); # $reader_name + my $target_db = &$get_target_db($writer_name); #$writer_name); my $connectidentifier = $db->connectidentifier(); my $tid = threadid(); @@ -848,7 +851,8 @@ sub transfer_table { tablethreadingdebug('starting reader thread',getlogger(__PACKAGE__)); $reader = threads->create(\&_reader, - { queue => $queue, + { name => $reader_name, + queue => $queue, errorstates => \%errorstates, #readererrorstate_ref => \$readererrorstate, #writererrorstate_ref => \$writererrorstate, @@ -867,7 +871,8 @@ sub transfer_table { tablethreadingdebug('starting writer thread',getlogger(__PACKAGE__)); $writer = threads->create(\&_writer, - { queue => $queue, + { #name => $writer_name, + queue => $queue, errorstates => \%errorstates, readertid => $reader->tid(), #readererrorstate_ref => \$readererrorstate, @@ -882,17 +887,35 @@ sub transfer_table { destroy_dbs_code => $destroy_target_dbs_code, }); + my $signal_handler = sub { + my $tid = threadid(); + $errorstate = $STOP; + enable_threading_info(1); + tablethreadingdebug("[$tid] interrupt signal received",getlogger(__PACKAGE__)); + #print("[$tid] interrupt signal received"); + #_info($context,"interrupt signal received"); + #$result = 0; + my $errorstates = \%errorstates; + lock $errorstates; + $errorstates->{$tid} = $STOP; + }; + local $SIG{TERM} = $signal_handler; + local $SIG{INT} = $signal_handler; + local $SIG{QUIT} = $signal_handler; + local $SIG{HUP} = $signal_handler; + $reader->join(); tablethreadingdebug('reader thread joined',getlogger(__PACKAGE__)); $writer->join(); tablethreadingdebug('writer thread joined',getlogger(__PACKAGE__)); + $errorstate = $COMPLETED if $errorstate == $RUNNING; #$errorstate = $readererrorstate | $writererrorstate; - $errorstate = _get_other_threads_state(\%errorstates,$tid); + $errorstate |= _get_other_threads_state(\%errorstates,$tid); tablethreadingdebug('restoring db connections ...',getlogger(__PACKAGE__)); - #$db = &$get_db($reader_connection_name,1); + #$db = &$get_db($reader_name,1); $target_db = &$get_target_db(undef,1); if ($default_connection_reconnect) { $default_connection = &$get_db(undef,1); @@ -904,10 +927,10 @@ sub transfer_table { #$db->db_disconnect(); #undef $db; - #$db = &$get_db($reader_connection_name); + #$db = &$get_db($reader_name); #$target_db->db_disconnect(); #undef $target_db; - #$target_db = &$get_target_db($writer_connection_name); + #$target_db = &$get_target_db($writer_name); eval { $db->db_get_begin($selectstatement,@$values) if $db->rowblock_transactional; #$tablename @@ -1028,6 +1051,7 @@ sub process_table { my %params = @_; my ($get_db, + $name, $class, $process_code, $read_code, @@ -1042,6 +1066,7 @@ sub process_table { $select, $values) = @params{qw/ get_db + name class process_code read_code @@ -1061,7 +1086,7 @@ sub process_table { if (ref $get_db eq 'CODE') { - my $db = &$get_db($reader_connection_name,1); + my $db = &$get_db($name,1); #$reader_name my $connectidentifier = $db->connectidentifier(); my $tid = threadid(); @@ -1122,7 +1147,8 @@ sub process_table { tablethreadingdebug('starting reader thread',getlogger(__PACKAGE__)); $reader = threads->create(\&_reader, - { queue => $queue, + { name => $name, + queue => $queue, errorstates => \%errorstates, #readererrorstate_ref => \$readererrorstate, #writererrorstate_ref => \$processorerrorstate, @@ -1143,7 +1169,8 @@ sub process_table { tablethreadingdebug('starting processor thread ' . ($i + 1) . ' of ' . $tableprocessing_threads,getlogger(__PACKAGE__)); my $processor = threads->create(\&_process, _create_process_context($static_context, - { queue => $queue, + { name => $name, + queue => $queue, errorstates => \%errorstates, readertid => $reader->tid(), #readererrorstate_ref => \$readererrorstate, @@ -1174,6 +1201,23 @@ sub process_table { # } #} + my $signal_handler = sub { + my $tid = threadid(); + $errorstate = $STOP; + enable_threading_info(1); + tablethreadingdebug("[$tid] interrupt signal received",getlogger(__PACKAGE__)); + #print("[$tid] interrupt signal received"); + #_info($context,"interrupt signal received"); + #$result = 0; + my $errorstates = \%errorstates; + lock $errorstates; + $errorstates->{$tid} = $STOP; + }; + local $SIG{TERM} = $signal_handler; + local $SIG{INT} = $signal_handler; + local $SIG{QUIT} = $signal_handler; + local $SIG{HUP} = $signal_handler; + $reader->join(); tablethreadingdebug('reader thread joined',getlogger(__PACKAGE__)); #print 'threads running: ' . (scalar threads->list(threads::running)); @@ -1195,12 +1239,13 @@ sub process_table { sleep($thread_sleep_secs); } + $errorstate = $COMPLETED if $errorstate == $RUNNING; #$errorstate = $readererrorstate | $processorerrorstate; - $errorstate = (_get_other_threads_state(\%errorstates,$tid) & ~$RUNNING); + $errorstate |= (_get_other_threads_state(\%errorstates,$tid) & ~$RUNNING); tablethreadingdebug('restoring db connections ...',getlogger(__PACKAGE__)); - #$db = &$get_db($reader_connection_name,1); + #$db = &$get_db($reader_name,1); if ($default_connection_reconnect) { $default_connection = &$get_db(undef,1); } @@ -1210,8 +1255,8 @@ sub process_table { $blocksize //= _calc_blocksize($rowcount,scalar @fieldnames,0,undef); #$db->db_disconnect(); #undef $db; - #$db = &$get_db($reader_connection_name); - my $context = _create_process_context($static_context,{ tid => $tid }); + #$db = &$get_db($reader_name); + my $context = _create_process_context($static_context,{ tid => $tid, name => $name, }); my $rowblock_result = 1; eval { if (defined $init_process_context_code and 'CODE' eq ref $init_process_context_code) { @@ -1231,7 +1276,7 @@ sub process_table { $db->db_finish() unless $db->rowblock_transactional; my $realblocksize = scalar @$rowblock; if ($realblocksize > 0) { - processing_rows($tid,$i,$realblocksize,$rowcount,getlogger(__PACKAGE__)); + processing_rows($context,$i,$realblocksize,$rowcount,getlogger(__PACKAGE__)); $rowblock_result = &$process_code($context,$rowblock,$i); @@ -1251,6 +1296,7 @@ sub process_table { }; +print $@; if ($@) { $errorstate = $ERROR; } else { @@ -1274,6 +1320,7 @@ sub process_table { #$db->db_disconnect(); return 1; } else { + print "errorstate: $errorstate \n"; tableprocessingfailed($db,$tablename,$rowcount,getlogger(__PACKAGE__)); #$db->db_disconnect(); } @@ -1294,7 +1341,7 @@ sub _calc_blocksize { my $blocksize = int ( 10 ** $exp ); my $cellcount_in_memory = $columncount * $blocksize; if ($multithreaded) { - $cellcount_in_memory *= $threadqueuelength; + $cellcount_in_memory *= get_threadqueuelength($threadqueuelength); } while ( $cellcount_in_memory > $cells_transfer_memory_limit or @@ -1303,7 +1350,7 @@ sub _calc_blocksize { $blocksize = int ( 10 ** $exp ); $cellcount_in_memory = $columncount * $blocksize; if ($multithreaded) { - $cellcount_in_memory *= $threadqueuelength; + $cellcount_in_memory *= get_threadqueuelength($threadqueuelength); } } @@ -1353,16 +1400,19 @@ sub _get_stop_consumer_thread { $reader_state = $errorstates->{$context->{readertid}}; } $queuesize = $context->{queue}->pending(); - if (($other_threads_state & $ERROR) == 0 and ($queuesize > 0 or $reader_state == $RUNNING)) { + if (($other_threads_state & $ERROR) == 0 + and ($other_threads_state & $STOP) == 0 + and ($queuesize > 0 or $reader_state == $RUNNING)) { $result = 0; #keep the consumer thread running if there is no defunct thread and queue is not empty or reader is still running } if ($result) { tablethreadingdebug('[' . $tid . '] consumer thread is shutting down (' . - (($other_threads_state & $ERROR) == 0 ? 'no defunct thread(s)' : 'defunct thread(s)') . ', ' . - ($queuesize > 0 ? 'blocks pending' : 'no blocks pending') . ', ' . - ($reader_state == $RUNNING ? 'reader thread running' : 'reader thread not running') . ') ...' + (($other_threads_state & $ERROR) == 0 ? 'no defunct thread(s)' : uc('defunct thread(s)')) . ', ' . + (($other_threads_state & $STOP) == 0 ? 'no thread(s) stopping by signal' : uc('thread(s) stopping by signal')) . ', ' . + ($queuesize > 0 ? 'blocks pending' : uc('no blocks pending')) . ', ' . + ($reader_state == $RUNNING ? 'reader thread running' : uc('reader thread not running')) . ') ...' ,getlogger(__PACKAGE__)); } @@ -1387,7 +1437,7 @@ sub _reader { my $blockcount = 0; eval { - $reader_db = &{$context->{get_db}}(); #$reader_connection_name); + $reader_db = &{$context->{get_db}}($context->{name}); #$reader_name); $reader_db->db_get_begin($context->{selectstatement},@{$context->{values_ref}}) if $reader_db->rowblock_transactional; #$context->{tablename} tablethreadingdebug('[' . $tid . '] reader thread waiting for consumer threads',getlogger(__PACKAGE__)); while ((_get_other_threads_state($context->{errorstates},$tid) & $RUNNING) == 0) { #wait on cosumers to come up @@ -1396,7 +1446,7 @@ sub _reader { } my $i = 0; my $state = $RUNNING; #start at first - while (($state & $RUNNING) == $RUNNING and ($state & $ERROR) == 0) { #as long there is one running consumer and no defunct consumer + while (($state & $RUNNING) == $RUNNING and ($state & $ERROR) == 0 and ($state & $STOP) == 0) { #as long there is one running consumer and no defunct consumer fetching_rows($reader_db,$context->{tablename},$i,$context->{blocksize},$context->{rowcount},getlogger(__PACKAGE__)); $reader_db->db_get_begin($context->{selectstatement},$i,$context->{blocksize},@{$context->{values_ref}}) unless $reader_db->rowblock_transactional; my $rowblock = $reader_db->db_get_rowblock($context->{blocksize}); @@ -1417,7 +1467,7 @@ sub _reader { $context->{queue}->enqueue(\%packet); #$packet); $blockcount++; #wait if thequeue is full and there there is one running consumer - while (((($state = _get_other_threads_state($context->{errorstates},$tid)) & $RUNNING) == $RUNNING) and $context->{queue}->pending() >= $context->{threadqueuelength}) { + while (((($state = _get_other_threads_state($context->{errorstates},$tid)) & $RUNNING) == $RUNNING) and $context->{queue}->pending() >= get_threadqueuelength($context->{threadqueuelength})) { #yield(); sleep($thread_sleep_secs); } @@ -1432,10 +1482,11 @@ sub _reader { last; } } - if (not (($state & $RUNNING) == $RUNNING and ($state & $ERROR) == 0)) { + if (not (($state & $RUNNING) == $RUNNING and ($state & $ERROR) == 0 and ($state & $STOP) == 0)) { tablethreadingdebug('[' . $tid . '] reader thread is shutting down (' . - (($state & $RUNNING) == $RUNNING ? 'still running consumer threads' : 'no running consumer threads') . ', ' . - (($state & $ERROR) == 0 ? 'no defunct thread(s)' : 'defunct thread(s)') . ') ...' + (($state & $RUNNING) == $RUNNING ? 'still running consumer threads' : uc('no running consumer threads')) . ', ' . + (($state & $ERROR) == 0 ? 'no defunct thread(s)' : uc('defunct thread(s)')) . ', ' . + (($state & $STOP) == 0 ? 'no thread(s) stopping by signal' : uc('thread(s) stopping by signal')) . ') ...' ,getlogger(__PACKAGE__)); } $reader_db->db_finish() if $reader_db->rowblock_transactional; @@ -1453,7 +1504,7 @@ sub _reader { lock $context->{errorstates}; if ($@) { $context->{errorstates}->{$tid} = $ERROR; - } else { + } elsif ($context->{errorstates}->{$tid} != $STOP) { $context->{errorstates}->{$tid} = $COMPLETED; } return $context->{errorstates}->{$tid}; @@ -1475,7 +1526,7 @@ sub _writer { my $blockcount = 0; eval { - $writer_db = &{$context->{get_target_db}}(); #$writer_connection_name); + $writer_db = &{$context->{get_target_db}}($writer_name); #$writer_name); while (not _get_stop_consumer_thread($context,$tid)) { my $packet = $context->{queue}->dequeue_nb(); if (defined $packet) { @@ -1508,7 +1559,7 @@ sub _writer { lock $context->{errorstates}; if ($@) { $context->{errorstates}->{$tid} = $ERROR; - } else { + } elsif ($context->{errorstates}->{$tid} != $STOP) { $context->{errorstates}->{$tid} = $COMPLETED; } return $context->{errorstates}->{$tid}; @@ -1534,7 +1585,7 @@ sub _process { if (defined $context->{init_process_context_code} and 'CODE' eq ref $context->{init_process_context_code}) { &{$context->{init_process_context_code}}($context); } - #$writer_db = &{$context->{get_target_db}}($writer_connection_name); + #$writer_db = &{$context->{get_target_db}}($writer_name); while (not _get_stop_consumer_thread($context,$tid)) { my $packet = $context->{queue}->dequeue_nb(); if (defined $packet) { @@ -1548,7 +1599,7 @@ sub _process { #$i += $realblocksize; - processing_rows($tid,$packet->{row_offset},$packet->{size},$context->{rowcount},getlogger(__PACKAGE__)); + processing_rows($context,$packet->{row_offset},$packet->{size},$context->{rowcount},getlogger(__PACKAGE__)); $rowblock_result = &{$context->{process_code}}($context,$packet->{rows},$packet->{row_offset}); @@ -1582,7 +1633,7 @@ sub _process { lock $context->{errorstates}; if ($err) { $context->{errorstates}->{$tid} = $ERROR; - } else { + } elsif ($context->{errorstates}->{$tid} != $STOP) { $context->{errorstates}->{$tid} = $COMPLETED; #(not $rowblock_result) ? $ERROR : $COMPLETED; } return $context->{errorstates}->{$tid}; diff --git a/lib/NGCP/BulkProcessor/SqlRecord.pm b/lib/NGCP/BulkProcessor/SqlRecord.pm index 1685d7f..5dbcfd0 100644 --- a/lib/NGCP/BulkProcessor/SqlRecord.pm +++ b/lib/NGCP/BulkProcessor/SqlRecord.pm @@ -11,6 +11,8 @@ use NGCP::BulkProcessor::SqlProcessor qw(init_record); use NGCP::BulkProcessor::Utils qw(load_module); +use NGCP::BulkProcessor::Closure qw(is_code); + require Exporter; our @ISA = qw(Exporter); our @EXPORT_OK = qw(); @@ -66,24 +68,25 @@ sub load_relation { $transform = $include->{transform}; if (exists $include->{include}) { $include = $include->{include}; + } elsif (exists $include->{load}) { + $include = $include->{load}; } elsif ($transform or $filter) { $include = 1; } } - if (('CODE' eq ref $include and $include->($self)) + if ((is_code($include) and $self->_calc_closure($relation_path,'load',$include,$load_recursive->{_context},$load_recursive->{_cache},$self)) or (not ref $include and $include)) { load_module($findby); no strict "refs"; ## no critic (ProhibitNoStrict) $load_recursive->{_relation_path} = $relation_path; $self->{$relation} = $findby->(@findby_args); if ('ARRAY' eq ref $self->{$relation} - and 'CODE' eq ref $filter) { - my $closure = _closure($filter,$load_recursive->{_context}); - $self->{$relation} = [ grep { $closure->($_); } @{$self->{$relation}}]; + and is_code($filter)) { + my $cache = $load_recursive->{_cache} // {}; + $self->{$relation} = [ grep { $self->_calc_closure($relation_path,'filter',$filter,$load_recursive->{_context},$cache,$_,$self); } @{$self->{$relation}} ]; } - if ('CODE' eq ref $transform) { - my $closure = _closure($transform,$load_recursive->{_context}); - $self->{$relation} = $closure->($self->{$relation}); + if (is_code($transform)) { + $self->{$relation} = $self->_calc_closure($relation_path,'transform',$transform,$load_recursive->{_context},$load_recursive->{_cache},$self->{$relation},$self); } $load_recursive->{_relation_path} = $relation_path_backup; return 1; @@ -92,15 +95,15 @@ sub load_relation { return 0; } -sub _closure { - my ($sub,$context) = @_; - return sub { - foreach my $key (keys %$context) { - no strict "refs"; ## no critic (ProhibitNoStrict) - *{"main::$key"} = $context->{$key} if 'CODE' eq ref $context->{$key}; - } - return $sub->(@_,$context); - }; +sub _calc_closure { + + my $self = shift; + my ($relation_path,$func,$code,$context,$cache,@args) = @_; + my $id = '_relations_' . $func . '_' . $relation_path; + $cache //= {}; + $cache->{$id} = NGCP::BulkProcessor::Closure->new($code,$context,"relations '$relation_path' $func'") unless exists $cache->{$id}; + return $cache->{$id}->calc($context,@args); + } 1; diff --git a/lib/NGCP/BulkProcessor/Utils.pm b/lib/NGCP/BulkProcessor/Utils.pm index 2027c21..9a6fe6a 100644 --- a/lib/NGCP/BulkProcessor/Utils.pm +++ b/lib/NGCP/BulkProcessor/Utils.pm @@ -117,6 +117,8 @@ our @EXPORT_OK = qw( run load_module + + is_in_eval ); our $chmod_umask = 0777; @@ -143,7 +145,7 @@ sub round { sub stringtobool { my $inputstring = shift; - if (lc($inputstring) eq 'y' or lc($inputstring) eq 'true' or $inputstring >= 1) { + if (lc($inputstring) eq 'y' or lc($inputstring) eq 'yes' or lc($inputstring) eq 'true' or lc($inputstring) eq 'on' or $inputstring >= 1) { return 1; } else { return 0; @@ -572,11 +574,12 @@ sub fixdirpath { } sub makepath { - my ($dirpath,$fileerrorcode,$logger) = @_; + my ($dirpath,$fileerrorcode,$logger,%opts) = @_; #print $chmod_umask ."\n"; #changemod($dirpath); + %opts = ('chmod' => $chmod_umask,) unless scalar keys %opts; make_path($dirpath,{ - 'chmod' => $chmod_umask, + %opts, 'verbose' => 1, 'error' => \my $err }); if (@$err) { @@ -1108,6 +1111,7 @@ sub run { } sub load_module { + my $package_element = shift; eval { (my $module = $package_element) =~ s/::[a-zA-Z_0-9]+$//g; @@ -1118,6 +1122,22 @@ sub load_module { } or do { die($@); }; + +} + + sub is_in_eval{ + + my $i=0; + while(1) { + my ($package, $filename, $line, $subroutine, $hasargs, $wantarray, $evaltext, $is_require, $hints, $bitmask, $hinthash) = caller($i); + last unless defined $package; + $i++; + if ($subroutine eq "(eval)" || $evaltext) { + return 1; + } + }; + return 0; + } 1;