sipwise
/
bulk-processor
mirror of https://github.com/sipwise/bulk-processor.git
1
0
Fork 0
Code Issues Releases Wiki Activity

TT#51110 ama: "skipped" extended cdr export status

Browse Source 
(cherry picked from commit 6d36444184)

Change-Id: I4ee3c9226c848dec44e1d1933b4647a8412fd623
changes/13/26813/1
Rene Krenn 7 years ago
parent 8e0d25df2f
commit 1fba08b9e9
14 changed files with 170 additions and 83 deletions
Show Stats Download Patch File Download Diff File

18
lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr.pm
Unescape View File

@ -382,6 +382,24 @@ sub buildrecords_fromrows {
}
sub is_xfer {
my $self = shift;
if (length($self->{call_id}) > length($XFERSUFFIX)
and substr($self->{call_id},-1 * length($XFERSUFFIX)) eq $XFERSUFFIX) {
return 1;
}
return 0;
}
sub is_pbx {
my $self = shift;
if (length($self->{call_id}) > length($PBXSUFFIX)
and substr($self->{call_id},-1 * length($PBXSUFFIX)) eq $PBXSUFFIX) {
return 1;
}
return 0;
}
sub gettablename {
return $tablename;

2
lib/NGCP/BulkProcessor/Dao/Trunk/accounting/cdr_export_status_data.pm
Unescape View File

@ -39,10 +39,12 @@ our @EXPORT_OK = qw(
$UNEXPORTED
$EXPORTED
$SKIPPED
);
our $UNEXPORTED = 'unexported';
our $EXPORTED = 'exported';
our $SKIPPED = 'skipped';
my $tablename = 'cdr_export_status_data';
my $get_db = \&get_accounting_db;

24
lib/NGCP/BulkProcessor/Projects/Export/Ama/CDR.pm → lib/NGCP/BulkProcessor/Projects/Export/Ama/Ccs/CDR.pm
Unescape View File

@ -1,11 +1,11 @@
package NGCP::BulkProcessor::Projects::Export::Ama::CDR;
package NGCP::BulkProcessor::Projects::Export::Ama::Ccs::CDR;
use strict;
## no critic
use threads::shared qw();
use NGCP::BulkProcessor::Projects::Export::Ama::Settings qw(
use NGCP::BulkProcessor::Projects::Export::Ama::Ccs::Settings qw(
$skip_errors
@ -279,15 +279,27 @@ sub _export_cdrs_init_context {
my $call_id_prefix = NGCP::BulkProcessor::Dao::Trunk::accounting::cdr::get_callidprefix($call_id);
if (exists $context->{block_call_id_map}->{$call_id_prefix}) {
$context->{cdrs} = NGCP::BulkProcessor::Dao::Trunk::accounting::cdr::findby_callidprefix($context->{db},
$call_id,$export_cdr_joins,$export_cdr_conditions);
$call_id,$export_cdr_joins,$export_cdr_conditions); #already sorted
my $cdrs_in_block = delete $context->{block_call_id_map}->{$call_id_prefix};
if ((scalar @{$context->{cdrs}}) == $cdrs_in_block) {
if ((scalar @{$context->{cdrs}}) == 2
and not $context->{cdrs}->[0]->is_xfer()
and $context->{cdrs}->[1]->is_xfer()
) {
$result = 1;
} else {
print "blah";
}
foreach my $cdr (@{$context->{cdrs}}) {
if ($result) {
$cdr->{_extended_export_status} = $NGCP::BulkProcessor::Dao::Trunk::accounting::cdr_export_status_data::EXPORTED;
} else {
$cdr->{_extended_export_status} = $NGCP::BulkProcessor::Dao::Trunk::accounting::cdr_export_status_data::SKIPPED;
}
$context->{file_cdr_id_map}->{$cdr->{id}} = $cdr; #->{start_time};
lock $rowcount;
$rowcount += 1;
}
$result = 1;
}
}
}
@ -321,10 +333,10 @@ sub _commit_export_status {
NGCP::BulkProcessor::Dao::Trunk::accounting::cdr_export_status_data::upsert_row($context->{db},
cdr_id => $id,
status_id => $context->{export_status_id},
export_status => $NGCP::BulkProcessor::Dao::Trunk::accounting::cdr_export_status_data::EXPORTED,
export_status => $context->{file_cdr_id_map}->{$id}->{_extended_export_status},
cdr_start_time => $context->{file_cdr_id_map}->{$id}->{start_time},
);
_info($context,"export_status set for cdr id $id",1);
_info($context,"export_status '$context->{file_cdr_id_map}->{$id}->{_extended_export_status}' set for cdr id $id",1);
}
NGCP::BulkProcessor::Dao::Trunk::accounting::mark::insert_system_mark($context->{db},
$export_cdr_stream,

58
lib/NGCP/BulkProcessor/Projects/Export/Ama/Settings.pm → lib/NGCP/BulkProcessor/Projects/Export/Ama/Ccs/Settings.pm
Unescape View File

@ -1,4 +1,4 @@
package NGCP::BulkProcessor::Projects::Export::Ama::Settings;
package NGCP::BulkProcessor::Projects::Export::Ama::Ccs::Settings;
use strict;
## no critic
@ -27,7 +27,7 @@ use NGCP::BulkProcessor::LoadConfig qw(
split_tuple
parse_regexp
);
use NGCP::BulkProcessor::Utils qw(prompt timestampdigits stringtobool);
use NGCP::BulkProcessor::Utils qw(prompt timestampdigits); #stringtobool
#format_number check_ipnet
require Exporter;
@ -36,8 +36,6 @@ our @EXPORT_OK = qw(
update_settings
$input_path
$output_path
$tempfile_path
$defaultsettings
$defaultconfig
@ -54,25 +52,13 @@ our @EXPORT_OK = qw(
$export_cdr_limit
$export_cdr_stream
$export_cdr_rollover_fsn
$export_cdr_use_temp_files
$domestic_destination_pattern
$international_destination_pattern
$ama_filename_format
);
#check_dry
#$dry
#update_provider_config
#$deadlock_retries
#$generate_cdr_count
our $defaultconfig = 'config.cfg';
our $defaultsettings = 'settings.cfg';
our $input_path = $working_path . 'input/';
our $output_path = $working_path . 'output/';
our $tempfile_path = $working_path . 'temp/';
our $force = 0;
#our $dry = 0;
@ -86,12 +72,6 @@ our $export_cdr_conditions = [];
our $export_cdr_limit = undef;
our $export_cdr_stream = undef;
our $export_cdr_rollover_fsn = 0;
our $export_cdr_use_temp_files = 0;
our $domestic_destination_pattern = undef;
our $international_destination_pattern = undef;
our $ama_filename_format = '%1$s%2$s%3$s';
sub update_settings {
@ -101,20 +81,8 @@ sub update_settings {
my $result = 1;
#&$configurationinfocode("testinfomessage",$configlogger);
$result &= _prepare_working_paths(1);
#if ($data->{report_filename}) {
# $report_filename = $output_path . sprintf('/' . $data->{report_filename},timestampdigits());
# if (-e $report_filename and (unlink $report_filename) == 0) {
# filewarn('cannot remove ' . $report_filename . ': ' . $!,getlogger(__PACKAGE__));
# $report_filename = undef;
# }
#} else {
# $report_filename = undef;
#}
#$dry = $data->{dry} if exists $data->{dry};
$skip_errors = $data->{skip_errors} if exists $data->{skip_errors};
$export_cdr_multithreading = $data->{export_cdr_multithreading} if exists $data->{export_cdr_multithreading};
@ -129,21 +97,7 @@ sub update_settings {
$export_cdr_limit = $data->{export_cdr_limit} if exists $data->{export_cdr_limit};
$export_cdr_stream = $data->{export_cdr_stream} if exists $data->{export_cdr_stream};
$export_cdr_rollover_fsn = stringtobool($data->{export_cdr_rollover_fsn}) if exists $data->{export_cdr_rollover_fsn};
$export_cdr_use_temp_files = stringtobool($data->{export_cdr_use_temp_files}) if exists $data->{export_cdr_use_temp_files};
#if ((confval("MAINTENANCE") // 'no') eq 'yes') {
# exit(0);
#}
my $regexp_result;
$domestic_destination_pattern = $data->{domestic_destination_pattern} if exists $data->{domestic_destination_pattern};
($regexp_result,$domestic_destination_pattern) = parse_regexp($domestic_destination_pattern,$configfile);
$result &= $regexp_result;
$international_destination_pattern = $data->{international_destination_pattern} if exists $data->{international_destination_pattern};
($regexp_result,$international_destination_pattern) = parse_regexp($international_destination_pattern,$configfile);
$result &= $regexp_result;
$ama_filename_format = $data->{ama_filename_format} if exists $data->{ama_filename_format};
$export_cdr_rollover_fsn = $data->{export_cdr_rollover_fsn} if exists $data->{export_cdr_rollover_fsn};
return $result;
@ -160,10 +114,6 @@ sub _prepare_working_paths {
($path_result,$input_path) = create_path($working_path . 'input',$input_path,$create,\&fileerror,getlogger(__PACKAGE__));
$result &= $path_result;
($path_result,$output_path) = create_path($working_path . 'output',$output_path,$create,\&fileerror,getlogger(__PACKAGE__));
$result &= $path_result;
($path_result,$tempfile_path) = create_path($working_path . 'temp',$output_path,$create,\&fileerror,getlogger(__PACKAGE__));
$result &= $path_result;
return $result;

0
lib/NGCP/BulkProcessor/Projects/Export/Ama/config.cfg → lib/NGCP/BulkProcessor/Projects/Export/Ama/Ccs/config.cfg
Unescape View File

0
lib/NGCP/BulkProcessor/Projects/Export/Ama/config.debug.cfg → lib/NGCP/BulkProcessor/Projects/Export/Ama/Ccs/config.debug.cfg
Unescape View File

13
lib/NGCP/BulkProcessor/Projects/Export/Ama/process.pl → lib/NGCP/BulkProcessor/Projects/Export/Ama/Ccs/process.pl
Unescape View File

@ -4,21 +4,21 @@ use strict;
use File::Basename;
use Cwd;
use lib Cwd::abs_path(File::Basename::dirname(__FILE__) . '/../../../../../');
use lib Cwd::abs_path(File::Basename::dirname(__FILE__) . '/../../../../../../');
use Getopt::Long qw(GetOptions);
use Fcntl qw(LOCK_EX LOCK_NB);
use NGCP::BulkProcessor::Globals qw();
use NGCP::BulkProcessor::Projects::Export::Ama::Settings qw(
update_settings
use NGCP::BulkProcessor::Projects::Export::Ama::Format::Settings qw(
$output_path
$tempfile_path
);
use NGCP::BulkProcessor::Projects::Export::Ama::Ccs::Settings qw(
$defaultsettings
$defaultconfig
$skip_errors
$force
);
#$dry
#check_dry
@ -66,7 +66,7 @@ use NGCP::BulkProcessor::ConnectorPool qw(destroy_dbs);
#use NGCP::BulkProcessor::Dao::Trunk::billing::voip_subscribers qw();
#use NGCP::BulkProcessor::Dao::Trunk::provisioning::voip_dbaliases qw();
use NGCP::BulkProcessor::Projects::Export::Ama::CDR qw(
use NGCP::BulkProcessor::Projects::Export::Ama::Ccs::CDR qw(
export_cdrs
reset_fsn
reset_export_status
@ -121,7 +121,8 @@ sub init {
my $result = load_config($configfile);
init_log();
$result &= load_config($settingsfile,\&update_settings,$SIMPLE_CONFIG_TYPE);
$result &= load_config($settingsfile,\&NGCP::BulkProcessor::Projects::Export::Ama::Format::Settings::update_settings,$SIMPLE_CONFIG_TYPE);
$result &= load_config($settingsfile,\&NGCP::BulkProcessor::Projects::Export::Ama::Ccs::Settings::update_settings,$SIMPLE_CONFIG_TYPE);
#$result &= load_config($providers_yml,\&update_provider_config,$YAML_CONFIG_TYPE);
return $result;

4
lib/NGCP/BulkProcessor/Projects/Export/Ama/settings.cfg → lib/NGCP/BulkProcessor/Projects/Export/Ama/Ccs/settings.cfg
Unescape View File

@ -7,8 +7,8 @@ export_cdr_joins = { 'billing.voip_subscribers source_voip_subscribers' => { 'so
export_cdr_conditions =
#{ 'accounting.cdr.export_status' => { '=' => '"unexported"' } }, { 'accounting.cdr.call_status' => { '=' => '"ok"' } }, { 'accounting.cdr.rating_status' => { '=' => '"ok"' } }
export_cdr_limit = 10000
export_cdr_stream = ama_simple
export_cdr_stream = ama_ccs
export_cdr_rollover_fsn = 1
export_cdr_use_temp_files = 1
use_tempfiles = 1
ama_filename_format = %1$sP%3$02d%4$02d%5$02d%6$02d%7$02d%9$02dAMA%10$s

4
lib/NGCP/BulkProcessor/Projects/Export/Ama/settings.debug.cfg → lib/NGCP/BulkProcessor/Projects/Export/Ama/Ccs/settings.debug.cfg
Unescape View File

@ -8,10 +8,10 @@ export_cdr_blocksize = 1000
#, { 'accounting.cdr.rating_status' => { '=' => '"ok"' } }
export_cdr_conditions =
#{ 'accounting.cdr.call_status' => { '=' => '"ok"' } }
export_cdr_stream = ama_simple
export_cdr_stream = ama_ccs
#default
export_cdr_limit = 10000
export_cdr_rollover_fsn = 1
export_cdr_use_temp_files = 1
use_tempfiles = 1
ama_filename_format = %1$sP%3$02d%4$02d%5$02d%6$02d%7$02d%9$02dAMA.debug%10$s

0
lib/NGCP/BulkProcessor/Projects/Export/Ama/test.ama → lib/NGCP/BulkProcessor/Projects/Export/Ama/Ccs/test.ama
Unescape View File

4
lib/NGCP/BulkProcessor/Projects/Export/Ama/test.pl → lib/NGCP/BulkProcessor/Projects/Export/Ama/Ccs/test.pl
Unescape View File

@ -4,9 +4,9 @@ use strict;
use File::Basename;
use Cwd;
use lib Cwd::abs_path(File::Basename::dirname(__FILE__) . '/../../../../../');
use lib Cwd::abs_path(File::Basename::dirname(__FILE__) . '/../../../../../../');
use NGCP::BulkProcessor::Projects::Export::Ama::Settings qw(
use NGCP::BulkProcessor::Projects::Export::Ama::Ccs::Settings qw(
update_settings
);

2
lib/NGCP/BulkProcessor/Projects/Export/Ama/Format/Fields/DomesticInternational.pm
Unescape View File

@ -3,7 +3,7 @@ use strict;
## no critic
use NGCP::BulkProcessor::Projects::Export::Ama::Settings qw(
use NGCP::BulkProcessor::Projects::Export::Ama::Format::Settings qw(
$domestic_destination_pattern
$international_destination_pattern
);

20
lib/NGCP/BulkProcessor/Projects/Export/Ama/Format/File.pm
Unescape View File

@ -3,10 +3,10 @@ use strict;
## no critic
use NGCP::BulkProcessor::Projects::Export::Ama::Settings qw(
use NGCP::BulkProcessor::Projects::Export::Ama::Format::Settings qw(
$output_path
$ama_filename_format
$export_cdr_use_temp_files
$use_tempfiles
$tempfile_path
);
@ -49,7 +49,7 @@ sub reset {
$self->{record_count} = 0;
$self->_save_transfer_in(undef);
$self->_save_transfer_out(undef);
$self->{tempfilename} = tempfilename('XXXX',$tempfile_path,$ama_file_extension) if $export_cdr_use_temp_files;
$self->{tempfilename} = tempfilename('XXXX',$tempfile_path,$ama_file_extension) if $use_tempfiles;
return;
}
@ -94,7 +94,7 @@ sub add_record {
sub get_filename {
my $self = shift;
my ($show_tempfilename) = @_;
return $self->{tempfilename} if ($export_cdr_use_temp_files and $show_tempfilename);
return $self->{tempfilename} if ($use_tempfiles and $show_tempfilename);
return sprintf($ama_filename_format,
$output_path,
$self->{transfer_in}->get_structure()->get_date_field()->{dt}->year,
@ -111,7 +111,7 @@ sub get_filename {
sub get_filesize {
my $self = shift;
return -s ($export_cdr_use_temp_files ? $self->{tempfilename} : $self->get_filename());
return -s ($use_tempfiles ? $self->{tempfilename} : $self->get_filename());
}
sub _rename {
@ -134,14 +134,14 @@ sub flush {
fileerror($filename . ' already exists',getlogger(__PACKAGE__));
return 0;
} else {
if (open(my $fh,">:raw",($export_cdr_use_temp_files ? $self->{tempfilename} : $filename))) {
if (open(my $fh,">:raw",($use_tempfiles ? $self->{tempfilename} : $filename))) {
foreach my $block (@{$self->{blocks}}) {
print $fh pack('H*',$block->get_hex());
}
close $fh;
if (defined $commit_cb) {
if (&$commit_cb(@_)) {
if (not $export_cdr_use_temp_files or $self->_rename($filename)) {
if (not $use_tempfiles or $self->_rename($filename)) {
return 1;
} else {
my $err = $!;
@ -153,8 +153,8 @@ sub flush {
}
} else {
eval {
unlink $filename unless $export_cdr_use_temp_files;
unlink $self->{tempfilename} if $export_cdr_use_temp_files;
unlink $filename unless $use_tempfiles;
unlink $self->{tempfilename} if $use_tempfiles;
};
return 0;
}
@ -163,7 +163,7 @@ sub flush {
}
#restdebug($self,"$self->{crt_path} saved",getlogger(__PACKAGE__));
} else {
fileerror('failed to open ' . ($export_cdr_use_temp_files ? $self->{tempfilename} : $filename) . ": $!",getlogger(__PACKAGE__));
fileerror('failed to open ' . ($use_tempfiles ? $self->{tempfilename} : $filename) . ": $!",getlogger(__PACKAGE__));
return 0;
}
}

104
lib/NGCP/BulkProcessor/Projects/Export/Ama/Format/Settings.pm
Unescape View File

@ -0,0 +1,104 @@
package NGCP::BulkProcessor::Projects::Export::Ama::Format::Settings;
use strict;
## no critic
use NGCP::BulkProcessor::Globals qw(
$working_path
$enablemultithreading
$cpucount
create_path
);
use NGCP::BulkProcessor::Logging qw(
getlogger
scriptinfo
configurationinfo
);
use NGCP::BulkProcessor::LogError qw(
fileerror
filewarn
configurationwarn
configurationerror
);
use NGCP::BulkProcessor::LoadConfig qw(
split_tuple
parse_regexp
);
use NGCP::BulkProcessor::Utils qw(prompt timestampdigits); #stringtobool
#format_number check_ipnet
require Exporter;
our @ISA = qw(Exporter);
our @EXPORT_OK = qw(
update_settings
$output_path
$tempfile_path
$domestic_destination_pattern
$international_destination_pattern
$ama_filename_format
$use_tempfiles
);
our $output_path = $working_path . 'output/';
our $tempfile_path = $working_path . 'temp/';
our $use_tempfiles = 0;
our $domestic_destination_pattern = undef;
our $international_destination_pattern = undef;
our $ama_filename_format = '%1$sP%3$02d%4$02d%5$02d%6$02d%7$02d%9$02dAMA%10$s';
sub update_settings {
my ($data,$configfile) = @_;
if (defined $data) {
my $result = 1;
#&$configurationinfocode("testinfomessage",$configlogger);
$result &= _prepare_working_paths(1);
$use_tempfiles = $data->{use_tempfiles} if exists $data->{use_tempfiles};
my $regexp_result;
$domestic_destination_pattern = $data->{domestic_destination_pattern} if exists $data->{domestic_destination_pattern};
($regexp_result,$domestic_destination_pattern) = parse_regexp($domestic_destination_pattern,$configfile);
$result &= $regexp_result;
$international_destination_pattern = $data->{international_destination_pattern} if exists $data->{international_destination_pattern};
($regexp_result,$international_destination_pattern) = parse_regexp($international_destination_pattern,$configfile);
$result &= $regexp_result;
$ama_filename_format = $data->{ama_filename_format} if exists $data->{ama_filename_format};
return $result;
}
return 0;
}
sub _prepare_working_paths {
my ($create) = @_;
my $result = 1;
my $path_result;
($path_result,$output_path) = create_path($working_path . 'output',$output_path,$create,\&fileerror,getlogger(__PACKAGE__));
$result &= $path_result;
($path_result,$tempfile_path) = create_path($working_path . 'temp',$output_path,$create,\&fileerror,getlogger(__PACKAGE__));
$result &= $path_result;
return $result;
}
1;
Write Preview
Loading…
Cancel
Save