From 80e2be033a1143df5022084ceefbdc3b157d690a Mon Sep 17 00:00:00 2001 From: Richard Fuchs Date: Wed, 9 Dec 2015 08:36:05 -0500 Subject: [PATCH] MT#14459 implement legacy export options Change-Id: I90503a88e0ea5a6ed35fd08e526beb70129963cf --- NGCP/CDR/Export.pm | 23 ++++++++++++-- NGCP/CDR/Exporter.pm | 74 ++++++++++++++++++++++++++++++-------------- cdr-exporter.pl | 23 +++++++++++--- 3 files changed, 89 insertions(+), 31 deletions(-) diff --git a/NGCP/CDR/Export.pm b/NGCP/CDR/Export.pm index c5b08fe..baa9e26 100644 --- a/NGCP/CDR/Export.pm +++ b/NGCP/CDR/Export.pm @@ -1,6 +1,8 @@ package NGCP::CDR::Export; use Digest::MD5; +use warnings; +use strict; our $reseller_id_col = 'contract_id'; @@ -107,6 +109,7 @@ sub chownmod { sub write_file { my ( $lines, $dircomp, $prefix, $version, $ts, $lastseq, $suffix, + $format, $file_data, ) = @_; my $fn = sprintf('%s/%s_%s_%s_%010i.%s', $dircomp, $prefix, $version, $ts, $lastseq, $suffix); @@ -116,16 +119,30 @@ sub write_file { my $ctx = Digest::MD5->new; my $num = @{ $lines }; - unshift(@{ $lines }, sprintf('%s,%04i', $version, $num)); + if ($format eq 'kabelplus') { + unshift(@{ $lines }, "'$num'". ',' x 15 ."'hdr',,,'".($$file_data[0]//'')."','".($$file_data[1]//'')."',,,'".($$file_data[2]//'')."'," . + "'".($$file_data[3]//'')."','".($$file_data[4]//'')."'". ',' x 10); + } + else { + unshift(@{ $lines }, sprintf('%s,%04i', $version, $num)); + } + + my $nl = "\n"; + $format eq 'kabelplus' and $nl = "\r\n"; for my $l (@{ $lines }) { - my $ol = "$l\n"; + my $ol = "$l$nl"; print $fd ($ol); $ctx->add($ol); } my $md5 = $ctx->hexdigest; - print $fd ("$md5\n"); + if ($format eq 'kabelplus') { + print $fd (",,'$md5'". ',' x 13 ."'md5'". ',' x 19 . "$nl"); + } + else { + print $fd ("$md5$nl"); + } print("### $num data lines written to $tfn, checksum is $md5\n"); close($fd) or die ("failed to close tmp-file $tfn ($!), stop"); diff --git a/NGCP/CDR/Exporter.pm b/NGCP/CDR/Exporter.pm index 2692b96..a2f4d7b 100644 --- a/NGCP/CDR/Exporter.pm +++ b/NGCP/CDR/Exporter.pm @@ -23,8 +23,10 @@ BEGIN { our $debug = 0; our $collid = "exporter"; +my $last_admin_field; our @admin_fields; our @reseller_fields; +our @data_fields; my @joins; my @conditions; my $dbh; @@ -33,11 +35,13 @@ my $sth; my %reseller_names; my %reseller_ids; my %reseller_lines; +my %reseller_file_data; my %mark; my $dname; my $tempdir; my $file_ts; my @reseller_positions; +my @data_positions; # default config values my %config = ( @@ -73,6 +77,18 @@ sub config2array { return $val; } +sub get_config_fields { + my ($name) = @_; + my @ret; + foreach my $f(config2array($name)) { + $f or next; + $f =~ s/^#.+//; next unless($f); + $f =~ s/^\'//; $f =~ s/\'$//; + push @ret, $f; + } + return @ret; +} + sub get_config { my ($coll, $cf, $conf_upd) = @_; @@ -101,16 +117,9 @@ sub get_config { die "Invalid destination directory '".$config{'default.DESTDIR'}."'\n" unless(-d $config{'default.DESTDIR'}); - foreach my $f(config2array('ADMIN_EXPORT_FIELDS')) { - $f =~ s/^#.+//; next unless($f); - $f =~ s/^\'//; $f =~ s/\'$//; - push @admin_fields, $f; - } - foreach my $f(config2array('RESELLER_EXPORT_FIELDS')) { - $f =~ s/^#.+//; next unless($f); - $f =~ s/^\'//; $f =~ s/\'$//; - push @reseller_fields, $f; - } + @admin_fields = get_config_fields('ADMIN_EXPORT_FIELDS'); + @reseller_fields = get_config_fields('RESELLER_EXPORT_FIELDS'); + @data_fields = get_config_fields('DATA_FIELDS'); foreach my $f(@{confval('EXPORT_JOINS')}) { $f =~ s/^\s*\{?\s*//; $f =~ s/\}\s*\}\s*$/}/; @@ -143,6 +152,25 @@ sub confval { return $config{'default.' . $val}; } +sub extract_field_positions { + my (@fields) = @_; + # extract positions of data fields from admin fields + my %index; + my @positions; + @index{@admin_fields} = (0..$#admin_fields); + for(my $i = 0; $i < @fields; $i++) { + my $name = $fields[$i]; + if (! exists $index{$name}) { + push(@admin_fields, $name); + push(@positions, $#admin_fields); + } + else { + push @positions, $index{$name}; + } + } + return @positions; +}; + sub prepare_dbh { my ($trailer, $table) = @_; @@ -172,16 +200,9 @@ sub prepare_dbh { push @trail, "$key $val"; } - # extract positions of reseller fields from admin fields - my %reseller_index; - @reseller_index{@admin_fields} = (0..$#admin_fields); - for(my $i = 0; $i < @reseller_fields; $i++) { - my $name = $reseller_fields[$i]; - unless(exists $reseller_index{$name}) { - die "Invalid RESELLER_EXPORT_FIELDS element '$name', not available in ADMIN_EXPORT_FIELDS!"; - } - push @reseller_positions, $reseller_index{$name}; - } + $last_admin_field = $#admin_fields; + @reseller_positions = extract_field_positions(@reseller_fields); + @data_positions = extract_field_positions(@data_fields); $q = "select " . join(", ", @admin_fields) . " from $table " . @@ -223,24 +244,27 @@ sub run { my $sth = $dbh->prepare($q); $sth->execute(); while(my $row = $sth->fetchrow_arrayref) { + my @admin_row = @$row[0 .. $last_admin_field]; my @res_row = @$row[@reseller_positions]; - $cb->($row, \@res_row); + my @data_row = @$row[@data_positions]; + $cb->(\@admin_row, \@res_row, \@data_row); } } sub write_reseller { - my ($reseller, $line) = @_; + my ($reseller, $line, $callback, $callback_arg) = @_; push(@{$reseller_lines{$reseller}}, $line); + $callback and $callback->($callback_arg, \$reseller_file_data{$reseller}); write_wrap($reseller); } sub write_reseller_id { - my ($id, $line) = @_; + my ($id, $line, $callback, $callback_arg) = @_; if(!exists $reseller_names{$id}) { $reseller_names{$id} = NGCP::CDR::Export::get_reseller_name($dbh, $id); $reseller_ids{$reseller_names{$id}} = $id; } - write_reseller($reseller_names{$id}, $line); + write_reseller($reseller_names{$id}, $line, $callback, $callback_arg); } sub write_wrap { @@ -286,8 +310,10 @@ sub write_wrap { NGCP::CDR::Export::write_file( \@filevals, $reseller_tempdir, confval('PREFIX'), confval('VERSION'), $file_ts, $file_idx, confval('SUFFIX'), + confval('FILE_FORMAT') // 'default', $reseller_file_data{$reseller}, ); $rec_idx -= $recs; + delete($reseller_file_data{$reseller}); } while($rec_idx > 0); diff --git a/cdr-exporter.pl b/cdr-exporter.pl index 12062f0..d1485b1 100755 --- a/cdr-exporter.pl +++ b/cdr-exporter.pl @@ -49,8 +49,23 @@ my @ids; +sub filestats_callback { + my ($data_row, $ref) = @_; + + my $out = $$ref || [0, 0, 0, 0, 0]; + for my $i (0 .. 4) { $$data_row[$i] //= 0 } + + ($$data_row[0] lt $$out[0] || !$$out[0]) and $$out[0] = $$data_row[0]; # min call start + $$out[1] += $$data_row[1]; # sum duration + ($$data_row[0] gt $$out[2] || !$$out[2]) and $$out[2] = $$data_row[0]; # max call start + $$out[3] += $$data_row[2]; # sum carrier cost + $$out[4] += $$data_row[3]; # sum customer cost + + $$ref = $out; +} + sub callback { - my ($row, $res_row) = @_; + my ($row, $res_row, $data_row) = @_; my $quotes = NGCP::CDR::Exporter::confval('QUOTES'); my @fields = @{ $row }; my $id = shift @fields; @@ -66,13 +81,13 @@ sub callback { } my $line = join ",", @fields; - write_reseller('system', $line); + write_reseller('system', $line, \&filestats_callback, $data_row); push(@ids, $id); my $reseller_line = join ",", map { defined $_ ? $quotes . $_ . $quotes : $quotes. $quotes } (@$res_row); if($src_uuid ne "0") { - write_reseller_id($src_provid, $reseller_line); + write_reseller_id($src_provid, $reseller_line, \&filestats_callback, $data_row); } if($dst_uuid ne "0") { if(confval('EXPORT_INCOMING') eq "no" && $src_provid ne $dst_provid) { @@ -81,7 +96,7 @@ sub callback { if ($src_uuid ne '0' && $src_provid eq $dst_provid) { # skip duplicate entries } else { - write_reseller_id($dst_provid, $reseller_line); + write_reseller_id($dst_provid, $reseller_line, \&filestats_callback, $data_row); } } }