sipwise
/
cdr-exporter
mirror of https://github.com/sipwise/cdr-exporter.git
1
0
Fork 0
Code Issues Releases Wiki Activity

MT#8969 Use Config::Simple to manage configs

Browse Source 
fix perl warnings (both perl files):

Stricture disabled at line [37|73], column 9. See page 429 of PBP. (Severity: 5)

Change-Id: I2ffef64fef4e68ac9305e29da6c0743f00c25c1f
changes/60/160/7
Victor Seva 12 years ago
parent 2c31646433
commit f8a5e3a7e8
4 changed files with 91 additions and 109 deletions
Show Stats Download Patch File Download Diff File

97
cdr-exporter.pl
Unescape View File

@ -3,44 +3,21 @@
use strict;
use warnings;
use Config::Simple;
use DBI;
use Digest::MD5;
our $DBHOST;
our $DBUSER;
our $DBPASS;
our $DBDB;
our $CDRDIR;
our $PREFIX;
our $VERSION;
our $DAILY_DIR;
our $MONTHLY_DIR;
our $FULL_NAMES;
our $EXPORT_UNRATED;
our $EXPORT_INCOMING;
our $EXPORT_FAILED;
our $FILES_OWNER = 'cdrexport';
our $FILES_GROUP = 'cdrexport';
our $FILES_MASK = '022';
# default config values
my $config = {
'default.FILES_OWNER' => 'cdrexport',
'default.FILES_GROUP' => 'cdrexport',
'default.FILES_MASK' => '022'
};
my $config_file = "/etc/ngcp-cdr-exporter/cdr-exporter.conf";
open my $CONFIG, '<', "$config_file" or die "Program stopping, couldn't open the configuration file '$config_file'.\n";
while (<$CONFIG>) {
chomp; # no newline
s/#.*//; # no comments
s/^\s+//; # no leading white
s/\s+$//; # no trailing white
next unless length; # anything left?
my ($var, $value) = split(/\s*=\s*/, $_, 2);
no strict 'refs';
$$var = $value;
}
close $CONFIG;
Config::Simple->import_from("$config_file" , \%{$config}) or
die "Program stopping, couldn't open the configuration file '$config_file'.\n";
sub chownmod {
@ -58,10 +35,13 @@ sub chownmod {
my $DBH = DBI->connect("dbi:mysql:$DBDB;host=$DBHOST", $DBUSER, $DBPASS);
my $DBH = DBI->connect("dbi:mysql:" . $config->{'default.DBDB'} .
";host=".$config->{'default.DBHOST'},
$config->{'default.DBUSER'}, $config->{'default.DBPASS'});
$DBH or return 0;
print("+++ Start run with DB " . ($DBUSER || "(undef)") . "\@$DBDB to $PREFIX\n");
print("+++ Start run with DB " . ($config->{'default.DBUSER'} || "(undef)") .
"\@".$config->{'default.DBDB'}." to ".$config->{'default.PREFIX'}."\n");
my $COLLID = "exporter";
my %MARKS; # last seq etc
@ -133,23 +113,33 @@ my @CDR_RESELLER_BODY_FIELDS = qw(
{
my ($dir1, $dir2, $ts);
$ts = sprintf('%04i%02i%02i%02i%02i%02i', $NOW[5] + 1900, $NOW[4] + 1, @NOW[3,2,1,0]);
$FULL_NAMES = ($FULL_NAMES && $FULL_NAMES =~ /1|y/i);
$config->{'default.FULL_NAMES'} = ($config->{'default.FULL_NAMES'} &&
$config->{'default.FULL_NAMES'} =~ /1|y/i);
if ($MONTHLY_DIR && $MONTHLY_DIR =~ /1|y/i) {
if ($config->{'default.MONTHLY_DIR'} &&
$config->{'default.MONTHLY_DIR'} =~ /1|y/i)
{
$dir1 = sprintf('%04i%02i', $NOW[5] + 1900, $NOW[4] + 1);
if ($DAILY_DIR && $DAILY_DIR =~ /1|y/i) {
if ($config->{'default.DAILY_DIR'} &&
$config->{'default.DAILY_DIR'} =~ /1|y/i)
{
$dir2 = sprintf('%02i', $NOW[3]);
$FULL_NAMES or $ts = sprintf('%02i%02i%02i', @NOW[2,1,0]);
$config->{'default.FULL_NAMES'} or
$ts = sprintf('%02i%02i%02i', @NOW[2,1,0]);
}
else {
$dir2 = '.';
$FULL_NAMES or $ts = sprintf('%02i%02i%02i%02i', @NOW[3,2,1,0]);
$config->{'default.FULL_NAMES'} or
$ts = sprintf('%02i%02i%02i%02i', @NOW[3,2,1,0]);
}
}
elsif ($DAILY_DIR && $DAILY_DIR =~ /1|y/i) {
elsif ($config->{'default.DAILY_DIR'} &&
$config->{'default.DAILY_DIR'} =~ /1|y/i)
{
$dir1 = sprintf('%04i%02i%02i', $NOW[5] + 1900, $NOW[4] + 1, $NOW[3]);
$dir2 = '.';
$FULL_NAMES or $ts = sprintf('%02i%02i%02i', @NOW[2,1,0]);
$config->{'default.FULL_NAMES'} or
$ts = sprintf('%02i%02i%02i', @NOW[2,1,0]);
}
else {
$dir1 = $dir2 = '.';
@ -201,8 +191,8 @@ my @CDR_RESELLER_BODY_FIELDS = qw(
LEFT JOIN billing.voip_subscribers source_bvs ON cdr.source_user_id = source_bvs.uuid
LEFT JOIN billing.voip_subscribers destination_bvs ON cdr.destination_user_id = destination_bvs.uuid
where cdr.export_status = 'unexported' AND cdr.id > ?
". ($EXPORT_INCOMING eq 'yes' ? '' : "and source_user_id != '0'") ."
". ($EXPORT_FAILED eq 'yes' ? '' : "and call_status = 'ok'") ."
". ($config->{'default.EXPORT_INCOMING'} eq 'yes' ? '' : "and source_user_id != '0'") ."
". ($config->{'default.EXPORT_FAILED'} eq 'yes' ? '' : "and call_status = 'ok'") ."
order by
cdr.id
limit $limit
@ -214,7 +204,7 @@ my @CDR_RESELLER_BODY_FIELDS = qw(
while (my $r = $s->fetchrow_hashref()) {
# finish export to give rate-o-mat time to catch up
if ($r->{rating_status} eq 'unrated') {
last if $EXPORT_UNRATED !~ /y|1|true/i;
last if $config->{'default.EXPORT_UNRATED'} !~ /y|1|true/i;
}
else {
unless(defined $r->{source_carrier_zone}) { # platform internal, no peering cost calculated
@ -270,9 +260,9 @@ my @CDR_RESELLER_BODY_FIELDS = qw(
my ($f, @dirs) = @$ref;
my $num = scalar(@$f);
unshift(@$f, sprintf('%s,%04i', $VERSION, $num));
unshift(@$f, sprintf('%s,%04i', $config->{'default.VERSION'}, $num));
my $dircomp = $CDRDIR;
my $dircomp = $config->{'default.CDRDIR'};
my @dirlist;
for my $dirpart (@dirs, $dir1, $dir2) {
$dircomp .= "/$dirpart";
@ -282,11 +272,16 @@ my @CDR_RESELLER_BODY_FIELDS = qw(
for my $dd (@dirlist) {
if (! -d $dd) {
mkdir($dd) or die("failed to create target directory $dd ($!), stop");
chownmod($dd, $FILES_OWNER, $FILES_GROUP, '0777', $FILES_MASK);
chownmod($dd, $config->{'default.FILES_OWNER'},
$config->{'default.FILES_GROUP'}, '0777', $config->{'default.FILES_MASK'});
}
}
my $fn = sprintf('%s/%s_%s_%s_%010i.cdr', $dircomp, $PREFIX, $VERSION, $ts, $MARKS{lastseq});
my $tfn = sprintf('%s/%s_%s_%s_%010i.cdr.'.$$, $dircomp, $PREFIX, $VERSION, $ts, $MARKS{lastseq});
my $fn = sprintf('%s/%s_%s_%s_%010i.cdr', $dircomp,
$config->{'default.PREFIX'}, $config->{'default.VERSION'},
$ts, $MARKS{lastseq});
my $tfn = sprintf('%s/%s_%s_%s_%010i.cdr.'.$$, $dircomp,
$config->{'default.PREFIX'}, $config->{'default.VERSION'},
$ts, $MARKS{lastseq});
my $fd;
open($fd, ">", $tfn) or die("failed to open tmp-file $tfn ($!), stop");
my $ctx = Digest::MD5->new;
@ -306,7 +301,9 @@ my @CDR_RESELLER_BODY_FIELDS = qw(
rename($tfn, $fn) or die("failed to move tmp-file $tfn to $fn ($!), stop");
print("### successfully moved $tfn to $fn\n");
chownmod($fn, $FILES_OWNER, $FILES_GROUP, '0666', $FILES_MASK);
chownmod($fn, $config->{'default.FILES_OWNER'},
$config->{'default.FILES_GROUP'},
'0666', $config->{'default.FILES_MASK'});
}
# update exported cdrs

3
debian/control vendored
Unescape View File

@ -8,7 +8,8 @@ Homepage: http://sipwise.com/
Package: ngcp-cdr-exporter
Architecture: all
Depends: libclass-dbi-mysql-perl,
Depends: libconfig-simple-perl,
libclass-dbi-mysql-perl,
libio-pty-perl,
libnet-sftp-foreign-perl,
perl-modules,

4
event-exporter.conf
Unescape View File

@ -29,5 +29,5 @@ EXPORT_FIELDS = 'accounting.events.id', 'accounting.events.type', 'billing.contr
EXPORT_JOINS = { 'billing.voip_subscribers' => { 'billing.voip_subscribers.id' => 'accounting.events.subscriber_id' } }, { 'provisioning.voip_subscribers' => { 'provisioning.voip_subscribers.uuid' => 'billing.voip_subscribers.uuid' } }, { 'billing.contracts' => { 'billing.contracts.id' => 'billing.voip_subscribers.contract_id' } }, { 'billing.contacts' => { 'billing.contacts.id' => 'billing.contracts.contact_id' } }, { '(select * from provisioning.voip_dbaliases order by is_primary asc, id asc) as voip_numbers_tmp' => { 'provisioning.voip_subscribers.id' => 'voip_numbers_tmp.subscriber_id' } }, { 'provisioning.voip_subscriber_profiles as old_profile' => { 'old_profile.id' => 'accounting.events.old_status' } }, { 'provisioning.voip_subscriber_profiles as new_profile' => { 'new_profile.id' => 'accounting.events.new_status' } }
EXPORT_CONDITIONS = { 'accounting.events.export_status' => { '=' => '"unexported"' } }
# important last comma if there is just one
EXPORT_CONDITIONS = { 'accounting.events.export_status' => { '=' => '"unexported"' } },

96
event-exporter.pl
Unescape View File

@ -2,43 +2,29 @@
use strict;
use v5.14;
use Config::Simple;
use DBI;
use File::Temp;
use File::Copy;
use NGCP::CDR::Export;
use NGCP::CDR::Transfer;
our $DBHOST;
our $DBUSER;
our $DBPASS;
our $DBDB;
our $MAX_ROWS_PER_FILE;
our $EDRDIR;
our $FILTER_FLAPPING = 0;
our $PREFIX = 'sipwise';
our $VERSION = '001';
our $SUFFIX = 'edr';
our $FILES_OWNER = 'cdrexport';
our $FILES_GROUP = 'cdrexport';
our $FILES_MASK = '022';
our $TRANSFER_TYPE = "none";
our $TRANSFER_HOST;
our $TRANSFER_PORT = 22;
our $TRANSFER_USER = "cdrexport";
our $TRANSFER_PASS;
our $TRANSFER_REMOTE = "/home/jail/home/cdrexport";
our $EXPORT_FIELDS;
our $EXPORT_JOINS;
our $EXPORT_CONDITIONS;
my $collid = "eventexporter";
my $debug = 0;
# default config values
my $config = {
'default.FILTER_FLAPPING' => 0,
'default.PREFIX' => 'sipwise',
'default.VERSION' => '001',
'default.SUFFIX' => 'edr',
'default.FILES_OWNER' => 'cdrexport',
'default.FILES_GROUP' => 'cdrexport',
'default.FILES_MASK' => '022',
'default.TRANSFER_TYPE' => "none",
'default.TRANSFER_PORT' => 22,
'default.TRANSFER_USER' => "cdrexport",
'default.TRANSFER_REMOTE' => "/home/jail/home/cdrexport"
};
sub DEBUG {
say join (' ', @_);
@ -61,32 +47,22 @@ foreach my $cp(@config_paths) {
die "Config file $cf not found in path " . (join " or ", @config_paths) . "\n"
unless $config_file;
open my $CONFIG, '<', "$config_file" or die "Couldn't open the configuration file '$config_file'.\n";
while (<$CONFIG>) {
chomp; # no newline
s/^\s+//; # no leading white
s/^#.*//; # no comments
s/\s+$//; # no trailing white
next unless length; # anything left?
my ($var, $value) = split(/\s*=\s*/, $_, 2);
no strict 'refs';
$$var = $value;
}
close $CONFIG;
Config::Simple->import_from("$config_file" , \%{$config}) or
die "Couldn't open the configuration file '$config_file'.\n";
die "Invalid destination directory '$EDRDIR'\n"
unless(-d $EDRDIR);
die "Invalid destination directory '".$config->{'default.EDRDIR'}."'\n"
unless(-d $config->{'default.EDRDIR'});
my @fields = ();
foreach my $f(split('\'\s*,\s*\#?\s*\'', $EXPORT_FIELDS)) {
foreach my $f(@{$config->{'default.EXPORT_FIELDS'}}) {
$f =~ s/^#.+//; next unless($f);
$f =~ s/^\'//; $f =~ s/\'$//;
push @fields, $f;
}
my @joins = ();
foreach my $f(split('\}\s*,\s*{', $EXPORT_JOINS)) {
foreach my $f(@{$config->{'default.EXPORT_JOINS'}}) {
$f =~ s/^\s*\{?\s*//; $f =~ s/\}\s*\}\s*$/}/;
my ($a, $b) = split('\s*=>\s*{\s*', $f);
$a =~ s/^\s*\'//; $a =~ s/\'$//g;
@ -99,7 +75,8 @@ foreach my $f(split('\}\s*,\s*{', $EXPORT_JOINS)) {
}
my @conditions = ();
foreach my $f(split('\}\s*,\s*{', $EXPORT_CONDITIONS)) {
foreach my $f(@{$config->{'default.EXPORT_CONDITIONS'}}) {
next unless($f);
$f =~ s/^\s*\{?\s*//; $f =~ s/\}\s*\}\s*$/}/;
my ($a, $b) = split('\s*=>\s*{\s*', $f);
$a =~ s/^\s*\'//; $a =~ s/\'$//g;
@ -111,7 +88,8 @@ foreach my $f(split('\}\s*,\s*{', $EXPORT_CONDITIONS)) {
push @conditions, { $a => { $c => $d } };
}
my $dbh = DBI->connect('DBI:mysql:'.$DBDB, $DBUSER, $DBPASS)
my $dbh = DBI->connect('DBI:mysql:'.$config->{'default.DBDB'},
$config->{'default.DBUSER'}, $config->{'default.DBPASS'})
or die "failed to connect to db: $DBI::errstr";
$dbh->{mysql_auto_reconnect} = 1;
$dbh->{AutoCommit} = 0;
@ -174,7 +152,7 @@ while(my $row = shift @{ $rows }) {
my ($id, $sub_id, $type, $old, $new) = @head;
my @fields = map { defined $_ ? "\"$_\"" : '""' } (@{ $row }[5 .. @{ $row }-1]);
if($FILTER_FLAPPING) {
if($config->{'default.FILTER_FLAPPING'}) {
if($type =~ /^start_(.+)$/) {
my $t = $1;
my $k = "$sub_id;$t;$new";
@ -217,7 +195,7 @@ while(my $row = shift @{ $rows }) {
my @vals = map { $lines{$_} } sort { int($a) <=> int($b) } keys %lines;
my @ids = keys %lines;
my $max = $MAX_ROWS_PER_FILE // $rec_idx;
my $max = $config->{'default.MAX_ROWS_PER_FILE'} // $rec_idx;
do {
my $recs = ($rec_idx > $max) ? $max : $rec_idx;
@ -225,7 +203,8 @@ do {
my @filevals = @vals[0 .. $recs-1];
@vals = @vals[$recs .. @vals-1];
NGCP::CDR::Export::write_file(
\@filevals, $tempdir, $PREFIX, $VERSION, $file_ts, $file_idx, $SUFFIX,
\@filevals, $tempdir, $config->{'default.PREFIX'},
$config->{'default.VERSION'}, $file_ts, $file_idx, $config->{'default.SUFFIX'},
);
$rec_idx -= $recs;
@ -240,15 +219,20 @@ $dbh->commit or die("failed to commit db changes: " . $dbh->errstr);
opendir(my $fh, $tempdir);
foreach my $file(readdir($fh)) {
my $src = "$tempdir/$file";
my $dst = "$EDRDIR/$file";
my $dst = $config->{'default.EDRDIR'}."/$file";
if(-f $src) {
DEBUG "### moving $src to $dst\n";
copy($src, $dst);
NGCP::CDR::Export::chownmod($dst, $FILES_OWNER, $FILES_GROUP, '0666', $FILES_MASK);
if($TRANSFER_TYPE eq "sftp") {
NGCP::CDR::Export::chownmod($dst, $config->{'default.FILES_OWNER'},
$config->{'default.FILES_GROUP'}, '0666',
$config->{'default.FILES_MASK'});
if($config->{'default.TRANSFER_TYPE'} eq "sftp") {
NGCP::CDR::Transfer::sftp(
$dst, $TRANSFER_HOST, $TRANSFER_PORT,
$TRANSFER_REMOTE, $TRANSFER_USER, $TRANSFER_PASS,
$dst, $config->{'default.TRANSFER_HOST'},
$config->{'default.TRANSFER_PORT'},
$config->{'default.TRANSFER_REMOTE'},
$config->{'default.TRANSFER_USER'},
$config->{'default.TRANSFER_PASS'},
);
}

Write Preview
Loading…
Cancel
Save