From f8a5e3a7e8bf4e386ca9befe11ced7fccc446ffd Mon Sep 17 00:00:00 2001 From: Victor Seva Date: Fri, 5 Sep 2014 09:24:11 +0200 Subject: [PATCH] MT#8969 Use Config::Simple to manage configs fix perl warnings (both perl files): Stricture disabled at line [37|73], column 9. See page 429 of PBP. (Severity: 5) Change-Id: I2ffef64fef4e68ac9305e29da6c0743f00c25c1f --- cdr-exporter.pl | 97 ++++++++++++++++++++++----------------------- debian/control | 3 +- event-exporter.conf | 4 +- event-exporter.pl | 96 +++++++++++++++++++------------------------- 4 files changed, 91 insertions(+), 109 deletions(-) diff --git a/cdr-exporter.pl b/cdr-exporter.pl index e02d923..5b30d7d 100755 --- a/cdr-exporter.pl +++ b/cdr-exporter.pl @@ -3,44 +3,21 @@ use strict; use warnings; +use Config::Simple; use DBI; use Digest::MD5; -our $DBHOST; -our $DBUSER; -our $DBPASS; -our $DBDB; -our $CDRDIR; -our $PREFIX; -our $VERSION; -our $DAILY_DIR; -our $MONTHLY_DIR; -our $FULL_NAMES; -our $EXPORT_UNRATED; -our $EXPORT_INCOMING; -our $EXPORT_FAILED; -our $FILES_OWNER = 'cdrexport'; -our $FILES_GROUP = 'cdrexport'; -our $FILES_MASK = '022'; +# default config values +my $config = { + 'default.FILES_OWNER' => 'cdrexport', + 'default.FILES_GROUP' => 'cdrexport', + 'default.FILES_MASK' => '022' +}; my $config_file = "/etc/ngcp-cdr-exporter/cdr-exporter.conf"; -open my $CONFIG, '<', "$config_file" or die "Program stopping, couldn't open the configuration file '$config_file'.\n"; - -while (<$CONFIG>) { - chomp; # no newline - s/#.*//; # no comments - s/^\s+//; # no leading white - s/\s+$//; # no trailing white - next unless length; # anything left? - my ($var, $value) = split(/\s*=\s*/, $_, 2); - no strict 'refs'; - $$var = $value; -} -close $CONFIG; - - - +Config::Simple->import_from("$config_file" , \%{$config}) or + die "Program stopping, couldn't open the configuration file '$config_file'.\n"; sub chownmod { @@ -58,10 +35,13 @@ sub chownmod { -my $DBH = DBI->connect("dbi:mysql:$DBDB;host=$DBHOST", $DBUSER, $DBPASS); +my $DBH = DBI->connect("dbi:mysql:" . $config->{'default.DBDB'} . + ";host=".$config->{'default.DBHOST'}, + $config->{'default.DBUSER'}, $config->{'default.DBPASS'}); $DBH or return 0; -print("+++ Start run with DB " . ($DBUSER || "(undef)") . "\@$DBDB to $PREFIX\n"); +print("+++ Start run with DB " . ($config->{'default.DBUSER'} || "(undef)") . + "\@".$config->{'default.DBDB'}." to ".$config->{'default.PREFIX'}."\n"); my $COLLID = "exporter"; my %MARKS; # last seq etc @@ -133,23 +113,33 @@ my @CDR_RESELLER_BODY_FIELDS = qw( { my ($dir1, $dir2, $ts); $ts = sprintf('%04i%02i%02i%02i%02i%02i', $NOW[5] + 1900, $NOW[4] + 1, @NOW[3,2,1,0]); - $FULL_NAMES = ($FULL_NAMES && $FULL_NAMES =~ /1|y/i); + $config->{'default.FULL_NAMES'} = ($config->{'default.FULL_NAMES'} && + $config->{'default.FULL_NAMES'} =~ /1|y/i); - if ($MONTHLY_DIR && $MONTHLY_DIR =~ /1|y/i) { + if ($config->{'default.MONTHLY_DIR'} && + $config->{'default.MONTHLY_DIR'} =~ /1|y/i) + { $dir1 = sprintf('%04i%02i', $NOW[5] + 1900, $NOW[4] + 1); - if ($DAILY_DIR && $DAILY_DIR =~ /1|y/i) { + if ($config->{'default.DAILY_DIR'} && + $config->{'default.DAILY_DIR'} =~ /1|y/i) + { $dir2 = sprintf('%02i', $NOW[3]); - $FULL_NAMES or $ts = sprintf('%02i%02i%02i', @NOW[2,1,0]); + $config->{'default.FULL_NAMES'} or + $ts = sprintf('%02i%02i%02i', @NOW[2,1,0]); } else { $dir2 = '.'; - $FULL_NAMES or $ts = sprintf('%02i%02i%02i%02i', @NOW[3,2,1,0]); + $config->{'default.FULL_NAMES'} or + $ts = sprintf('%02i%02i%02i%02i', @NOW[3,2,1,0]); } } - elsif ($DAILY_DIR && $DAILY_DIR =~ /1|y/i) { + elsif ($config->{'default.DAILY_DIR'} && + $config->{'default.DAILY_DIR'} =~ /1|y/i) + { $dir1 = sprintf('%04i%02i%02i', $NOW[5] + 1900, $NOW[4] + 1, $NOW[3]); $dir2 = '.'; - $FULL_NAMES or $ts = sprintf('%02i%02i%02i', @NOW[2,1,0]); + $config->{'default.FULL_NAMES'} or + $ts = sprintf('%02i%02i%02i', @NOW[2,1,0]); } else { $dir1 = $dir2 = '.'; @@ -201,8 +191,8 @@ my @CDR_RESELLER_BODY_FIELDS = qw( LEFT JOIN billing.voip_subscribers source_bvs ON cdr.source_user_id = source_bvs.uuid LEFT JOIN billing.voip_subscribers destination_bvs ON cdr.destination_user_id = destination_bvs.uuid where cdr.export_status = 'unexported' AND cdr.id > ? - ". ($EXPORT_INCOMING eq 'yes' ? '' : "and source_user_id != '0'") ." - ". ($EXPORT_FAILED eq 'yes' ? '' : "and call_status = 'ok'") ." + ". ($config->{'default.EXPORT_INCOMING'} eq 'yes' ? '' : "and source_user_id != '0'") ." + ". ($config->{'default.EXPORT_FAILED'} eq 'yes' ? '' : "and call_status = 'ok'") ." order by cdr.id limit $limit @@ -214,7 +204,7 @@ my @CDR_RESELLER_BODY_FIELDS = qw( while (my $r = $s->fetchrow_hashref()) { # finish export to give rate-o-mat time to catch up if ($r->{rating_status} eq 'unrated') { - last if $EXPORT_UNRATED !~ /y|1|true/i; + last if $config->{'default.EXPORT_UNRATED'} !~ /y|1|true/i; } else { unless(defined $r->{source_carrier_zone}) { # platform internal, no peering cost calculated @@ -270,9 +260,9 @@ my @CDR_RESELLER_BODY_FIELDS = qw( my ($f, @dirs) = @$ref; my $num = scalar(@$f); - unshift(@$f, sprintf('%s,%04i', $VERSION, $num)); + unshift(@$f, sprintf('%s,%04i', $config->{'default.VERSION'}, $num)); - my $dircomp = $CDRDIR; + my $dircomp = $config->{'default.CDRDIR'}; my @dirlist; for my $dirpart (@dirs, $dir1, $dir2) { $dircomp .= "/$dirpart"; @@ -282,11 +272,16 @@ my @CDR_RESELLER_BODY_FIELDS = qw( for my $dd (@dirlist) { if (! -d $dd) { mkdir($dd) or die("failed to create target directory $dd ($!), stop"); - chownmod($dd, $FILES_OWNER, $FILES_GROUP, '0777', $FILES_MASK); + chownmod($dd, $config->{'default.FILES_OWNER'}, + $config->{'default.FILES_GROUP'}, '0777', $config->{'default.FILES_MASK'}); } } - my $fn = sprintf('%s/%s_%s_%s_%010i.cdr', $dircomp, $PREFIX, $VERSION, $ts, $MARKS{lastseq}); - my $tfn = sprintf('%s/%s_%s_%s_%010i.cdr.'.$$, $dircomp, $PREFIX, $VERSION, $ts, $MARKS{lastseq}); + my $fn = sprintf('%s/%s_%s_%s_%010i.cdr', $dircomp, + $config->{'default.PREFIX'}, $config->{'default.VERSION'}, + $ts, $MARKS{lastseq}); + my $tfn = sprintf('%s/%s_%s_%s_%010i.cdr.'.$$, $dircomp, + $config->{'default.PREFIX'}, $config->{'default.VERSION'}, + $ts, $MARKS{lastseq}); my $fd; open($fd, ">", $tfn) or die("failed to open tmp-file $tfn ($!), stop"); my $ctx = Digest::MD5->new; @@ -306,7 +301,9 @@ my @CDR_RESELLER_BODY_FIELDS = qw( rename($tfn, $fn) or die("failed to move tmp-file $tfn to $fn ($!), stop"); print("### successfully moved $tfn to $fn\n"); - chownmod($fn, $FILES_OWNER, $FILES_GROUP, '0666', $FILES_MASK); + chownmod($fn, $config->{'default.FILES_OWNER'}, + $config->{'default.FILES_GROUP'}, + '0666', $config->{'default.FILES_MASK'}); } # update exported cdrs diff --git a/debian/control b/debian/control index 66745d0..6561c19 100644 --- a/debian/control +++ b/debian/control @@ -8,7 +8,8 @@ Homepage: http://sipwise.com/ Package: ngcp-cdr-exporter Architecture: all -Depends: libclass-dbi-mysql-perl, +Depends: libconfig-simple-perl, + libclass-dbi-mysql-perl, libio-pty-perl, libnet-sftp-foreign-perl, perl-modules, diff --git a/event-exporter.conf b/event-exporter.conf index a8e39a0..cca854b 100644 --- a/event-exporter.conf +++ b/event-exporter.conf @@ -29,5 +29,5 @@ EXPORT_FIELDS = 'accounting.events.id', 'accounting.events.type', 'billing.contr EXPORT_JOINS = { 'billing.voip_subscribers' => { 'billing.voip_subscribers.id' => 'accounting.events.subscriber_id' } }, { 'provisioning.voip_subscribers' => { 'provisioning.voip_subscribers.uuid' => 'billing.voip_subscribers.uuid' } }, { 'billing.contracts' => { 'billing.contracts.id' => 'billing.voip_subscribers.contract_id' } }, { 'billing.contacts' => { 'billing.contacts.id' => 'billing.contracts.contact_id' } }, { '(select * from provisioning.voip_dbaliases order by is_primary asc, id asc) as voip_numbers_tmp' => { 'provisioning.voip_subscribers.id' => 'voip_numbers_tmp.subscriber_id' } }, { 'provisioning.voip_subscriber_profiles as old_profile' => { 'old_profile.id' => 'accounting.events.old_status' } }, { 'provisioning.voip_subscriber_profiles as new_profile' => { 'new_profile.id' => 'accounting.events.new_status' } } -EXPORT_CONDITIONS = { 'accounting.events.export_status' => { '=' => '"unexported"' } } - +# important last comma if there is just one +EXPORT_CONDITIONS = { 'accounting.events.export_status' => { '=' => '"unexported"' } }, diff --git a/event-exporter.pl b/event-exporter.pl index 9eb0ccd..4ec2343 100755 --- a/event-exporter.pl +++ b/event-exporter.pl @@ -2,43 +2,29 @@ use strict; use v5.14; +use Config::Simple; use DBI; use File::Temp; use File::Copy; use NGCP::CDR::Export; use NGCP::CDR::Transfer; -our $DBHOST; -our $DBUSER; -our $DBPASS; -our $DBDB; - -our $MAX_ROWS_PER_FILE; -our $EDRDIR; - -our $FILTER_FLAPPING = 0; - -our $PREFIX = 'sipwise'; -our $VERSION = '001'; -our $SUFFIX = 'edr'; -our $FILES_OWNER = 'cdrexport'; -our $FILES_GROUP = 'cdrexport'; -our $FILES_MASK = '022'; - -our $TRANSFER_TYPE = "none"; -our $TRANSFER_HOST; -our $TRANSFER_PORT = 22; -our $TRANSFER_USER = "cdrexport"; -our $TRANSFER_PASS; -our $TRANSFER_REMOTE = "/home/jail/home/cdrexport"; - -our $EXPORT_FIELDS; -our $EXPORT_JOINS; -our $EXPORT_CONDITIONS; - my $collid = "eventexporter"; my $debug = 0; - +# default config values +my $config = { + 'default.FILTER_FLAPPING' => 0, + 'default.PREFIX' => 'sipwise', + 'default.VERSION' => '001', + 'default.SUFFIX' => 'edr', + 'default.FILES_OWNER' => 'cdrexport', + 'default.FILES_GROUP' => 'cdrexport', + 'default.FILES_MASK' => '022', + 'default.TRANSFER_TYPE' => "none", + 'default.TRANSFER_PORT' => 22, + 'default.TRANSFER_USER' => "cdrexport", + 'default.TRANSFER_REMOTE' => "/home/jail/home/cdrexport" +}; sub DEBUG { say join (' ', @_); @@ -61,32 +47,22 @@ foreach my $cp(@config_paths) { die "Config file $cf not found in path " . (join " or ", @config_paths) . "\n" unless $config_file; -open my $CONFIG, '<', "$config_file" or die "Couldn't open the configuration file '$config_file'.\n"; - -while (<$CONFIG>) { - chomp; # no newline - s/^\s+//; # no leading white - s/^#.*//; # no comments - s/\s+$//; # no trailing white - next unless length; # anything left? - my ($var, $value) = split(/\s*=\s*/, $_, 2); - no strict 'refs'; - $$var = $value; -} -close $CONFIG; +Config::Simple->import_from("$config_file" , \%{$config}) or + die "Couldn't open the configuration file '$config_file'.\n"; -die "Invalid destination directory '$EDRDIR'\n" - unless(-d $EDRDIR); +die "Invalid destination directory '".$config->{'default.EDRDIR'}."'\n" + unless(-d $config->{'default.EDRDIR'}); my @fields = (); -foreach my $f(split('\'\s*,\s*\#?\s*\'', $EXPORT_FIELDS)) { +foreach my $f(@{$config->{'default.EXPORT_FIELDS'}}) { + $f =~ s/^#.+//; next unless($f); $f =~ s/^\'//; $f =~ s/\'$//; push @fields, $f; } my @joins = (); -foreach my $f(split('\}\s*,\s*{', $EXPORT_JOINS)) { +foreach my $f(@{$config->{'default.EXPORT_JOINS'}}) { $f =~ s/^\s*\{?\s*//; $f =~ s/\}\s*\}\s*$/}/; my ($a, $b) = split('\s*=>\s*{\s*', $f); $a =~ s/^\s*\'//; $a =~ s/\'$//g; @@ -99,7 +75,8 @@ foreach my $f(split('\}\s*,\s*{', $EXPORT_JOINS)) { } my @conditions = (); -foreach my $f(split('\}\s*,\s*{', $EXPORT_CONDITIONS)) { +foreach my $f(@{$config->{'default.EXPORT_CONDITIONS'}}) { + next unless($f); $f =~ s/^\s*\{?\s*//; $f =~ s/\}\s*\}\s*$/}/; my ($a, $b) = split('\s*=>\s*{\s*', $f); $a =~ s/^\s*\'//; $a =~ s/\'$//g; @@ -111,7 +88,8 @@ foreach my $f(split('\}\s*,\s*{', $EXPORT_CONDITIONS)) { push @conditions, { $a => { $c => $d } }; } -my $dbh = DBI->connect('DBI:mysql:'.$DBDB, $DBUSER, $DBPASS) +my $dbh = DBI->connect('DBI:mysql:'.$config->{'default.DBDB'}, + $config->{'default.DBUSER'}, $config->{'default.DBPASS'}) or die "failed to connect to db: $DBI::errstr"; $dbh->{mysql_auto_reconnect} = 1; $dbh->{AutoCommit} = 0; @@ -174,7 +152,7 @@ while(my $row = shift @{ $rows }) { my ($id, $sub_id, $type, $old, $new) = @head; my @fields = map { defined $_ ? "\"$_\"" : '""' } (@{ $row }[5 .. @{ $row }-1]); - if($FILTER_FLAPPING) { + if($config->{'default.FILTER_FLAPPING'}) { if($type =~ /^start_(.+)$/) { my $t = $1; my $k = "$sub_id;$t;$new"; @@ -217,7 +195,7 @@ while(my $row = shift @{ $rows }) { my @vals = map { $lines{$_} } sort { int($a) <=> int($b) } keys %lines; my @ids = keys %lines; -my $max = $MAX_ROWS_PER_FILE // $rec_idx; +my $max = $config->{'default.MAX_ROWS_PER_FILE'} // $rec_idx; do { my $recs = ($rec_idx > $max) ? $max : $rec_idx; @@ -225,7 +203,8 @@ do { my @filevals = @vals[0 .. $recs-1]; @vals = @vals[$recs .. @vals-1]; NGCP::CDR::Export::write_file( - \@filevals, $tempdir, $PREFIX, $VERSION, $file_ts, $file_idx, $SUFFIX, + \@filevals, $tempdir, $config->{'default.PREFIX'}, + $config->{'default.VERSION'}, $file_ts, $file_idx, $config->{'default.SUFFIX'}, ); $rec_idx -= $recs; @@ -240,15 +219,20 @@ $dbh->commit or die("failed to commit db changes: " . $dbh->errstr); opendir(my $fh, $tempdir); foreach my $file(readdir($fh)) { my $src = "$tempdir/$file"; - my $dst = "$EDRDIR/$file"; + my $dst = $config->{'default.EDRDIR'}."/$file"; if(-f $src) { DEBUG "### moving $src to $dst\n"; copy($src, $dst); - NGCP::CDR::Export::chownmod($dst, $FILES_OWNER, $FILES_GROUP, '0666', $FILES_MASK); - if($TRANSFER_TYPE eq "sftp") { + NGCP::CDR::Export::chownmod($dst, $config->{'default.FILES_OWNER'}, + $config->{'default.FILES_GROUP'}, '0666', + $config->{'default.FILES_MASK'}); + if($config->{'default.TRANSFER_TYPE'} eq "sftp") { NGCP::CDR::Transfer::sftp( - $dst, $TRANSFER_HOST, $TRANSFER_PORT, - $TRANSFER_REMOTE, $TRANSFER_USER, $TRANSFER_PASS, + $dst, $config->{'default.TRANSFER_HOST'}, + $config->{'default.TRANSFER_PORT'}, + $config->{'default.TRANSFER_REMOTE'}, + $config->{'default.TRANSFER_USER'}, + $config->{'default.TRANSFER_PASS'}, ); }