sipwise
/
ngcp-panel
mirror of https://github.com/sipwise/ngcp-panel.git
1
0
Fork 0
Code Issues Releases Wiki Activity

MT#7227 Upload billing_fees into 2 stages, for speed and unique records

Browse Source 
Change-Id: Ie3b4711fa1d31cc900d997e1863800777e526464
changes/21/1821/8
Irina Peshinskaya 11 years ago
parent 718113a04f
commit 2e2c008542
3 changed files with 120 additions and 101 deletions
Show Stats Download Patch File Download Diff File

64
lib/NGCP/Panel/Controller/API/BillingFees.pm
Unescape View File

@ -9,7 +9,7 @@ use HTTP::Status qw(:constants);
use MooseX::ClassAttribute qw(class_has);
use NGCP::Panel::Utils::DateTime;
use Path::Tiny qw(path);
use Text::CSV_XS;
use NGCP::Panel::Utils::Billing;
BEGIN { extends 'Catalyst::Controller::ActionRole'; }
require Catalyst::ActionRole::ACL;
require Catalyst::ActionRole::CheckTrailingSlash;
@ -141,12 +141,11 @@ sub OPTIONS :Allow {
sub POST :Allow {
my ($self, $c) = @_;
my $guard = $c->model('DB')->txn_scope_guard;
{
my $schema = $c->model('DB');
my $resource;
my $data= $self->get_valid_raw_post_data(
my $data = $self->get_valid_raw_post_data(
c => $c,
media_type => [qw#application/json text/csv#],
);
@ -180,60 +179,23 @@ sub POST :Allow {
}
if ($data) {
# csv bulk upload
my $csv = Text::CSV_XS->new({allow_whitespace => 1, binary => 1, keep_meta_info => 1});
my @cols = @{ $c->config->{fees_csv}->{element_order} };
if ($resource->{purge_existing}) {
$profile->billing_fees->delete;
$profile->billing_fees_raw->delete;
}
my @fails = ();
my $linenum = 0;
my @fees = ();
my %zones = ();
try {
foreach my $line(split /\r?\n/, $data) {
++$linenum;
chomp $line;
next unless length $line;
unless($csv->parse($line)) {
push @fails, $linenum;
next;
}
my $row = {};
my @fields = $csv->fields();
unless (scalar @fields == scalar @cols) {
push @fails, $linenum;
next;
}
for(my $i = 0; $i < @cols; ++$i) {
$row->{$cols[$i]} = $fields[$i];
}
my $k = $row->{zone}.'__NGCP__'.$row->{zone_detail};
unless(exists $zones{$k}) {
my $zone = $profile->billing_zones->find_or_create({
zone => $row->{zone},
detail => $row->{zone_detail}
});
$zones{$k} = $zone->id;
}
$row->{billing_zone_id} = $zones{$k};
delete $row->{zone};
delete $row->{zone_detail};
push @fees, $row;
}
$profile->billing_fees->populate(\@fees);
my $text = $c->loc('Billing Fee successfully uploaded');
if(@fails) {
$text .= $c->loc(", but skipped the following line numbers: ") . (join ", ", @fails);
}
$c->log->info($text);
(my($fees, $fails, $text_success)) = NGCP::Panel::Utils::Billing::process_billing_fees(
c => $c,
data => \$data,
profile => $profile,
schema => $schema,
);
$c->log->info( $$text_success );
$guard->commit;
$c->response->status(HTTP_CREATED);
$c->response->body(q());

68
lib/NGCP/Panel/Controller/Billing.pm
Unescape View File

@ -1,6 +1,5 @@
package NGCP::Panel::Controller::Billing;
use Sipwise::Base;
use Text::CSV_XS;
use DateTime::Format::ISO8601;
BEGIN { extends 'Catalyst::Controller'; }
@ -18,6 +17,7 @@ use NGCP::Panel::Utils::Navigation;
use NGCP::Panel::Utils::Datatables;
use NGCP::Panel::Utils::Preferences;
use NGCP::Panel::Utils::DateTime;
use NGCP::Panel::Utils::Billing;
sub auto :Does(ACL) :ACLDetachTo('/denied_page') :AllowedRole(admin) :AllowedRole(reseller) {
my ($self, $c) = @_;
@ -417,9 +417,7 @@ sub fees_upload :Chained('fees_list') :PathPart('upload') :Args(0) {
my $form = NGCP::Panel::Form::BillingFeeUpload->new;
my $upload = $c->req->upload('upload_fees');
my $posted = $c->req->method eq 'POST';
my @params = (
upload_fees => $posted ? $upload : undef,
);
my @params = ( upload_fees => $posted ? $upload : undef, );
$form->process(
posted => $posted,
params => { @params },
@ -430,62 +428,32 @@ sub fees_upload :Chained('fees_list') :PathPart('upload') :Args(0) {
# TODO: check by formhandler?
unless($upload) {
NGCP::Panel::Utils::Message->error(
c => $c,
c => $c,
desc => $c->loc('No Billing Fee file specified!'),
);
$c->response->redirect($c->uri_for($c->stash->{profile}->{id}, 'fees'));
return;
}
my $csv = Text::CSV_XS->new({allow_whitespace => 1, binary => 1, keep_meta_info => 1});
my @cols = $c->config->{fees_csv}->{element_order};
$csv->column_names (@cols);
if ($c->req->params->{purge_existing}) {
$c->stash->{'profile_result'}->billing_fees_raw->delete;
$c->stash->{'profile_result'}->billing_fees->delete;
}
my @fails = ();
my $linenum = 0;
my @fees = ();
my %zones = ();
my $data = $upload->slurp;
my($fees, $fails, $text_success);
try {
$c->model('DB')->txn_do(sub {
while(my $row = $csv->getline_hr($upload->fh)) {
++$linenum;
if($csv->is_missing(1)) {
push @fails, $linenum;
next;
}
my $k = $row->{zone}.'__NGCP__'.$row->{zone_detail};
unless(exists $zones{$k}) {
my $zone = $c->stash->{'profile_result'}
->billing_zones
->find_or_create({
zone => $row->{zone},
detail => $row->{zone_detail}
});
$zones{$k} = $zone->id;
}
$row->{billing_zone_id} = $zones{$k};
delete $row->{zone};
delete $row->{zone_detail};
push @fees, $row;
}
unless ($csv->eof()) {
die "Some lines could not be parsed. Did not reach eof. Last successful: $linenum.";
}
$c->stash->{'profile_result'}
->billing_fees->populate(\@fees);
my $schema = $c->model('DB');
$schema->txn_do(sub {
( $fees, $fails, $text_success ) = NGCP::Panel::Utils::Billing::process_billing_fees(
c => $c,
data => \$data,
profile => $c->stash->{'profile_result'},
schema => $schema,
);
});
my $text = $c->loc('Billing Fee successfully uploaded');
if(@fails) {
$text .= $c->loc(", but skipped the following line numbers: ") . (join ", ", @fails);
}
NGCP::Panel::Utils::Message->info(
c => $c,
desc => $text,
c => $c,
desc => $$text_success,
);
} catch($e) {
NGCP::Panel::Utils::Message->error(
@ -493,7 +461,7 @@ sub fees_upload :Chained('fees_list') :PathPart('upload') :Args(0) {
error => $e,
desc => $c->loc('Failed to upload Billing Fees'),
);
};
}
$c->response->redirect($c->uri_for($c->stash->{profile}->{id}, 'fees'));
return;

89
lib/NGCP/Panel/Utils/Billing.pm
Unescape View File

@ -0,0 +1,89 @@
package NGCP::Panel::Utils::Billing;
use strict;
use warnings;
use Text::CSV_XS;
sub process_billing_fees{
my(%params) = @_;
my ($c,$data,$profile,$schema) = @params{qw/c data profile schema/};
# csv bulk upload
my $csv = Text::CSV_XS->new({ allow_whitespace => 1, binary => 1, keep_meta_info => 1 });
my @cols = @{ $c->config->{fees_csv}->{element_order} };
my @fields ;
my @fails = ();
my $linenum = 0;
my @fees = ();
my %zones = ();
open my $fh, '<', $data;
while ( my $line = <$fh> ){
++$linenum;
chomp $line;
next unless length $line;
unless($csv->parse($line)) {
push @fails, $linenum;
next;
}
@fields = $csv->fields();
unless (scalar @fields == scalar @cols) {
push @fails, $linenum;
next;
}
my $row = {};
@{$row}{@cols} = @fields;
my $k = $row->{zone}.'__NGCP__'.$row->{zone_detail};
unless(exists $zones{$k}) {
my $zone = $profile->billing_zones->find_or_create({
zone => $row->{zone},
detail => $row->{zone_detail}
});
$zones{$k} = $zone->id;
}
$row->{billing_zone_id} = $zones{$k};
delete $row->{zone};
delete $row->{zone_detail};
push @fees, $row;
}
$profile->billing_fees_raw->populate(\@fees);
$schema->storage->dbh_do(sub{
my ($storage, $dbh) = @_;
$dbh->do("call billing.fill_billing_fees(?)", undef, $profile->id );
});
my $text = $c->loc('Billing Fee successfully uploaded');
if(@fails) {
$text .= $c->loc(", but skipped the following line numbers: ") . (join ", ", @fails);
}
return ( \@fees, \@fails, \$text );
}
1;
=head1 NAME
NGCP::Panel::Utils::Billing
=head1 DESCRIPTION
A temporary helper to manipulate billing plan related data
=head1 METHODS
=head2 process_billing_fees
Parse billing fees uploaded csv
=head1 AUTHOR
Irina Peshinskaya
=head1 LICENSE
This library is free software. You can redistribute it and/or modify
it under the same terms as Perl itself.
=cut
# vim: set tabstop=4 expandtab:
Write Preview
Loading…
Cancel
Save