TT#3994 Add download csv to LNP and EmergencyMapping API

Change-Id: I517cc28dd3244e2f62b6ff3ed3f5197bb53b3561
changes/20/8520/6
Irina Peshinskaya 9 years ago
parent 6fa78d147d
commit 85e93b5d19

@ -90,8 +90,31 @@ sub auto :Private {
$self->log_request($c);
}
sub check_create_csv :Private {
my ($self, $c) = @_;
my $reseller_id = $c->request->params->{reseller_id};
if(!$reseller_id){
$self->error($c, HTTP_BAD_REQUEST, 'reseller_id parameter is necessary to download csv data.');
return;
}
return 'emergency_mapping_list_reseller_'.$reseller_id.'.csv';
}
sub create_csv :Private {
my ($self, $c) = @_;
NGCP::Panel::Utils::EmergencyMapping::create_csv(
c => $c,
reseller_id => $c->request->params->{reseller_id},
);
}
sub GET :Allow {
my ($self, $c) = @_;
my $header_accept = $c->request->header('Accept');
if(defined $header_accept && $header_accept eq 'text/csv') {
$self->return_csv($c);
return;
}
my $page = $c->request->params->{page} // 1;
my $rows = $c->request->params->{rows} // 10;
{

@ -90,8 +90,25 @@ sub auto :Private {
$self->log_request($c);
}
sub check_create_csv :Private {
my ($self, $c) = @_;
return 'lnp_list.csv';
}
sub create_csv :Private {
my ($self, $c) = @_;
NGCP::Panel::Utils::Lnp::create_csv(
c => $c,
);
}
sub GET :Allow {
my ($self, $c) = @_;
my $header_accept = $c->request->header('Accept');
if(defined $header_accept && $header_accept eq 'text/csv') {
$self->return_csv($c);
return;
}
my $page = $c->request->params->{page} // 1;
my $rows = $c->request->params->{rows} // 10;
{

@ -583,7 +583,6 @@ sub emergency_mappings_download :Chained('list') :PathPart('download') :Args(0)
$c->stash(emergency_container_create_flag => 1);
$c->stash(emergency_container_form => $form);
} else {
my $schema = $c->model('DB');
$c->response->header ('Content-Disposition' => "attachment; filename=\"emergency_mapping_list_reseller_$reseller_id.csv\"");
$c->response->content_type('text/csv');
$c->response->status(200);

@ -719,5 +719,19 @@ sub update{
return $item;
}
sub return_csv(){
my($self,$c) = @_;
try{
my $filename = $self->check_create_csv($c);
return unless $filename;
$c->response->header ('Content-Disposition' => "attachment; filename=\"$filename\"");
$c->response->content_type('text/csv');
$c->response->status(200);
$self->create_csv($c);
$c->response->body(q());
}catch($e){
$self->error($c, HTTP_BAD_REQUEST, $e);
}
}
1;
# vim: set tabstop=4 expandtab:

@ -13,109 +13,116 @@ our @EXPORT_OK = qw(get_separator);
sub get_separator {
my %options = @_;
my $file_path = $options{path};
my $c = $options{c};
# check options
my $echo;
if ($options{echo}) {
$echo = 1;
print "\nDetecting field separator of $file_path\n";
$c->log->debug("Detecting field separator of $file_path") if $c;
}
my (@excluded, @included);
if (exists $options{exclude}) {
@excluded = @{$options{exclude}};
}
if (exists $options{include}) {
@included = @{$options{include}};
}
my ($lucky, $colon_timecol, $comma_decsep, $comma_groupsep);
if (exists $options{lucky} && $options{lucky} == 1) {
$lucky = 1;
print "Scalar context...\n\n" if $echo;
$c->log->debug("Scalar context...") if $c;
} else {
$colon_timecol = $comma_decsep = $comma_groupsep = 1;
print "List context...\n\n" if $echo;
$c->log->debug("List context...") if $c;
}
# options checked
# Default set of candidates
my @candidates = (',', ';', ':', '|', "\t");
my %survivors;
$survivors{$_} = [] foreach (@candidates);
if (@excluded > 0) {
foreach (@excluded) {
delete $survivors{$_};
_message('deleted', $_) if $echo;
_message('deleted', $_, $c) if $echo;
}
}
if (@included > 0) {
foreach (@included) {
if (length($_) == 1) {
$survivors{$_} = [];
}
_message('added', $_) if $echo;
_message('added', $_, $c) if $echo;
}
}
if (keys %survivors == 0) {
carp "No candidates left!";
return;
}
my $csv;
open ($csv, "<:crlf", $file_path) || croak "Couldn't open $file_path: $!";
my $record_count = 0; # if $echo
while (<$csv>) {
my $record = $_;
chomp $record;
if ($echo) {
$record_count++;
print "\nRecord #$record_count\n";
$c->log->debug("Record #$record_count: $record...") if $c;
}
foreach my $candidate (keys %survivors) {
_message('candidate', $candidate) if $echo;
_message('candidate', $candidate, $c) if $echo;
my $rex = qr/\Q$candidate\E/;
my $count = 0;
$count++ while ($record =~ /$rex/g);
print "Count: $count\n" if $echo;
$c->log->debug("Count: $count") if $c;
if ($count > 0 && !$lucky) {
push @{$survivors{$candidate}}, $count;
} elsif ($count == 0) {
delete $survivors{$candidate};
}
}
if (!$lucky) {
$colon_timecol = _regularity($record, 'timecol') if $colon_timecol;
$comma_decsep = _regularity($record, 'decsep') if $comma_decsep;
$comma_groupsep = _regularity($record, 'groupsep') if $comma_groupsep;
}
my @alive = keys %survivors;
my $survivors_count = @alive;
if ($survivors_count == 1) {
if ($echo) {
_message('detected', $alive[0]);
_message('detected', $alive[0], $c);
print "Returning control to caller...\n\n";
$c->log->debug("Returning control to caller...") if $c;
}
close $csv;
if (!$lucky) {
@ -128,53 +135,62 @@ sub get_separator {
return;
}
}
# More than 1 survivor. 2nd pass to determine count variability
if ($lucky) {
print "\nSeveral candidates left\n" if $echo;
$c->log->debug("Several candidates left") if $c;
carp "\nBad luck. Couldn't determine the separator of $file_path.\n";
$c->log->debug("Bad luck. Couldn't determine the separator of $file_path.") if $c;
return;
} else {
print "\nVariability:\n\n" if $echo;
$c->log->debug("Variability:") if $c;
my %std_dev;
foreach my $candidate (keys %survivors) {
my $mean = _mean(@{$survivors{$candidate}});
$std_dev{$candidate} = _std_dev($mean, @{$survivors{$candidate}});
if ($echo) {
_message('candidate', $candidate);
_message('candidate', $candidate, $c);
$c->log->debug("Mean: $mean\tStd Dev: $std_dev{$candidate}") if $c;
print "Mean: $mean\tStd Dev: $std_dev{$candidate}\n\n";
}
}
$c->log->debug("Couldn't determine the separator") if $c;
print "Couldn't determine the separator\n" if $echo;
close $csv;
my @penalized;
if ($colon_timecol) {
$c->log->debug("Detected time column") if $c;
print "Detected time column\n" if $echo;
delete $survivors{':'};
push @penalized, ':';
}
if ($comma_decsep || $comma_groupsep) {
delete $survivors{','};
push @penalized, ',';
if ($echo && $comma_decsep) {
$c->log->debug("Detected comma-separated decimal numbers column") if $c;
print "\nDetected comma-separated decimal numbers column\n";
}
if ($echo && $comma_groupsep) {
$c->log->debug("Detected comma-grouped numbers column") if $c;
print "\nDetected comma-grouped numbers column\n";
}
}
my @alive = sort {$std_dev{$a} <=> $std_dev{$b}} keys %survivors;
push @alive, sort {$std_dev{$a} <=> $std_dev{$b}} @penalized;
if ($echo) {
$c->log->debug("Remaining candidates: ") if $c;
print "Remaining candidates: ";
foreach my $left (@alive) {
_message('left', $left);
_message('left', $left, $c);
}
$c->log->debug("Returning control to caller...") if $c;
print "\n\nReturning control to caller...\n\n";
}
return @alive;
@ -183,34 +199,34 @@ sub get_separator {
sub _mean {
my @array = @_;
my $sum = 0;
$sum += $_ foreach (@array);
my $mean = $sum / scalar(@array);
return $mean;
}
sub _std_dev {
my ($mean, @array) = @_;
my $sum = 0;
$sum += ($_ - $mean)**2 foreach (@array);
my $std_dev = sqrt( $sum / scalar(@array) );
return $std_dev;
}
sub _regularity {
my ($string, $kind) = @_;
my $time_rx = qr/
(?:^|(?<=\s|[T,;|\t]))
(?:[01]?[0-9]|2[0-3]) # hours
:
(?:[0-5][0-9]) # minutes
(?:[0-5][0-9]) # minutes
(?::[0-5][0-9])? # seconds
(?:
Z
@ -224,7 +240,7 @@ sub _regularity {
)?
(?=$|\s|[,;|\t])
/x;
my $commadecsep_rx = qr/
(?:^|(?<=[^\d,.]))
(?:
@ -238,7 +254,7 @@ sub _regularity {
)
(?=$|[^\d,.])
/x;
my $commagroupsep_rx = qr/
(?:^|(?<=[^\d,.]))
(?:
@ -248,25 +264,25 @@ sub _regularity {
)
(?=$|[^\d,.])
/x;
return 0 if ($kind eq 'timecol' && $string !~ /$time_rx/);
return 0 if ($kind eq 'decsep' && $string !~ /$commadecsep_rx/);
return 0 if ($kind eq 'groupsep' && $string !~ /$commagroupsep_rx/);
return 1;
}
sub _message {
my ($type, $candidate) = @_;
my ($type, $candidate, $c) = @_;
my $char;
if (ord $candidate == 9) { # tab character
$char = "\\t";
} else {
$char = $candidate;
}
my %message = (
deleted => "Deleted $char from candidates list\n",
added => "Added $char to candidates list\n",
@ -274,7 +290,8 @@ sub _message {
detected => "\nSeparator detected: $char\n",
left => " $char ",
);
$c->log->debug($message{$type}) if $c;
print $message{$type};
}
@ -294,14 +311,14 @@ Version 0.20 - November 2, 2008
=head1 SYNOPSIS
use Text::CSV::Separator qw(get_separator);
my @char_list = get_separator(
path => $csv_path,
exclude => $array1_ref, # optional
include => $array2_ref, # optional
echo => 1, # optional
);
my $separator;
if (@char_list) {
if (@char_list == 1) { # successful detection
@ -311,19 +328,19 @@ Version 0.20 - November 2, 2008
} else { # no candidate passed the tests
# Some code here
}
# "I'm Feeling Lucky" alternative interface
# Don't forget to include the 'lucky' parameter
my $separator = get_separator(
path => $csv_path,
lucky => 1,
lucky => 1,
exclude => $array1_ref, # optional
include => $array2_ref, # optional
echo => 1, # optional
);
=head1 DESCRIPTION
@ -332,7 +349,7 @@ This module provides a fast detection of the field separator character (also
called field delimiter) of a CSV file, or more generally, of a character
separated text file (also called delimited text file), and returns it ready
to use in a CSV parser (e.g., Text::CSV_XS, Tie::CSV_File, or
Text::CSV::Simple).
Text::CSV::Simple).
This may be useful to the vulnerable -and often ignored- population of
programmers who need to process automatically CSV files from different sources.
@ -340,7 +357,7 @@ The default set of candidates contains the following characters:
',' ';' ':' '|' '\t'
The only required parameter is the CSV file path. Optionally, the user can
specify characters to be excluded or included in the list of candidates.
specify characters to be excluded or included in the list of candidates.
The routine returns an array containing the list of candidates that passed
the tests. If it succeeds, this array will contain only one value: the field
@ -456,25 +473,25 @@ default candidate not considered, the pipe character):
path => $csv_path,
exclude => [':', '|'],
);
if (@char_list) {
my $separator;
if (@char_list == 1) {
if (@char_list == 1) {
$separator = $char_list[0];
} else {
} else {
# Some code here
}
}
# Using the "I'm Feeling Lucky" interface:
my $separator = get_separator(
path => $csv_path,
lucky => 1,
exclude => [':', '|'],
);
=head1 MOTIVATION

@ -26,6 +26,7 @@ sub upload_csv {
my $separator = NGCP::Panel::Utils::CSVSeparator::get_separator(
path => $data,
lucky => 1,
c => $c,
);
unless(defined $separator) {
my $text = $c->loc("Failed to detect CSV separator");
@ -94,12 +95,14 @@ sub upload_csv {
sub create_csv {
my(%params) = @_;
my($c, $reseller_id) = @params{qw/c reseller_id/};
my($c, $reseller_id, $emergency_mapping_rs) = @params{qw/c reseller_id emergency_mapping_rs/};
$emergency_mapping_rs //= $c->stash->{emergency_mapping_rs} // $c->model('DB')->resultset('emergency_mappings');
#my @cols = @{ $c->config->{emergency_mapping_csv}->{element_order} };
my @cols = qw/name code prefix/;
my $mapping_rs = $c->stash->{emergency_mapping_rs}->search_rs({
my $mapping_rs = $emergency_mapping_rs->search_rs({
'emergency_container.reseller_id' => $reseller_id,
},
{

@ -88,12 +88,12 @@ sub upload_csv {
sub create_csv {
my(%params) = @_;
my($c) = @params{qw/c/};
my($c, $number_rs) = @params{qw/c number_rs/};
$number_rs //= $c->stash->{number_rs} // $c->model('DB')->resultset('lnp_numbers');
#my @cols = @{ $c->config->{lnp_csv}->{element_order} };
my @cols = qw/carrier_name carrier_prefix number routing_number start end authoritative skip_rewrite/;
my $lnp_rs = $c->stash->{number_rs}->search_rs(
my $lnp_rs = $number_rs->search_rs(
undef,
{
'+select' => ['lnp_provider.name','lnp_provider.prefix', 'lnp_provider.authoritative', 'lnp_provider.skip_rewrite'],

@ -0,0 +1,37 @@
use strict;
use warnings;
use Test::Collection;
use Test::FakeData;
use Test::More;
#init test_machine
my $test_machine = Test::Collection->new(
name => 'emergencymappingcontainers',
);
my $fake_data = Test::FakeData->new;
$test_machine->methods->{collection}->{allowed} = {map {$_ => 1} qw(GET HEAD OPTIONS POST)};
$test_machine->methods->{item}->{allowed} = {map {$_ => 1} qw(GET HEAD OPTIONS PUT PATCH DELETE)};
$fake_data->set_data_from_script({
'emergencymappingcontainers' => {
data => {
reseller_id => sub { return shift->get_id('resellers', @_); },
name => "apitest_emergency_mapping_cont",
},
'query' => ['name','reseller_id'],
},
});
$test_machine->DATA_ITEM_STORE($fake_data->process('emergencymappingcontainers'));
$test_machine->form_data_item( );
# create 3 new billing zones from DATA_ITEM
$test_machine->check_create_correct( 3, sub{ $_[0]->{name} .= $_[1]->{i} ; } );
$test_machine->check_get2put();
$test_machine->check_bundle();
$test_machine->clear_test_data_all();
done_testing;
# vim: set tabstop=4 expandtab:

@ -0,0 +1,105 @@
use strict;
use warnings;
use Test::Collection;
use Test::FakeData;
use Test::More;
use Data::Dumper;
use File::Temp qw/tempfile/;
use Clone qw/clone/;
#init test_machine
my $test_machine = Test::Collection->new(
name => 'emergencymappings',
);
my $fake_data = Test::FakeData->new;
$test_machine->methods->{collection}->{allowed} = {map {$_ => 1} qw(GET HEAD OPTIONS POST)};
$test_machine->methods->{item}->{allowed} = {map {$_ => 1} qw(GET HEAD OPTIONS PUT PATCH DELETE)};
$fake_data->set_data_from_script({
'emergencymappings' => {
data => {
json => {
emergency_container_id => sub { return shift->get_id('emergencymappingcontainers', @_); },
code => "112",
prefix => "000",
},
file => [ (tempfile())[1] ],
},
'create_special'=> sub {
my ($self,$name,$test_machine) = @_;
my $prev_params = $test_machine->get_cloned('content_type');
@{$test_machine->content_type}{qw/POST PUT/} = (('multipart/form-data') x 2);
$test_machine->check_create_correct(1);
$test_machine->set(%$prev_params);
},
'query' => ['code','emergency_container_id'],
},
});
my $data = $fake_data->process('emergencymappings');
$test_machine->DATA_ITEM_STORE($data);
{#test "usual" interface
$test_machine->DATA_ITEM($data->{json});
# create 3 new emergency mappings from DATA_ITEM
$test_machine->check_create_correct( 3, sub{ $_[0]->{code} .= $_[1]->{i}; } );
$test_machine->check_get2put();
$test_machine->check_bundle();
#we need to delete existing, if we want to check downloded content later
$test_machine->clear_test_data_all();#fake data aren't registered in this test machine, so they will stay.
}
{
#test "upload csv" interface variant
$test_machine->DATA_ITEM($data);
{
my $empty_container_data = clone $test_machine->DATA_ITEM;
delete $empty_container_data->{json}->{emergency_container_id};
my($res,$content) = $test_machine->request_post($empty_container_data);
$test_machine->http_code_msg(422, "Emergency Mapping Container field is required", $res, $content);
}
my $container = $test_machine->get_item_hal('emergencymappingcontainers');
my $csv_data = "$container->{content}->{name},217,1122\n$container->{content}->{name},413,5123\n";
my $csv_upload_url = '/api/emergencymappings/?reseller_id='.$container->{content}->{reseller_id};
my $csv_download_url = '/api/emergencymappings/?reseller_id='.$container->{content}->{reseller_id};
$test_machine->resource_fill_file($test_machine->DATA_ITEM->{file}->[0], $csv_data);
{
my $content_type_old = $test_machine->content_type;
$test_machine->content_type->{POST} = 'text/csv';
#my($res,$content) = $test_machine->request_post($test_machine->DATA_ITEM, $csv_upload_url);#.'&purge_existing=true'
my($res,$content) = $test_machine->request_post($csv_data, $csv_upload_url.'&purge_existing=true');#
$test_machine->http_code_msg(201, "check file upload", $res, $content);
$test_machine->content_type($content_type_old);
}
{
my $req = $test_machine->get_request_get( $csv_download_url );
$req->header('Accept' => 'text/csv');
my($res,$content) = $test_machine->request($req);
my $filename = "emergency_mapping_list_reseller_".$container->{content}->{reseller_id}.".csv";
$test_machine->http_code_msg(200, "check response code", $res, $content);
is($res->filename, $filename, "check downloaded csv filename: $filename;");
is($res->content, $csv_data, "check downloaded content;");
}
{
my $req = $test_machine->get_request_get( '/api/emergencymappings' );
$req->header('Accept' => 'text/csv');
my($res,$content) = $test_machine->request($req);
$test_machine->http_code_msg(400, "reseller_id parameter is necessary to download csv data", $res, $content);
}
{
#clear off uploaded mappings, as they out of the Collection control
my $content_type_old = $test_machine->content_type;
$test_machine->content_type->{POST} = 'text/csv';
my($res,$content) = $test_machine->request_post("nope", $csv_upload_url);
$test_machine->http_code_msg(201, "check file upload", $res, $content);
$test_machine->content_type($content_type_old);
}
}
$test_machine->clear_test_data_all();
undef $test_machine;
undef $fake_data;
done_testing;
# vim: set tabstop=4 expandtab:

@ -16,7 +16,7 @@ use Clone qw/clone/;
use File::Basename;
use Test::HTTPRequestAsCurl;
use Data::Dumper;
use File::Slurp qw/write_file/;
has 'local_test' => (
is => 'rw',
@ -316,12 +316,12 @@ sub get_item_hal{
}
}
if(!$resitem){
my ($reshal, $location);
my ($reshal, $location,$total_count);
$uri //= $self->get_uri_collection($name)."?page=1&rows=1";
my($res,$list_collection,$req) = $self->check_item_get($self->normalize_uri($uri));
($reshal,$location) = $self->get_hal_from_collection($list_collection,$name);
if($reshal->{total_count} || ('HASH' eq ref $reshal->{content} && $reshal->{content}->{total_count})){
$resitem = { num => 1, content => $reshal, res => $res, req => $req, location => $location };
($reshal,$location,$total_count) = $self->get_hal_from_collection($list_collection,$name);
if($total_count || ('HASH' eq ref $reshal->{content} && $reshal->{content}->{total_count})){
$resitem = { num => 1, content => $reshal, res => $res, req => $req, location => $location, total_count => $total_count };
$self->DATA_LOADED->{$name} ||= [];
push @{$self->DATA_LOADED->{$name}}, $resitem;
}
@ -332,20 +332,27 @@ sub get_hal_from_collection{
my($self,$list_collection,$name) = @_;
$name ||= $self->name;
my $hal_name = $self->get_hal_name($name);
my($reshal,$location);
if(ref $list_collection->{_links}->{$hal_name} eq "HASH") {
my($reshal,$location,$total_count);
if( $list_collection->{_embedded} && ref $list_collection->{_embedded}->{$hal_name} eq 'ARRAY') {
$reshal = $list_collection->{_embedded}->{$hal_name}->[0];
$location = $reshal->{_links}->{self}->{href};
$total_count = $reshal->{total_count};
} elsif( $list_collection->{_embedded} && ref $list_collection->{_embedded}->{$hal_name} eq 'HASH') {
$reshal = $list_collection->{_embedded}->{$hal_name};
$location = $reshal->{_links}->{self}->{href};
$total_count = $list_collection->{total_count};
} elsif(ref $list_collection->{_links}->{$hal_name} eq "HASH") {
#found first subscriber
$reshal = $list_collection;
$location = $reshal->{_links}->{$hal_name}->{href};
} elsif( $list_collection->{_embedded} && ref $list_collection->{_embedded}->{$hal_name} eq 'ARRAY') {
$reshal = $list_collection->{_embedded}->{$hal_name}->[0];
$location = $reshal->{_links}->{self}->{href};
}elsif( ref $list_collection eq 'HASH' && $list_collection->{_links}->{self}->{href}) {
$total_count = $reshal->{total_count};
} elsif( ref $list_collection eq 'HASH' && $list_collection->{_links}->{self}->{href}) {
#preferencedefs collection
$reshal = $list_collection;
$location = $reshal->{_links}->{self}->{href};
$total_count = $reshal->{total_count};
}
return ($reshal,$location);
return ($reshal,$location,$total_count);
}
sub get_created_first{
my($self) = @_;
@ -405,6 +412,12 @@ sub request_process{
my $rescontent = $self->get_response_content($res);
return ($res,$rescontent,$req);
}
sub get_request_get{
my($self, $uri) = @_;
$uri = $self->normalize_uri($uri);
my $req = HTTP::Request->new('GET', $uri);
return $req ;
}
sub get_request_put{
my($self,$content,$uri) = @_;
$uri ||= $self->get_uri_current;
@ -480,9 +493,11 @@ sub request_delete{
my $content = $self->get_response_content($res);
return($req,$res,$content);
}
sub request_get{
my($self,$uri) = @_;
my $req = HTTP::Request->new('GET', $self->normalize_uri($uri));
my($self,$uri,$req) = @_;
$uri = $self->normalize_uri($uri);
$req //= $self->get_request_get($uri);
my $res = $self->request($req);
my $content = $self->get_response_content($res);
return wantarray ? ($res, $content, $req) : $res;
@ -983,10 +998,9 @@ sub hash2params{
return join '&', map {$_.'='.uri_escape($hash->{$_})} keys %{ $hash };
}
sub resource_fill_file{
#$_[0]->{faxfile}->[0]
my $cmd = "echo 'aaa' > $_[1]";
print "cmd=$cmd;\n";
`$cmd`;
my($self,$filename,$data) = @_;
$data //= 'aaa';
write_file($filename,$data);
}
sub resource_clear_file{
my $cmd = "echo -n '' > $_[1]";

Loading…
Cancel
Save