MT#16777 Write chunks to db to not run out of mem

If we have a 20M lnp file to upload, putting everything into the
numbers array is killing the fcgi process on low-mem machines.
Make sure to write each 2k chunk to DB and clear the array
afterwards.

Change-Id: I857be5d6a0195607b1fcb86db3327a9fd10ce5f1

lib/NGCP/Panel/Controller/Lnp.pm

@@ -35,9 +35,9 @@ sub list :Chained('/') :PathPart('lnp') :CaptureArgs(0) {
     my $number_rs = $c->model('DB')->resultset('lnp_numbers');
     $c->stash(number_rs => $number_rs);
     $c->stash->{number_dt_columns} = NGCP::Panel::Utils::Datatables::set_columns($c, [
-        { name => "id", "search" => 1, "title" => $c->loc("#") },
+        { name => "id", "search" => 0, "title" => $c->loc("#") },
         { name => "number", "search" => 1, "title" => $c->loc("Number") },
-        { name => "routing_number", "search" => 1, "title" => $c->loc("Routing Number") },
+        { name => "routing_number", "search" => 0, "title" => $c->loc("Routing Number") },
         { name => "lnp_provider.name", "search" => 1, "title" => $c->loc("Carrier") },
         { name => "start", "search" => 0, "title" => $c->loc("Start Date") },
         { name => "end", "search" => 0, "title" => $c->loc("End Date") },

lib/NGCP/Panel/Form/Lnp/Upload.pm

@@ -15,7 +15,7 @@ sub build_form_element_class { [qw/form-horizontal/] }
 
 has_field 'upload_lnp' => ( 
     type => 'Upload',
-    max_size => '67108864', # 64MB
+    max_size => '2097152000', # 2GB
 );
 
 has_field 'purge_existing' => (

lib/NGCP/Panel/Utils/Lnp.pm

@@ -22,6 +22,7 @@ sub upload_csv {
     my %carriers = ();
     open(my $fh, '<:encoding(utf8)', $data);
     $start = time;
+    my $chunk_size = 2000;
     while ( my $line = $csv->getline($fh)) {
         ++$linenum;
         unless (scalar @{ $line } == scalar @cols) {
@@ -39,23 +40,26 @@ sub upload_csv {
             });
             $carriers{$k} = $carrier->id;
         }
+        $row->{start} ||= undef;
+        $row->{end} ||= undef;
         push @numbers, [$carriers{$k}, $row->{number}, $row->{start}, $row->{end}];
+
+        if($linenum % $chunk_size == 0) {
+            NGCP::Panel::Utils::MySQL::bulk_insert(
+                c => $c,
+                schema => $schema,
+                do_transaction => 0,
+                query => "INSERT INTO billing.lnp_numbers(lnp_provider_id, number, start, end)",
+                data => \@numbers,
+                chunk_size => $chunk_size
+            );
+            @numbers = ();
+        }
     }
     $end = time;
     close $fh;
-    $c->log->debug("Parsing LNP CSV took " . ($end - $start) . "s");
+    $c->log->debug("Parsing and uploading LNP CSV took " . ($end - $start) . "s");
 
-    $start = time;
-    NGCP::Panel::Utils::MySQL::bulk_insert(
-        c => $c,
-        schema => $schema,
-        do_transaction => 0,
-        query => "INSERT INTO billing.lnp_numbers(lnp_provider_id, number, start, end)",
-        data => \@numbers,
-        chunk_size => 2000
-    );
-    $end = time;
-    $c->log->debug("Bulk inserting LNP CSV took " . ($end - $start) . "s");
     
     my $text = $c->loc('LNP numbers successfully uploaded');
     if(@fails) {

share/templates/lnp/list.tt

@@ -46,6 +46,7 @@
     helper.messages = number_messages;
     helper.dt_columns = number_dt_columns;
     helper.length_change = 1;
+    helper.no_sort = 1;
 
     helper.close_target = number_close_target;
     helper.create_flag = number_create_flag;