From fe2a53185faddd00bc559fe68562fe54b598da00 Mon Sep 17 00:00:00 2001 From: Kirill Solomko Date: Thu, 24 Nov 2016 19:21:32 +0100 Subject: [PATCH] TT#5004 rework pbx device firmware data download/upload * add a helper module Utils/DeviceFirmware.pm * device firmware data upload internally splits the data into 10MB chunks and upload it into the new autoprov_firmwares_data table * NGCP::Panel::Utils::insert_firmware_data() * NGCP::Panel::Utils::download_firmware_data() Change-Id: I91b2ae3d5b6c2d34c9d540ad816bb7549b4543cb --- .../API/PbxDeviceFirmwareBinariesItem.pm | 6 +- .../Controller/API/PbxDeviceFirmwares.pm | 7 +- .../Controller/API/PbxDeviceFirmwaresItem.pm | 13 +++- lib/NGCP/Panel/Controller/Device.pm | 22 ++++-- lib/NGCP/Panel/Form/Device/Firmware.pm | 2 +- lib/NGCP/Panel/Role/API/PbxDeviceFirmwares.pm | 5 -- lib/NGCP/Panel/Utils/DeviceFirmware.pm | 73 +++++++++++++++++++ 7 files changed, 110 insertions(+), 18 deletions(-) create mode 100644 lib/NGCP/Panel/Utils/DeviceFirmware.pm diff --git a/lib/NGCP/Panel/Controller/API/PbxDeviceFirmwareBinariesItem.pm b/lib/NGCP/Panel/Controller/API/PbxDeviceFirmwareBinariesItem.pm index 4b4f278c43..014eb34400 100644 --- a/lib/NGCP/Panel/Controller/API/PbxDeviceFirmwareBinariesItem.pm +++ b/lib/NGCP/Panel/Controller/API/PbxDeviceFirmwareBinariesItem.pm @@ -6,6 +6,7 @@ use Sipwise::Base; use HTTP::Headers qw(); use HTTP::Status qw(:constants); +use NGCP::Panel::Utils::DeviceFirmware; use NGCP::Panel::Utils::DateTime; use NGCP::Panel::Utils::ValidateJSON qw(); use Path::Tiny qw(path); @@ -59,10 +60,11 @@ sub GET :Allow { last unless $self->resource_exists($c, pbxdevicefirmwarebinary => $item); my $resource = $self->resource_from_item($c, $item); - $resource->{data} = $item->data; $c->response->header ('Content-Disposition' => 'attachment; filename="' . $resource->{filename} . '"'); $c->response->content_type('application/octet-stream'); - $c->response->body($resource->{data}); + $c->response->body(NGCP::Panel::Utils::DeviceFirmware::get_firmware_data( + c => $c, fw_id => $item->id + )); return; } return; diff --git a/lib/NGCP/Panel/Controller/API/PbxDeviceFirmwares.pm b/lib/NGCP/Panel/Controller/API/PbxDeviceFirmwares.pm index 5cdc7fb761..7114ecfceb 100644 --- a/lib/NGCP/Panel/Controller/API/PbxDeviceFirmwares.pm +++ b/lib/NGCP/Panel/Controller/API/PbxDeviceFirmwares.pm @@ -10,6 +10,7 @@ use HTTP::Headers qw(); use HTTP::Status qw(:constants); use NGCP::Panel::Utils::DateTime; +use NGCP::Panel::Utils::DeviceFirmware; use Path::Tiny qw(path); use Safe::Isa qw($_isa); require Catalyst::ActionRole::ACL; @@ -199,12 +200,16 @@ sub POST :Allow { last; } - $resource->{data} = $binary; last unless($resource); my $item; try { $item = $model->autoprov_firmwares->create($resource); + if ($binary) { + NGCP::Panel::Utils::DeviceFirmware::insert_firmware_data( + c => $c, fw_id => $item->id, data_ref => \$binary + ); + } } catch($e) { $c->log->error("failed to create pbxdevicefirmware: $e"); # TODO: user, message, trace, ... $self->error($c, HTTP_INTERNAL_SERVER_ERROR, "Failed to create pbxdevicefirmware."); diff --git a/lib/NGCP/Panel/Controller/API/PbxDeviceFirmwaresItem.pm b/lib/NGCP/Panel/Controller/API/PbxDeviceFirmwaresItem.pm index 21dc7b6d2f..dbb5102d31 100644 --- a/lib/NGCP/Panel/Controller/API/PbxDeviceFirmwaresItem.pm +++ b/lib/NGCP/Panel/Controller/API/PbxDeviceFirmwaresItem.pm @@ -102,21 +102,26 @@ sub PUT :Allow { my $item = $self->item_by_id($c, $id); last unless $self->resource_exists($c, pbxdevicefirmware => $item); - my $recording = $self->get_valid_raw_put_data( + my $binary = $self->get_valid_raw_put_data( c => $c, id => $id, media_type => 'application/octet-stream', ); - last unless $recording; + last unless $binary; my $resource = $c->req->query_params; - $resource->{data} = $recording; my $form = $self->get_form($c); my $old_resource = $self->resource_from_item($c, $item, $form); $item = $self->update_item($c, $item, $old_resource, $resource, $form); last unless $item; - $guard->commit; + if ($binary) { + NGCP::Panel::Utils::DeviceFirmware::insert_firmware_data( + c => $c, fw_id => $item->id, data_ref => \$binary + ); + } + + $guard->commit; if ('minimal' eq $preference) { $c->response->status(HTTP_NO_CONTENT); diff --git a/lib/NGCP/Panel/Controller/Device.pm b/lib/NGCP/Panel/Controller/Device.pm index 7b55813935..49719f1e93 100644 --- a/lib/NGCP/Panel/Controller/Device.pm +++ b/lib/NGCP/Panel/Controller/Device.pm @@ -15,6 +15,7 @@ use NGCP::Panel::Form::Device::Profile; use NGCP::Panel::Utils::Navigation; use NGCP::Panel::Utils::DeviceBootstrap; use NGCP::Panel::Utils::Device; +use NGCP::Panel::Utils::DeviceFirmware; use NGCP::Panel::Utils::DateTime; use DateTime::Format::HTTP; @@ -573,9 +574,13 @@ sub devfw_create :Chained('base') :PathPart('firmware/create') :Args(0) :Does(AC $schema->txn_do(sub { my $file = delete $form->values->{data}; $form->values->{filename} = $file->filename; - $form->values->{data} = $file->slurp; my $devmod = $c->stash->{devmod_rs}->find($form->values->{device}{id},{'+columns' => [qw/mac_image front_image/]}); my $devfw = $devmod->create_related('autoprov_firmwares', $form->values); + if ($file->size) { + NGCP::Panel::Utils::DeviceFirmware::insert_firmware_data( + c => $c, fw_id => $devfw->id, data_fh => $file->fh + ); + } delete $c->session->{created_objects}->{device}; $c->session->{created_objects}->{firmware} = { id => $devfw->id }; }); @@ -611,7 +616,7 @@ sub devfw_base :Chained('base') :PathPart('firmware') :CaptureArgs(1) :Does(ACL) NGCP::Panel::Utils::Navigation::back_or($c, $c->uri_for('/device')); } - $c->stash->{devfw} = $c->stash->{devfw_rs}->find($devfw_id,{'+columns' => 'data'}); + $c->stash->{devfw} = $c->stash->{devfw_rs}->find($devfw_id); unless($c->stash->{devfw}) { NGCP::Panel::Utils::Message::error( c => $c, @@ -678,9 +683,14 @@ sub devfw_edit :Chained('devfw_base') :PathPart('edit') :Args(0) { delete $form->values->{device}; my $file = delete $form->values->{data}; $form->values->{filename} = $file->filename; - $form->values->{data} = $file->slurp; - $c->stash->{devfw}->update($form->values); + if ($file->size) { + NGCP::Panel::Utils::DeviceFirmware::insert_firmware_data( + c => $c, + fw_id => $c->stash->{devfw}->id, + data_fh => $file->fh + ); + } delete $c->session->{created_objects}->{device}; }); NGCP::Panel::Utils::Message::info( @@ -710,7 +720,9 @@ sub devfw_download :Chained('devfw_base') :PathPart('download') :Args(0) { $c->response->header ('Content-Disposition' => 'attachment; filename="' . $fw->filename . '"'); $c->response->content_type('application/octet-stream'); - $c->response->body($fw->data); + $c->response->body(NGCP::Panel::Utils::DeviceFirmware::get_firmware_data( + c => $c, fw_id => $fw->id + )); } sub devconf_ajax :Chained('base') :PathPart('config/ajax') :Args(0) :Does(ACL) :ACLDetachTo('/denied_page') :AllowedRole(admin) :AllowedRole(reseller) { diff --git a/lib/NGCP/Panel/Form/Device/Firmware.pm b/lib/NGCP/Panel/Form/Device/Firmware.pm index 6d22322b2e..293f5297cf 100644 --- a/lib/NGCP/Panel/Form/Device/Firmware.pm +++ b/lib/NGCP/Panel/Form/Device/Firmware.pm @@ -27,7 +27,7 @@ has_field 'data' => ( type => 'Upload', required => 1, label => 'Firmware File', - max_size => '134217728', # 128MB + max_size => '671088640', # 500MB ); has_field 'save' => ( diff --git a/lib/NGCP/Panel/Role/API/PbxDeviceFirmwares.pm b/lib/NGCP/Panel/Role/API/PbxDeviceFirmwares.pm index c54d935abc..5094a24470 100644 --- a/lib/NGCP/Panel/Role/API/PbxDeviceFirmwares.pm +++ b/lib/NGCP/Panel/Role/API/PbxDeviceFirmwares.pm @@ -65,7 +65,6 @@ sub hal_from_item { ); $resource->{id} = int($item->id); - delete $resource->{data}; $hal->resource($resource); return $hal; } @@ -74,7 +73,6 @@ sub resource_from_item { my ($self, $c, $item, $form) = @_; my $resource = { $item->get_inflated_columns }; - delete $resource->{data}; return $resource; } @@ -88,7 +86,6 @@ sub item_by_id { sub update_item { my ($self, $c, $item, $old_resource, $resource, $form) = @_; - my $binary = delete $resource->{data}; $form //= $self->get_form($c); return unless $self->validate_form( c => $c, @@ -113,8 +110,6 @@ sub update_item { last; } - $resource->{data} = $binary; - $item->update($resource); return $item; diff --git a/lib/NGCP/Panel/Utils/DeviceFirmware.pm b/lib/NGCP/Panel/Utils/DeviceFirmware.pm new file mode 100644 index 0000000000..7bb2f9a47f --- /dev/null +++ b/lib/NGCP/Panel/Utils/DeviceFirmware.pm @@ -0,0 +1,73 @@ +package NGCP::Panel::Utils::DeviceFirmware; + +use English; + +sub insert_firmware_data { + my (%args) = @_; + my $c = $args{c}; + my $fw_id = $args{fw_id}; + # either data_ref or data_fh are expected + my $data_ref = $args{data_ref}; + my $data_fh = $args{data_fh}; + + # do not set it larger than 10M + my $chunk_size = 10*1024*1024; # 10MB + + $c->model('DB')->resultset('autoprov_firmwares')->find({fw_id => $fw_id},{for => 'update'}); + + my $rs = $c->model('DB')->resultset('autoprov_firmwares_data')->search({ + fw_id => $fw_id + }); + if ($rs->count > 0) { + $rs->delete; + } + + my $fh; + if ($data_fh) { + $fh = $data_fh; + seek($fh, 0, 0); + } else { + open($fh, "<:raw", $data_ref) + or die "Cannot open firmware_data filehandler: ".$ERRNO; + } + + my $buffer; + my $offset = 0; + binmode $fh; + while (read($fh, $buffer, $chunk_size, 0)) { + $c->model('DB')->resultset('autoprov_firmwares_data')->create( + { fw_id => $fw_id, data => $buffer } + ); + $offset += $chunk_size; + seek($fh, $offset, 0); + } + + close $fh; + + return; +} + +sub get_firmware_data { + my (%args) = @_; + my $c = $args{c}; + my $fw_id = $args{fw_id}; + + my $rs = $c->model('DB')->resultset('autoprov_firmwares_data')->search({ + fw_id => $fw_id + }, + { + order_by => { -asc => 'id' } + }); + + my $data = ''; + if ($rs->first) { + foreach my $fw_data ($rs->all) { + $data = $data . $fw_data->data; + } + } + + return $data; +} +1; + +# vim: set tabstop=4 expandtab: