sipwise
/
bulk-processor
mirror of https://github.com/sipwise/bulk-processor.git
1
0
Fork 0
Code Issues Releases Wiki Activity

MT#56236 Simplify and extend kbytes2gigs() into humanize_bytes()

Browse Source 
Change the function to handle bytes by default, which is what most
callers are passing. Add support for terabytes and petabytes. Switch
kbytes2gigs() to be a wrapper over humanize_bytes() for backwards
compatibility for now until all callers are switched. Remove
unnecessary int() coercion from call sites.

Change-Id: I94be536d73930a453d588e0edbb8192d6844bbe0
mr11.2
Guillem Jover 2 years ago
parent 0604c71f15
commit 0db028f6e3
5 changed files with 64 additions and 18 deletions
Show Stats Download Patch File Download Diff File

18
lib/NGCP/BulkProcessor/AttachmentDownloader.pm
Unescape View File

@ -25,7 +25,7 @@ use LWP::UserAgent;
use HTTP::Request;
#use HTTP::Cookies;
use NGCP::BulkProcessor::Utils qw(kbytes2gigs changemod);
use NGCP::BulkProcessor::Utils qw(humanize_bytes changemod);
require Exporter;
our @ISA = qw(Exporter);
@ -116,15 +116,15 @@ sub _process_attachments {
if (defined $self->{checkfilenamecode} and ref $self->{checkfilenamecode} eq 'CODE') {
my $match = &{$self->{checkfilenamecode}}($attachment);
if ($match == $attachment_no_match) {
attachmentdownloaderinfo('attachment ' . $attachment->{filename} . ' (' . kbytes2gigs(int($attachment->{size} / 1024), undef, 1) . ' ' . $attachment->{content_type} . ') skipped',getlogger(__PACKAGE__));
attachmentdownloaderinfo('attachment ' . $attachment->{filename} . ' (' . humanize_bytes($attachment->{size}, undef, 1) . ' ' . $attachment->{content_type} . ') skipped',getlogger(__PACKAGE__));
next;
} elsif ($match == $attachment_found) {
attachmentdownloaderinfo('attachment ' . $attachment->{filename} . ' (' . kbytes2gigs(int($attachment->{size} / 1024), undef, 1) . ' ' . $attachment->{content_type} . ') found',getlogger(__PACKAGE__));
attachmentdownloaderinfo('attachment ' . $attachment->{filename} . ' (' . humanize_bytes($attachment->{size}, undef, 1) . ' ' . $attachment->{content_type} . ') found',getlogger(__PACKAGE__));
$found = 1;
} elsif ($match == $attachment_match) {
attachmentdownloaderinfo('attachment ' . $attachment->{filename} . ' (' . kbytes2gigs(int($attachment->{size} / 1024), undef, 1) . ' ' . $attachment->{content_type} . ') matched',getlogger(__PACKAGE__));
attachmentdownloaderinfo('attachment ' . $attachment->{filename} . ' (' . humanize_bytes($attachment->{size}, undef, 1) . ' ' . $attachment->{content_type} . ') matched',getlogger(__PACKAGE__));
} else {
attachmentdownloaderwarn('attachment ' . $attachment->{filename} . ' (' . kbytes2gigs(int($attachment->{size} / 1024), undef, 1) . ' ' . $attachment->{content_type} . ') - unknown match, skipped',getlogger(__PACKAGE__));
attachmentdownloaderwarn('attachment ' . $attachment->{filename} . ' (' . humanize_bytes($attachment->{size}, undef, 1) . ' ' . $attachment->{content_type} . ') - unknown match, skipped',getlogger(__PACKAGE__));
next;
}
}
@ -209,15 +209,15 @@ sub _process_body {
if (defined $self->{checkfilenamecode} and ref $self->{checkfilenamecode} eq 'CODE') {
my $match = &{$self->{checkfilenamecode}}($attachment);
if ($match == $attachment_no_match) {
attachmentdownloaderinfo('attachment ' . $attachment->{filename} . ' (' . kbytes2gigs(int($attachment->{size} / 1024), undef, 1) . ' ' . $attachment->{content_type} . ') skipped',getlogger(__PACKAGE__));
attachmentdownloaderinfo('attachment ' . $attachment->{filename} . ' (' . humanize_bytes($attachment->{size}, undef, 1) . ' ' . $attachment->{content_type} . ') skipped',getlogger(__PACKAGE__));
next;
} elsif ($match == $attachment_found) {
attachmentdownloaderinfo('attachment ' . $attachment->{filename} . ' (' . kbytes2gigs(int($attachment->{size} / 1024), undef, 1) . ' ' . $attachment->{content_type} . ') found',getlogger(__PACKAGE__));
attachmentdownloaderinfo('attachment ' . $attachment->{filename} . ' (' . humanize_bytes($attachment->{size}, undef, 1) . ' ' . $attachment->{content_type} . ') found',getlogger(__PACKAGE__));
$found = 1;
} elsif ($match == $attachment_match) {
attachmentdownloaderinfo('attachment ' . $attachment->{filename} . ' (' . kbytes2gigs(int($attachment->{size} / 1024), undef, 1) . ' ' . $attachment->{content_type} . ') matched',getlogger(__PACKAGE__));
attachmentdownloaderinfo('attachment ' . $attachment->{filename} . ' (' . humanize_bytes($attachment->{size}, undef, 1) . ' ' . $attachment->{content_type} . ') matched',getlogger(__PACKAGE__));
} else {
attachmentdownloaderwarn('attachment ' . $attachment->{filename} . ' (' . kbytes2gigs(int($attachment->{size} / 1024), undef, 1) . ' ' . $attachment->{content_type} . ') - unknown match, skipped',getlogger(__PACKAGE__));
attachmentdownloaderwarn('attachment ' . $attachment->{filename} . ' (' . humanize_bytes($attachment->{size}, undef, 1) . ' ' . $attachment->{content_type} . ') - unknown match, skipped',getlogger(__PACKAGE__));
next;
}
}

2
lib/NGCP/BulkProcessor/Downloaders/IMAPAttachmentDownloader.pm
Unescape View File

@ -14,8 +14,6 @@ use NGCP::BulkProcessor::LogError qw(
attachmentdownloaderwarn
);
use NGCP::BulkProcessor::Utils qw(kbytes2gigs); # changemod);
use IO::Socket::SSL;
use Mail::IMAPClient;
use MIME::Base64;

10
lib/NGCP/BulkProcessor/Logging.pm
Unescape View File

@ -16,7 +16,7 @@ use Log::Log4perl qw(get_logger);
use File::Basename qw(basename);
use NGCP::BulkProcessor::Utils qw(timestampdigits datestampdigits changemod chopstring trim kbytes2gigs);
use NGCP::BulkProcessor::Utils qw(timestampdigits datestampdigits changemod chopstring trim humanize_bytes);
use NGCP::BulkProcessor::Array qw (contains);
require Exporter;
@ -665,7 +665,7 @@ sub fileprocessingstarted {
my ($file,$logger) = @_;
if (defined $logger) {
$logger->info('file processing started: ' . basename($file) . ' (' . kbytes2gigs(int((-s $file)/ 1024)) . ')');
$logger->info('file processing started: ' . basename($file) . ' (' . humanize_bytes(-s $file) . ')');
}
}
@ -684,7 +684,7 @@ sub fileprocessingdone {
# my ($file,$start,$blocksize,$block_n,$logger) = @_;
# if (defined $logger) {
# if (defined $block_n) {
# $logger->info('fetching lines from ' . basename($file) . ': ' . kbytes2gigs($block_n));
# $logger->info('fetching lines from ' . basename($file) . ': ' . humanize_bytes($block_n));
# } else {
# $logger->info('fetching lines from ' . basename($file) . ': ' . ($start + 1) . '~' . ($start + $blocksize));
# }
@ -699,7 +699,7 @@ sub lines_read {
if (defined $logger) {
if (defined $block_n) {
if ($block_n > 0) {
$logger->info(basename($file) . ': ' . kbytes2gigs(int($block_n / 1024)) . ' read');
$logger->info(basename($file) . ': ' . humanize_bytes($block_n) . ' read');
}
} else {
$logger->info(basename($file) . ': lines ' . ($start + 1) . '~' . ($start + $blocksize) . ' read');
@ -714,7 +714,7 @@ sub processing_lines {
if (defined $logger) {
if (defined $block_n) {
if ($block_n > 0) {
$logger->info(($enablemultithreading ? '[' . $tid . '] ' : '') . 'processing lines: ' . kbytes2gigs(int($block_n / 1024)));
$logger->info(($enablemultithreading ? '[' . $tid . '] ' : '') . 'processing lines: ' . humanize_bytes($block_n));
}
} else {
$logger->info(($enablemultithreading ? '[' . $tid . '] ' : '') . 'processing lines: ' . ($start + 1) . '-' . ($start + $blocksize));

14
lib/NGCP/BulkProcessor/Utils.pm
Unescape View File

@ -88,6 +88,7 @@ our @EXPORT_OK = qw(
get_hostfqdn
getscriptpath
humanize_bytes
kbytes2gigs
cleanupdir
fixdirpath
@ -489,12 +490,15 @@ sub getscriptpath {
}
my @unit_suffix = qw(
Bytes
kBytes
MBytes
GBytes
TBytes
PBytes
);
sub kbytes2gigs {
sub humanize_bytes {
my ($number, $base, $round_integer) = @_;
$base = 1024 if $base <= 0;
@ -511,6 +515,14 @@ sub kbytes2gigs {
return "$number $unit_suffix[$unit]";
}
sub kbytes2gigs {
my ($number, $base, $round_integer) = @_;
$base //= 1024;
return humanize_bytes($number * $base, $base, $round_integer);
}
sub cleanupdir {
my ($dirpath,$keeproot,$filewarncode,$logger) = @_;

38
t/Utils.t
Unescape View File

@ -16,13 +16,14 @@
use strict;
use warnings;
use Test::More tests => 45;
use Test::More tests => 72;
use Time::Local;
require_ok('NGCP::BulkProcessor::Utils');
NGCP::BulkProcessor::Utils->import(qw(
zerofill
humanize_bytes
kbytes2gigs
secs_to_years
timestampdigits
@ -42,6 +43,41 @@ is(zerofill(25, 4), '0025');
is(zerofill(1000, 4), '1000');
# Unit conversion
is(humanize_bytes(1), '1 Bytes');
is(humanize_bytes(1024), '1 kBytes');
is(humanize_bytes(1024 ** 2), '1 MBytes');
is(humanize_bytes(1024 ** 3), '1 GBytes');
is(humanize_bytes(1024 ** 4), '1 TBytes');
is(humanize_bytes(1024 ** 5), '1 PBytes');
is(humanize_bytes(2), '2 Bytes');
is(humanize_bytes(2 * 1024), '2 kBytes');
is(humanize_bytes(2 * (1024 ** 2) + 1), '2 MBytes');
is(humanize_bytes(2 * (1024 ** 3) + 1024 + 1), '2 GBytes');
is(humanize_bytes(2 * (1024 ** 4) + (1024 ** 2) + 1024 + 1), '2 TBytes');
is(humanize_bytes(920), '920 Bytes');
is(humanize_bytes(920 * 1024), '920 kBytes');
is(humanize_bytes(942080 * 1024), '920 MBytes');
is(humanize_bytes(964689920 * 1024), '920 GBytes');
is(humanize_bytes(920, 1000), '920 Bytes');
is(humanize_bytes(920000, 1000), '920 kBytes');
is(humanize_bytes(920000000, 1000), '920 MBytes');
is(humanize_bytes(920000000000, 1000), '920 GBytes');
is(humanize_bytes(942172), '920.08 kBytes');
is(humanize_bytes(965632092), '920.89 MBytes');
is(humanize_bytes(920920, 1000), '920.92 kBytes');
is(humanize_bytes(920920920, 1000), '920.92 MBytes');
is(humanize_bytes(942172, 1024, 1), '920 kBytes');
is(humanize_bytes(965632092, 1024, 1), '920 MBytes');
is(humanize_bytes(920920, 1000, 1), '920 kBytes');
is(humanize_bytes(920920920, 1000,1 ), '920 MBytes');
is(kbytes2gigs(1), '1 kBytes');
is(kbytes2gigs(1024), '1 MBytes');
is(kbytes2gigs(1024 ** 2), '1 GBytes');

Write Preview
Loading…
Cancel
Save