From 0db028f6e38e6a48ec16aeb0d4e3c5bdff9e90c0 Mon Sep 17 00:00:00 2001 From: Guillem Jover Date: Fri, 16 Dec 2022 23:01:21 +0100 Subject: [PATCH] MT#56236 Simplify and extend kbytes2gigs() into humanize_bytes() Change the function to handle bytes by default, which is what most callers are passing. Add support for terabytes and petabytes. Switch kbytes2gigs() to be a wrapper over humanize_bytes() for backwards compatibility for now until all callers are switched. Remove unnecessary int() coercion from call sites. Change-Id: I94be536d73930a453d588e0edbb8192d6844bbe0 --- .../BulkProcessor/AttachmentDownloader.pm | 18 ++++----- .../Downloaders/IMAPAttachmentDownloader.pm | 2 - lib/NGCP/BulkProcessor/Logging.pm | 10 ++--- lib/NGCP/BulkProcessor/Utils.pm | 14 ++++++- t/Utils.t | 38 ++++++++++++++++++- 5 files changed, 64 insertions(+), 18 deletions(-) diff --git a/lib/NGCP/BulkProcessor/AttachmentDownloader.pm b/lib/NGCP/BulkProcessor/AttachmentDownloader.pm index 45c746e..a09c2f8 100644 --- a/lib/NGCP/BulkProcessor/AttachmentDownloader.pm +++ b/lib/NGCP/BulkProcessor/AttachmentDownloader.pm @@ -25,7 +25,7 @@ use LWP::UserAgent; use HTTP::Request; #use HTTP::Cookies; -use NGCP::BulkProcessor::Utils qw(kbytes2gigs changemod); +use NGCP::BulkProcessor::Utils qw(humanize_bytes changemod); require Exporter; our @ISA = qw(Exporter); @@ -116,15 +116,15 @@ sub _process_attachments { if (defined $self->{checkfilenamecode} and ref $self->{checkfilenamecode} eq 'CODE') { my $match = &{$self->{checkfilenamecode}}($attachment); if ($match == $attachment_no_match) { - attachmentdownloaderinfo('attachment ' . $attachment->{filename} . ' (' . kbytes2gigs(int($attachment->{size} / 1024), undef, 1) . ' ' . $attachment->{content_type} . ') skipped',getlogger(__PACKAGE__)); + attachmentdownloaderinfo('attachment ' . $attachment->{filename} . ' (' . humanize_bytes($attachment->{size}, undef, 1) . ' ' . $attachment->{content_type} . ') skipped',getlogger(__PACKAGE__)); next; } elsif ($match == $attachment_found) { - attachmentdownloaderinfo('attachment ' . $attachment->{filename} . ' (' . kbytes2gigs(int($attachment->{size} / 1024), undef, 1) . ' ' . $attachment->{content_type} . ') found',getlogger(__PACKAGE__)); + attachmentdownloaderinfo('attachment ' . $attachment->{filename} . ' (' . humanize_bytes($attachment->{size}, undef, 1) . ' ' . $attachment->{content_type} . ') found',getlogger(__PACKAGE__)); $found = 1; } elsif ($match == $attachment_match) { - attachmentdownloaderinfo('attachment ' . $attachment->{filename} . ' (' . kbytes2gigs(int($attachment->{size} / 1024), undef, 1) . ' ' . $attachment->{content_type} . ') matched',getlogger(__PACKAGE__)); + attachmentdownloaderinfo('attachment ' . $attachment->{filename} . ' (' . humanize_bytes($attachment->{size}, undef, 1) . ' ' . $attachment->{content_type} . ') matched',getlogger(__PACKAGE__)); } else { - attachmentdownloaderwarn('attachment ' . $attachment->{filename} . ' (' . kbytes2gigs(int($attachment->{size} / 1024), undef, 1) . ' ' . $attachment->{content_type} . ') - unknown match, skipped',getlogger(__PACKAGE__)); + attachmentdownloaderwarn('attachment ' . $attachment->{filename} . ' (' . humanize_bytes($attachment->{size}, undef, 1) . ' ' . $attachment->{content_type} . ') - unknown match, skipped',getlogger(__PACKAGE__)); next; } } @@ -209,15 +209,15 @@ sub _process_body { if (defined $self->{checkfilenamecode} and ref $self->{checkfilenamecode} eq 'CODE') { my $match = &{$self->{checkfilenamecode}}($attachment); if ($match == $attachment_no_match) { - attachmentdownloaderinfo('attachment ' . $attachment->{filename} . ' (' . kbytes2gigs(int($attachment->{size} / 1024), undef, 1) . ' ' . $attachment->{content_type} . ') skipped',getlogger(__PACKAGE__)); + attachmentdownloaderinfo('attachment ' . $attachment->{filename} . ' (' . humanize_bytes($attachment->{size}, undef, 1) . ' ' . $attachment->{content_type} . ') skipped',getlogger(__PACKAGE__)); next; } elsif ($match == $attachment_found) { - attachmentdownloaderinfo('attachment ' . $attachment->{filename} . ' (' . kbytes2gigs(int($attachment->{size} / 1024), undef, 1) . ' ' . $attachment->{content_type} . ') found',getlogger(__PACKAGE__)); + attachmentdownloaderinfo('attachment ' . $attachment->{filename} . ' (' . humanize_bytes($attachment->{size}, undef, 1) . ' ' . $attachment->{content_type} . ') found',getlogger(__PACKAGE__)); $found = 1; } elsif ($match == $attachment_match) { - attachmentdownloaderinfo('attachment ' . $attachment->{filename} . ' (' . kbytes2gigs(int($attachment->{size} / 1024), undef, 1) . ' ' . $attachment->{content_type} . ') matched',getlogger(__PACKAGE__)); + attachmentdownloaderinfo('attachment ' . $attachment->{filename} . ' (' . humanize_bytes($attachment->{size}, undef, 1) . ' ' . $attachment->{content_type} . ') matched',getlogger(__PACKAGE__)); } else { - attachmentdownloaderwarn('attachment ' . $attachment->{filename} . ' (' . kbytes2gigs(int($attachment->{size} / 1024), undef, 1) . ' ' . $attachment->{content_type} . ') - unknown match, skipped',getlogger(__PACKAGE__)); + attachmentdownloaderwarn('attachment ' . $attachment->{filename} . ' (' . humanize_bytes($attachment->{size}, undef, 1) . ' ' . $attachment->{content_type} . ') - unknown match, skipped',getlogger(__PACKAGE__)); next; } } diff --git a/lib/NGCP/BulkProcessor/Downloaders/IMAPAttachmentDownloader.pm b/lib/NGCP/BulkProcessor/Downloaders/IMAPAttachmentDownloader.pm index 3478cd3..087f7fa 100644 --- a/lib/NGCP/BulkProcessor/Downloaders/IMAPAttachmentDownloader.pm +++ b/lib/NGCP/BulkProcessor/Downloaders/IMAPAttachmentDownloader.pm @@ -14,8 +14,6 @@ use NGCP::BulkProcessor::LogError qw( attachmentdownloaderwarn ); -use NGCP::BulkProcessor::Utils qw(kbytes2gigs); # changemod); - use IO::Socket::SSL; use Mail::IMAPClient; use MIME::Base64; diff --git a/lib/NGCP/BulkProcessor/Logging.pm b/lib/NGCP/BulkProcessor/Logging.pm index c699ff9..dd8d84e 100644 --- a/lib/NGCP/BulkProcessor/Logging.pm +++ b/lib/NGCP/BulkProcessor/Logging.pm @@ -16,7 +16,7 @@ use Log::Log4perl qw(get_logger); use File::Basename qw(basename); -use NGCP::BulkProcessor::Utils qw(timestampdigits datestampdigits changemod chopstring trim kbytes2gigs); +use NGCP::BulkProcessor::Utils qw(timestampdigits datestampdigits changemod chopstring trim humanize_bytes); use NGCP::BulkProcessor::Array qw (contains); require Exporter; @@ -665,7 +665,7 @@ sub fileprocessingstarted { my ($file,$logger) = @_; if (defined $logger) { - $logger->info('file processing started: ' . basename($file) . ' (' . kbytes2gigs(int((-s $file)/ 1024)) . ')'); + $logger->info('file processing started: ' . basename($file) . ' (' . humanize_bytes(-s $file) . ')'); } } @@ -684,7 +684,7 @@ sub fileprocessingdone { # my ($file,$start,$blocksize,$block_n,$logger) = @_; # if (defined $logger) { # if (defined $block_n) { -# $logger->info('fetching lines from ' . basename($file) . ': ' . kbytes2gigs($block_n)); +# $logger->info('fetching lines from ' . basename($file) . ': ' . humanize_bytes($block_n)); # } else { # $logger->info('fetching lines from ' . basename($file) . ': ' . ($start + 1) . '~' . ($start + $blocksize)); # } @@ -699,7 +699,7 @@ sub lines_read { if (defined $logger) { if (defined $block_n) { if ($block_n > 0) { - $logger->info(basename($file) . ': ' . kbytes2gigs(int($block_n / 1024)) . ' read'); + $logger->info(basename($file) . ': ' . humanize_bytes($block_n) . ' read'); } } else { $logger->info(basename($file) . ': lines ' . ($start + 1) . '~' . ($start + $blocksize) . ' read'); @@ -714,7 +714,7 @@ sub processing_lines { if (defined $logger) { if (defined $block_n) { if ($block_n > 0) { - $logger->info(($enablemultithreading ? '[' . $tid . '] ' : '') . 'processing lines: ' . kbytes2gigs(int($block_n / 1024))); + $logger->info(($enablemultithreading ? '[' . $tid . '] ' : '') . 'processing lines: ' . humanize_bytes($block_n)); } } else { $logger->info(($enablemultithreading ? '[' . $tid . '] ' : '') . 'processing lines: ' . ($start + 1) . '-' . ($start + $blocksize)); diff --git a/lib/NGCP/BulkProcessor/Utils.pm b/lib/NGCP/BulkProcessor/Utils.pm index 2d01b25..5c9586b 100644 --- a/lib/NGCP/BulkProcessor/Utils.pm +++ b/lib/NGCP/BulkProcessor/Utils.pm @@ -88,6 +88,7 @@ our @EXPORT_OK = qw( get_hostfqdn getscriptpath + humanize_bytes kbytes2gigs cleanupdir fixdirpath @@ -489,12 +490,15 @@ sub getscriptpath { } my @unit_suffix = qw( + Bytes kBytes MBytes GBytes + TBytes + PBytes ); -sub kbytes2gigs { +sub humanize_bytes { my ($number, $base, $round_integer) = @_; $base = 1024 if $base <= 0; @@ -511,6 +515,14 @@ sub kbytes2gigs { return "$number $unit_suffix[$unit]"; } +sub kbytes2gigs { + my ($number, $base, $round_integer) = @_; + + $base //= 1024; + + return humanize_bytes($number * $base, $base, $round_integer); +} + sub cleanupdir { my ($dirpath,$keeproot,$filewarncode,$logger) = @_; diff --git a/t/Utils.t b/t/Utils.t index 2efc260..6b12df1 100644 --- a/t/Utils.t +++ b/t/Utils.t @@ -16,13 +16,14 @@ use strict; use warnings; -use Test::More tests => 45; +use Test::More tests => 72; use Time::Local; require_ok('NGCP::BulkProcessor::Utils'); NGCP::BulkProcessor::Utils->import(qw( zerofill + humanize_bytes kbytes2gigs secs_to_years timestampdigits @@ -42,6 +43,41 @@ is(zerofill(25, 4), '0025'); is(zerofill(1000, 4), '1000'); # Unit conversion +is(humanize_bytes(1), '1 Bytes'); +is(humanize_bytes(1024), '1 kBytes'); +is(humanize_bytes(1024 ** 2), '1 MBytes'); +is(humanize_bytes(1024 ** 3), '1 GBytes'); +is(humanize_bytes(1024 ** 4), '1 TBytes'); +is(humanize_bytes(1024 ** 5), '1 PBytes'); + +is(humanize_bytes(2), '2 Bytes'); +is(humanize_bytes(2 * 1024), '2 kBytes'); +is(humanize_bytes(2 * (1024 ** 2) + 1), '2 MBytes'); +is(humanize_bytes(2 * (1024 ** 3) + 1024 + 1), '2 GBytes'); +is(humanize_bytes(2 * (1024 ** 4) + (1024 ** 2) + 1024 + 1), '2 TBytes'); + +is(humanize_bytes(920), '920 Bytes'); +is(humanize_bytes(920 * 1024), '920 kBytes'); +is(humanize_bytes(942080 * 1024), '920 MBytes'); +is(humanize_bytes(964689920 * 1024), '920 GBytes'); + +is(humanize_bytes(920, 1000), '920 Bytes'); +is(humanize_bytes(920000, 1000), '920 kBytes'); +is(humanize_bytes(920000000, 1000), '920 MBytes'); +is(humanize_bytes(920000000000, 1000), '920 GBytes'); + +is(humanize_bytes(942172), '920.08 kBytes'); +is(humanize_bytes(965632092), '920.89 MBytes'); + +is(humanize_bytes(920920, 1000), '920.92 kBytes'); +is(humanize_bytes(920920920, 1000), '920.92 MBytes'); + +is(humanize_bytes(942172, 1024, 1), '920 kBytes'); +is(humanize_bytes(965632092, 1024, 1), '920 MBytes'); + +is(humanize_bytes(920920, 1000, 1), '920 kBytes'); +is(humanize_bytes(920920920, 1000,1 ), '920 MBytes'); + is(kbytes2gigs(1), '1 kBytes'); is(kbytes2gigs(1024), '1 MBytes'); is(kbytes2gigs(1024 ** 2), '1 GBytes');