TT#51166 add custom csv header/footer format support

* csv header and footer can now be specified in the config
      as as formatted string, ${version},{rows,%04i}
      where the values are applied to it when the csv is created.
      currently supported names where optinally sprintf spec
      is supported after comma:
        - version
        - rows
        - checksum (only for footer)
        - first_seq (first sequence id)
        - last_seq (last sequence id)
        - datetime (instead of the sprintf spec,
                    strftime spec is expected)

Change-Id: Ifadfbdb1d40a3be32b869f50070c5ba3d3f63f67

NGCP/CDR/Export.pm

@@ -1,6 +1,7 @@
 package NGCP::CDR::Export;
 
 use Digest::MD5;
+use DateTime;
 use warnings;
 use strict;
 
@@ -121,7 +122,7 @@ sub chownmod {
 sub write_file {
     my (
         $lines, $dircomp, $prefix, $version, $ts, $lastseq, $suffix,
-        $format, $file_data,
+        $format, $file_data, $csv_header, $csv_footer
     ) = @_;
 
     my $fn =  sprintf('%s/%s_%s_%s_%010i.%s', $dircomp, $prefix, $version, $ts, $lastseq, $suffix);
@@ -136,7 +137,16 @@ sub write_file {
                 "'".($$file_data[3]//'')."','".($$file_data[4]//'')."'". ',' x 10);
     }
     else {
-        unshift(@{ $lines }, sprintf('%s,%04i', $version, $num));
+        my $str =
+            apply_format($csv_header, {
+                            rows      => $num,
+                            version   => $version,
+                            checksum  => undef,
+                            first_seq => $lastseq,
+                            last_seq  => $lastseq+$num,
+                            _ts       => $ts,
+            });
+        unshift(@{ $lines }, $str) if $str;
     }
 
     my $nl = "\n";
@@ -151,9 +161,17 @@ sub write_file {
     my $md5 = $ctx->hexdigest;
     if ($format eq 'kabelplus') {
         print $fd (",,'$md5'". ',' x 13 ."'md5'". ',' x 19 . "$nl");
-    }
-    else {
-        print $fd ("$md5$nl");
+    } else {
+        my $str =
+            apply_format($csv_footer, {
+                            rows      => $num,
+                            version   => $version,
+                            checksum  => $md5,
+                            first_seq => $lastseq,
+                            last_seq  => $lastseq+$num,
+                            _ts       => $ts,
+            });
+        print $fd "$str$nl" if $str;
     }
 
     NGCP::CDR::Exporter::DEBUG("$num data lines written to $tfn, checksum is $md5\n");
@@ -164,5 +182,46 @@ sub write_file {
     NGCP::CDR::Exporter::DEBUG("successfully moved $tfn to $fn\n");
 }
 
+sub apply_format {
+    my ($str, $data) = @_;
+
+    return unless $str;
+
+    my @m_formats = ();
+    my $applied = $str;
+
+    my @dt_seq = $data->{_ts} =~ /^(\d{4})(\d{2})(\d{2})(\d{2})(\d{2})(\d{2})$/;
+    my $dt_seq_idx = 0;
+    my $dt = DateTime->new(map { $_ => $dt_seq[$dt_seq_idx++] }
+                                 qw(year month day hour minute second) );
+
+    while ($str =~ /(?<!\\)(\$\{([^\$\[\]]+)\})/) {
+        my $m = {
+            'mac' => $1,
+            'inp' => $2,
+            'pos' => $-[1],
+            'len' => length($1),
+        };
+        substr($str, $m->{pos}, $m->{len}) = "##$m->{inp}#";
+        push @m_formats, $m;
+    }
+
+    foreach my $m (reverse @m_formats) {
+        my ($mac, $inp, $pos, $len) = @{$m}{qw(mac inp pos len)};
+        my ($name,$strf) = split(/,/, $inp);
+
+        my $out = '';
+        if ($name eq 'datetime') { # special handling
+            $out = $dt->strftime($strf // '%Y-%m-%d %H:%M:%S');
+        } elsif ($data->{$name}) {
+            $out = sprintf($strf // '%s', $data->{$name});
+        }
+
+        substr($applied, $pos, $len) = $out;
+    }
+
+    return $applied;
+}
+
 1;
 # vim: set tabstop=4 expandtab:

NGCP/CDR/Exporter.pm

@@ -65,6 +65,8 @@ my %config = (
     'default.QUOTES' => "'",
     'default.CSV_SEP' => ',',
     'default.CSV_ESC' => "\\",
+    'default.CSV_HEADER' => '${version},${lines,%04i}',
+    'default.CSV_FOOTER' => '${checksum}',
     'default.WRITE_EXTENDED_EXPORT_STATUS' => 0,
 );
 
@@ -419,6 +421,7 @@ sub write_wrap {
             \@filevals, $reseller_tempdir, confval('PREFIX'),
             confval('VERSION'), $file_ts, $file_idx, confval('SUFFIX'),
             confval('FILE_FORMAT') // 'default', $reseller_file_data{$reseller},
+            confval('CSV_HEADER'), confval('CSV_FOOTER')
         );
         $rec_idx -= $recs;
         delete($reseller_file_data{$reseller});