bulk-processor/lib/NGCP/BulkProcessor/Projects/ETL/EDR/Settings.pm

package NGCP::BulkProcessor::Projects::ETL::EDR::Settings;
use strict;

## no critic

use File::Basename qw(fileparse);

use NGCP::BulkProcessor::Globals qw(
    $working_path
    $enablemultithreading
    $cpucount
    create_path
);

use NGCP::BulkProcessor::Logging qw(
    getlogger
    scriptinfo
    configurationinfo
);

use NGCP::BulkProcessor::LogError qw(
    fileerror
    filewarn
    configurationwarn
    configurationerror
);

use NGCP::BulkProcessor::LoadConfig qw(
    split_tuple
    parse_regexp
);

use NGCP::BulkProcessor::Utils qw(prompt timestampdigits threadid load_module);

use NGCP::BulkProcessor::Array qw(contains);

require Exporter;
our @ISA = qw(Exporter);
our @EXPORT_OK = qw(
    update_settings

    get_export_filename

    $ignore_period_events_unique
    $period_events_single_row_txn

    $sqlite_db_file
    $csv_dir

    check_dry

    $output_path
    $input_path

    $subscriber_profiles_export_filename_format

    $defaultsettings
    $defaultconfig

    $dry
    $skip_errors
    $force

    $export_subscriber_profiles_multithreading
    $export_subscriber_profiles_numofthreads
    $export_subscriber_profiles_blocksize
    
    $export_subscriber_profiles_joins
    $export_subscriber_profiles_conditions
    $export_subscriber_profiles_limit
   
);

our $defaultconfig = 'config.cfg';
our $defaultsettings = 'settings.cfg';

our $ignore_period_events_unique = 0;
our $period_events_single_row_txn = 1;

our $output_path = $working_path . 'output/';
our $input_path = $working_path . 'input/';
our $csv_dir = 'events';

our $subscriber_profiles_export_filename_format = undef;

our $force = 0;
our $dry = 0;
our $skip_errors = 0;

our $sqlite_db_file = 'sqlite';

our $export_subscriber_profiles_multithreading = $enablemultithreading;
our $export_subscriber_profiles_numofthreads = $cpucount;
our $export_subscriber_profiles_blocksize = 1000;

our $export_subscriber_profiles_joins = [];
our $export_subscriber_profiles_conditions = [];
our $export_subscriber_profiles_limit = undef;

sub update_settings {

    my ($data,$configfile) = @_;

    if (defined $data) {

        my $result = 1;
        my $regexp_result;

        #&$configurationinfocode("testinfomessage",$configlogger);

        $result &= _prepare_working_paths(1);

        $subscriber_profiles_export_filename_format = $data->{subscriber_profiles_export_filename} if exists $data->{subscriber_profiles_export_filename};
        get_export_filename($data->{subscriber_profiles_export_filename},$configfile);
              
        $sqlite_db_file = $data->{sqlite_db_file} if exists $data->{sqlite_db_file};
        $csv_dir = $data->{csv_dir} if exists $data->{csv_dir};

        $dry = $data->{dry} if exists $data->{dry};
        $skip_errors = $data->{skip_errors} if exists $data->{skip_errors};
        
        my $parse_result;
        ($parse_result,$export_subscriber_profiles_joins) = _parse_export_joins($data->{export_subscriber_profiles_joins},$configfile);
        $result &= $parse_result;
        ($parse_result,$export_subscriber_profiles_conditions) = _parse_export_conditions($data->{export_subscriber_profiles_conditions},$configfile);
        $result &= $parse_result;
        
        $export_subscriber_profiles_limit = $data->{export_subscriber_profiles_limit} if exists $data->{export_subscriber_profiles_limit};

        $export_subscriber_profiles_multithreading = $data->{export_subscriber_profiles_multithreading} if exists $data->{export_subscriber_profiles_multithreading};
        $export_subscriber_profiles_numofthreads = _get_numofthreads($cpucount,$data,'export_subscriber_profiles_numofthreads');
        $export_subscriber_profiles_blocksize = $data->{export_subscriber_profiles_blocksize} if exists $data->{export_subscriber_profiles_blocksize};

        $period_events_single_row_txn = $data->{period_events_single_row_txn} if exists $data->{period_events_single_row_txn};
        $ignore_period_events_unique = $data->{ignore_period_events_unique} if exists $data->{ignore_period_events_unique};
        
        return $result;
    }
    return 0;

}

sub _prepare_working_paths {

    my ($create) = @_;
    my $result = 1;
    my $path_result;

    ($path_result,$input_path) = create_path($working_path . 'input',$input_path,$create,\&fileerror,getlogger(__PACKAGE__));
    $result &= $path_result;
    ($path_result,$output_path) = create_path($working_path . 'output',$output_path,$create,\&fileerror,getlogger(__PACKAGE__));
    $result &= $path_result;

    return $result;

}

sub _get_numofthreads {
    my ($default_value,$data,$key) = @_;
    my $numofthreads = $default_value;
    $numofthreads = $data->{$key} if exists $data->{$key};
    $numofthreads = $cpucount if $numofthreads > $cpucount;
    return $numofthreads;
}

sub get_export_filename {
    my ($filename_format,$configfile) = @_;
    my $export_filename;
    my $export_format;
    if ($filename_format) {
        $export_filename = sprintf($filename_format,timestampdigits(),threadid());
        unless ($export_filename =~ /^\//) {
            $export_filename = $output_path . $export_filename;
        }
        if (-e $export_filename and (unlink $export_filename) == 0) {
            filewarn('cannot remove ' . $export_filename . ': ' . $!,getlogger(__PACKAGE__));
            $export_filename = undef;
        }
        my ($name,$path,$suffix) = fileparse($export_filename,".csv");
        if ($suffix eq '.csv') {
            $export_format = 'csv';
        } else {
            configurationerror($configfile,"$filename_format: .csv export file format required");
        }
    }
    return ($export_filename,$export_format);
}

sub _parse_export_joins {
    my ($token,$file) = @_;
    my @joins = ();
    if (defined $token and length($token) > 0) {
        foreach my $f (_split(\$token)) {
            next unless($f);
            $f =~ s/^\s*\{?\s*//;
            $f =~ s/\}\s*\}\s*$/}/;
            my ($a, $b) = split(/\s*=>\s*{\s*/, $f);
            $a =~ s/^\s*\'//;
            $a =~ s/\'$//g;
            $b =~ s/\s*\}\s*$//;
            my ($c, $d) = split(/\s*=>\s*/, $b);
            $c =~ s/^\s*\'//g;
            $c =~ s/\'\s*//;
            $d =~ s/^\s*\'//g;
            $d =~ s/\'\s*//;
            push @joins, { $a => { $c => $d } };
        }
    }
    return (1,\@joins);
}

sub _parse_export_conditions {
    my ($token,$file) = @_;
    my @conditions = ();
    if (defined $token and length($token) > 0) {
        foreach my $f (_split(\$token)) {
            next unless($f);
            $f =~ s/^\s*\{?\s*//;
            $f =~ s/\}\s*\}\s*$/}/;
            my ($a, $b) = split(/\s*=>\s*{\s*/, $f);
            $a =~ s/^\s*\'//;
            $a =~ s/\'$//g;
            $b =~ s/\s*\}\s*$//;
            my ($c, $d) = split(/\s*=>\s*/, $b);
            $c =~ s/^\s*\'//g;
            $c =~ s/\'\s*//;
            $d =~ s/^\s*\'//g;
            $d =~ s/\'\s*//;
            push @conditions, { $a => { $c => $d } };
        }
    }
    return (1,\@conditions);
}

1;