sipwise
/
bulk-processor
mirror of https://github.com/sipwise/bulk-processor.git
1
0
Fork 0
Code Issues Releases Wiki Activity

TT#18881 implement teletek importer #4

Browse Source 
+ "allowed_cli" import gets a different meaning,
  it are additional "numbers" (alias+allowed_cli) to import

+ unique cc+ac+sn in subscriber file
+ unique cc+ac+sn in allowed_cli file

+ logging of cc+ac+sn of allowed_cli records
  already found in subscriber file
+ logging of sip_usernames of allowed_cli records
  not found in subscriber file

+ reseller mapping

Change-Id: I620db415f5dab91e8d5c6336b7a45b3a0ffeacb9
changes/14/15314/2
Rene Krenn 8 years ago
parent 7dbb015f27
commit 275afbbfb3
8 changed files with 138 additions and 22 deletions
Show Stats Download Patch File Download Diff File

2
lib/NGCP/BulkProcessor/Projects/Migration/Teletek/Dao/import/AllowedCli.pm
Unescape View File

@ -158,7 +158,7 @@ sub findby_ccacsn {
}
sub findby_domain_sipusername {
sub findby_sipusername {
my ($sip_username,$load_recursive) = @_;

21
lib/NGCP/BulkProcessor/Projects/Migration/Teletek/Dao/import/Subscriber.pm
Unescape View File

@ -38,6 +38,7 @@ our @EXPORT_OK = qw(
findby_domain_sipusername
findby_domain_webusername
list_domain_billingprofilename_resellernames
findby_sipusername
update_delta
findby_delta
@ -204,6 +205,26 @@ sub findby_domain_sipusername {
}
sub findby_sipusername {
my ($sip_username,$load_recursive) = @_;
check_table();
my $db = &$get_db();
my $table = $db->tableidentifier($tablename);
#return [] unless (defined $cc or defined $ac or defined $sn);
my $rows = $db->db_get_all_arrayref(
'SELECT * FROM ' .
$table .
' WHERE ' . $db->columnidentifier('sip_username') . ' = ?'
,$sip_username);
return buildrecords_fromrows($rows,$load_recursive);
}
sub findby_domain_webusername {
my ($domain,$web_username,$load_recursive) = @_;

36
lib/NGCP/BulkProcessor/Projects/Migration/Teletek/Import.pm
Unescape View File

@ -273,6 +273,26 @@ sub import_allowedcli {
$rownum++;
next if (scalar @$row) == 0;
my $record = NGCP::BulkProcessor::Projects::Migration::Teletek::Dao::import::AllowedCli->new($row);
if (NGCP::BulkProcessor::Projects::Migration::Teletek::Dao::import::Subscriber::findby_ccacsn($record->{cc},$record->{ac},$record->{sn})) {
my $number = ($record->{cc} // '') . ($record->{ac} // '') . ($record->{sn} // '');
if ($skip_errors) {
_warn($context,"duplicate number: $number");
} else {
_error($context,"duplicate number: $number");
}
next;
}
if ((scalar @{NGCP::BulkProcessor::Projects::Migration::Teletek::Dao::import::Subscriber::findby_sipusername($record->{sip_username})}) == 0) {
if ($skip_errors) {
_warn($context,"sip username $record->{sip_username} not found");
} else {
_error($context,"sip username $record->{sip_username} not found");
}
next;
}
$record->{rownum} = $rownum;
my %r = %$record; my @allowedcli_row = @r{@NGCP::BulkProcessor::Projects::Migration::Teletek::Dao::import::AllowedCli::fieldnames};
if ($context->{upsert}) {
@ -333,14 +353,14 @@ sub import_allowedcli {
sub _import_allowedcli_checks {
my ($file) = @_;
my $result = 1;
#my $optioncount = 0;
#eval {
# $optioncount = NGCP::BulkProcessor::Projects::Migration::Teletek::Dao::import::FeatureOption::countby_subscribernumber_option();
#};
#if ($@ or $optioncount == 0) {
# fileprocessingerror($file,'please import subscriber features first',getlogger(__PACKAGE__));
# $result = 0; #even in skip-error mode..
#}
my $subscribercount = 0;
eval {
$subscribercount = NGCP::BulkProcessor::Projects::Migration::Teletek::Dao::import::Subscriber::countby_ccacsn();
};
if ($@ or $subscribercount == 0) {
fileprocessingerror($file,'please import subscribers first',getlogger(__PACKAGE__));
$result = 0; #even in skip-error mode..
}
#my $userpasswordcount = 0;
#eval {
# $userpasswordcount = NGCP::BulkProcessor::Projects::Migration::Teletek::Dao::import::UsernamePassword::countby_fqdn();

50
lib/NGCP/BulkProcessor/Projects/Migration/Teletek/Provisioning.pm
Unescape View File

@ -17,6 +17,8 @@ use NGCP::BulkProcessor::Projects::Migration::Teletek::Settings qw(
$provision_subscriber_multithreading
$provision_subscriber_numofthreads
$reseller_mapping
);
#$batch
@ -42,7 +44,7 @@ use NGCP::BulkProcessor::LogError qw(
);
use NGCP::BulkProcessor::Projects::Migration::Teletek::Dao::import::Subscriber qw();
#use NGCP::BulkProcessor::Projects::Migration::Teletek::Dao::import::AllowedCli qw();
use NGCP::BulkProcessor::Projects::Migration::Teletek::Dao::import::AllowedCli qw();
use NGCP::BulkProcessor::Dao::Trunk::billing::billing_profiles qw();
use NGCP::BulkProcessor::Dao::Trunk::billing::products qw();
@ -231,7 +233,7 @@ sub _provision_subscribers_checks {
rowprocessingerror(threadid(),"empty billing profile name detected",getlogger(__PACKAGE__));
$result = 0; #even in skip-error mode..
}
my $resellername = $domain_billingprofilename_resellername->{reseller_name};
my $resellername = _apply_reseller_mapping($domain_billingprofilename_resellername->{reseller_name});
unless ($resellername) {
rowprocessingerror(threadid(),"empty reseller name detected",getlogger(__PACKAGE__));
$result = 0; #even in skip-error mode..
@ -617,7 +619,7 @@ sub _provision_susbcriber_init_context {
}
$context->{domain} = $context->{domain_map}->{$first->{domain}};
$context->{reseller} = $context->{reseller_map}->{$first->{reseller_name}};
$context->{reseller} = $context->{reseller_map}->{_apply_reseller_mapping($first->{reseller_name})};
$context->{billing_profile} = $context->{reseller}->{billingprofile_map}->{$first->{billing_profile_name}};
$context->{prov_subscriber} = {};
@ -631,7 +633,7 @@ sub _provision_susbcriber_init_context {
@{NGCP::BulkProcessor::Projects::Migration::Teletek::Dao::import::Subscriber::findby_domain_webusername(
$first->{domain},$context->{prov_subscriber}->{webusername})};
if ((scalar keys %webusername_dupes) > 1) {
_warn($context,"duplicate web_username $context->{prov_subscriber}->{webusername}, using sip_username");
#_warn($context,"duplicate web_username $context->{prov_subscriber}->{webusername}, using sip_username");
$context->{prov_subscriber}->{webusername} = $first->{sip_username};
}
}
@ -658,10 +660,11 @@ sub _provision_susbcriber_init_context {
sn => $subscriber->{sn} // '',
number => $number,
delta => $subscriber->{delta},
additional => 0,
});
$number_dupes{$number} = 1;
} else {
_warn($context,'duplicate number $number ignored');
_warn($context,'duplicate number $number (subscriber table) ignored');
}
if (not exists $contact_dupes{$subscriber->{contact_hash}}) {
@ -692,8 +695,37 @@ sub _provision_susbcriber_init_context {
}
}
foreach my $allowed_cli (@{NGCP::BulkProcessor::Projects::Migration::Teletek::Dao::import::AllowedCli::findby_sipusername($first->{sip_username})}) {
my $number = ($allowed_cli->{cc} // '') . ($allowed_cli->{ac} // '') . ($allowed_cli->{sn} // '');
if (not exists $number_dupes{$number}) {
push(@numbers,{
cc => $allowed_cli->{cc} // '',
ac => $allowed_cli->{ac} // '',
sn => $allowed_cli->{sn} // '',
number => $number,
delta => $allowed_cli->{delta},
additional => 1,
});
$number_dupes{$number} = 1;
} else {
_warn($context,'duplicate number $number (allowed_cli table) ignored');
}
}
$context->{numbers} = {};
$context->{numbers}->{other} = sort_by_configs(\@numbers,[
{ numeric => 1,
dir => 1, #-1,
memberchain => [ 'additional' ],
},
{ numeric => 0,
dir => 1, #-1,
memberchain => [ 'cc' ],
},
{ numeric => 0,
dir => 1, #-1,
memberchain => [ 'ac' ],
},
{ numeric => 0,
dir => 1, #-1,
memberchain => [ 'sn' ],
@ -727,6 +759,14 @@ sub _provision_susbcriber_init_context {
# );
#}
sub _apply_reseller_mapping {
my $reseller_name = shift;
if (defined $reseller_name and exists $reseller_mapping->{$reseller_name}) {
return $reseller_mapping->{$reseller_name};
}
return $reseller_name;
}
sub _error {
my ($context,$message) = @_;

30
lib/NGCP/BulkProcessor/Projects/Migration/Teletek/Settings.pm
Unescape View File

@ -32,6 +32,7 @@ require Exporter;
our @ISA = qw(Exporter);
our @EXPORT_OK = qw(
update_settings
update_reseller_mapping
check_dry
$input_path
@ -52,7 +53,8 @@ our @EXPORT_OK = qw(
$ignore_subscriber_unique
$subscriber_import_single_row_txn
$subscriber_import_unfold_ranges
$reseller_mapping_yml
$reseller_mapping
@allowedcli_filenames
$allowedcli_import_numofthreads
@ -106,6 +108,8 @@ our $subscriber_import_numofthreads = $cpucount;
our $ignore_subscriber_unique = 0;
our $subscriber_import_single_row_txn = 1;
our $subscriber_import_unfold_ranges = 1;
our $reseller_mapping_yml = undef;
our $reseller_mapping = {};
our @allowedcli_filenames = ();
our $allowedcli_import_numofthreads = $cpucount;
@ -163,6 +167,7 @@ sub update_settings {
$ignore_subscriber_unique = $data->{ignore_subscriber_unique} if exists $data->{ignore_subscriber_unique};
$subscriber_import_single_row_txn = $data->{subscriber_import_single_row_txn} if exists $data->{subscriber_import_single_row_txn};
$subscriber_import_unfold_ranges = $data->{subscriber_import_unfold_ranges} if exists $data->{subscriber_import_unfold_ranges};
$reseller_mapping_yml = $data->{reseller_mapping_yml} if exists $data->{reseller_mapping_yml};
@allowedcli_filenames = _get_import_filenames(\@allowedcli_filenames,$data,'allowedcli_filenames');
$allowedcli_import_numofthreads = _get_numofthreads($cpucount,$data,'allowedcli_import_numofthreads');
@ -284,4 +289,27 @@ sub check_dry {
}
sub update_reseller_mapping {
my ($data,$configfile) = @_;
if (defined $data) {
my $result = 1;
eval {
$reseller_mapping = $data->{'mapping'};
};
if ($@ or 'HASH' ne ref $reseller_mapping) { # or (scalar keys %$reseller_mapping) == 0) {
$reseller_mapping //= {};
configurationerror($configfile,'invalid reseller mapping',getlogger(__PACKAGE__));
$result = 0;
}
return $result;
}
return 0;
}
1;

10
lib/NGCP/BulkProcessor/Projects/Migration/Teletek/process.pl
Unescape View File

@ -12,6 +12,7 @@ use Fcntl qw(LOCK_EX LOCK_NB);
use NGCP::BulkProcessor::Globals qw();
use NGCP::BulkProcessor::Projects::Migration::Teletek::Settings qw(
update_settings
update_reseller_mapping
check_dry
$output_path
$defaultsettings
@ -21,7 +22,7 @@ use NGCP::BulkProcessor::Projects::Migration::Teletek::Settings qw(
$force
$run_id
@subscriber_filenames
$reseller_mapping_yml
@allowedcli_filenames
);
@ -161,6 +162,7 @@ sub init {
my $result = load_config($configfile);
init_log();
$result &= load_config($settingsfile,\&update_settings,$SIMPLE_CONFIG_TYPE);
$result &= load_config($reseller_mapping_yml,\&update_reseller_mapping,$YAML_CONFIG_TYPE);
return $result;
}
@ -316,7 +318,7 @@ sub import_subscriber_task {
($result,$warning_count) = import_subscriber(@subscriber_filenames);
};
my $err = $@;
my $stats = ($skip_errors ? ": $warning_count warnings" : '');
my $stats = ": $warning_count warnings";
eval {
$stats .= "\n total subscriber records: " .
NGCP::BulkProcessor::Projects::Migration::Teletek::Dao::import::Subscriber::countby_ccacsn() . ' rows';
@ -378,7 +380,7 @@ sub import_allowedcli_task {
($result,$warning_count) = import_allowedcli(@allowedcli_filenames);
};
my $err = $@;
my $stats = ($skip_errors ? ": $warning_count warnings" : '');
my $stats = ": $warning_count warnings";
eval {
$stats .= "\n total allowed cli records: " .
NGCP::BulkProcessor::Projects::Migration::Teletek::Dao::import::AllowedCli::countby_ccacsn() . ' rows';
@ -439,7 +441,7 @@ sub create_subscriber_task {
($result,$warning_count) = provision_subscribers();
};
my $err = $@;
my $stats = ($skip_errors ? ": $warning_count warnings" : '');
my $stats = ": $warning_count warnings";
eval {
#$stats .= "\n total contracts: " .
# NGCP::BulkProcessor::Dao::Trunk::billing::contracts::countby_status_resellerid(undef,$reseller_id) . ' rows';

4
lib/NGCP/BulkProcessor/Projects/Migration/Teletek/reseller_mapping.yml
Unescape View File

@ -0,0 +1,4 @@
mapping:
0700: 'Teleman'
Digifon: 'Teleman'
RingEtt: 'Teleman'

7
lib/NGCP/BulkProcessor/Projects/Migration/Teletek/settings.cfg
Unescape View File

@ -10,16 +10,17 @@ import_multithreading = 0
#subscriber_filename = /home/rkrenn/temp/teletek/export_nummer_sip3.csv
subscriber_filenames = /home/rkrenn/temp/teletek/export_kundinfo_leica.170823.csv,/home/rkrenn/temp/teletek/export_nummer_sip3.170904.csv
subscriber_import_numofthreads = 2
ignore_subscriber_unique = 1
ignore_subscriber_unique = 0
subscriber_import_single_row_txn = 1
subscriber_import_unfold_ranges = 1
reseller_mapping_yml = reseller_mapping.yml
#allowedcli_filename = /home/rkrenn/test/teletek/export_multiple_DID_Leica.csv
#allowedcli_filename = /home/rkrenn/temp/teletek/export_screeningOnly_170824.csv
allowedcli_filenames = /home/rkrenn/temp/teletek/export_MultipleDID_170823.csv
allowedcli_filenames = /home/rkrenn/temp/teletek/export_MultipleDID_170823.csv,/home/rkrenn/temp/teletek/export_screeningOnly_170824.csv
allowedcli_import_numofthreads = 2
ignore_allowedcli_unique = 1
ignore_allowedcli_unique = 0
allowedcli_import_single_row_txn = 1

Write Preview
Loading…
Cancel
Save