sipwise
/
ngcp-panel
mirror of https://github.com/sipwise/ngcp-panel.git
1
0
Fork 0
Code Issues Releases Wiki Activity

TT#36660 SQL proc for new billing_mappings schema

Browse Source 
+ enable filtering obsolete effective start time records
+ compare-test with perl impl

Change-Id: I3acdb5a32371f4682c0db00cb84a45dd5a9254df
changes/09/21309/5
Rene Krenn 7 years ago
parent f7f2f4d701
commit db0adbde4e
2 changed files with 325 additions and 37 deletions
Show Stats Download Patch File Download Diff File

2
lib/NGCP/Panel/Utils/BillingMappings.pm
Unescape View File

@ -40,8 +40,8 @@ sub get_actual_billing_mapping_stmt {
join billing.products product on actual_billing_mapping.product_id = product.id
where
actual_billing_mapping.contract_id = %s
and (actual_billing_mapping.end_date >= %s or actual_billing_mapping.end_date is null)
and (actual_billing_mapping.start_date <= %s or actual_billing_mapping.start_date is null)
and (actual_billing_mapping.end_date >= %s or actual_billing_mapping.end_date is null)
order by actual_billing_mapping.start_date desc, actual_billing_mapping.id desc limit 1
EOS
, $projection, $contract_id_alias, ( map { '"'.$_.'"'; } ( $dtf->format_datetime($now), $dtf->format_datetime($now) ) ));

368
sandbox/billing_mapping_effective_date.t
Unescape View File

@ -4,6 +4,7 @@ use warnings;
use Test::More;
use DateTime::Format::ISO8601 qw();
use DateTime::TimeZone qw();
use Time::HiRes qw(time);
use Tie::IxHash;
# try using the db directly ...
@ -13,6 +14,7 @@ eval '
use lib "/home/rkrenn/sipwise/git/sipwise-base/lib";
use NGCP::Schema;
';
print $@;
unless ($@) {
diag("connecting to ngcp db");
$schema = NGCP::Schema->connect({
@ -27,6 +29,10 @@ unless ($@) {
# ... or a separate csv file otherwise:
my $filename = 'api_balanceintervals_test_reference.csv';
my @perl_records = ();
my @sql_records = ();
#goto SKIP;
test_contracts(sub {
my $contract = shift;
@ -43,49 +49,295 @@ test_contracts(sub {
my $id = $mapping->{id};
$mappings{$id} = $mapping;
my $s = $mapping->{start_date};
$s = $contract->{contract_create} unless $s;
if ($s) {
$s = dt_from_string($s)->epoch;
} else {
$s = 0;
}
#$s = $contract->{contract_create} unless $s;
#$s = dt_from_string($s)->epoch;
my $e = $mapping->{end_date};
my $e_tree;
if ($e) {
$e = dt_from_string($e)->epoch;
$e_tree = $e;
} else {
$e = 2147483647;
$e_tree = 2147483647.0;
$e = $e_tree - 0.001;
}
$tree->insert($s,$e,$id);
$event_list->Push($s => $id);
$event_list->Push($e => $id);
$tree->insert($s,$e_tree,$id);
$mapping->{"s"} = $s;
$mapping->{"e"} = $e;
$event_list->Push($s."-0" => $id);
$event_list->Push($e."-1" => $id);
}
#if ($contract->{contract_id} == 89) {
# print "blah";
#}
# 2. sort events by time ascending:
$event_list->Reorder( sort { $a <=> $b } $event_list->Keys );
$event_list->Reorder( sort { my ($t_a,$is_end_a) = split(/-/,$a); my ($t_b,$is_end_b) = split(/-/,$b); $t_a <=> $t_b || $is_end_a <=> $is_end_b; } $event_list->Keys );
# 3. generate the "effective start" list by determining the mappings effective at any event time:
my $effective_start_list = create_linked_hash();
foreach my $t ($event_list->Keys) {
my $msec = 0.000;
foreach my $id (sort { $a <=> $b } @{$tree->find($t)}) { # sort by max(billing_mapping_id)
if ($effective_start_list->EXISTS($t + $msec)) {
die("MUST NOT HAPPEN");
} else {
$effective_start_list->Push(($t + $msec) => $mappings{$id});
$msec += 0.001; # to allow unique effective start times per contract, we use microsecond resolution
}
}
}
# 4. done, save it.
my @effective_start_list = ();
my $old_bm_ids = '';
foreach my $se ($event_list->Keys) {
my ($t,$is_end) = split(/-/,$se);
my @group = ();
my $max_bm_id;
my $bm_ids = "";
my $max_s;
foreach my $id (sort { $mappings{$b}->{"s"} <=> $mappings{$a}->{"s"} || $mappings{$a}->{id} <=> $mappings{$b}->{id}; } @{$tree->find($t)}) { # sort by max(billing_mapping_id)
my $mapping = $mappings{$id};
if ($is_end) {
next if $mapping->{"e"} == $t;
}
$max_s = $mapping->{"s"} unless defined $max_s;
last unless $max_s == $mapping->{"s"};
my $row = {
contract_id => $contract->{contract_id},
billing_mapping_id => $id,
"last" => 0,
start_date => ($mapping->{start_date} ? $mapping->{start_date} : undef),
end_date => ($mapping->{end_date} ? $mapping->{end_date} : undef),
effective_start_date => sprintf("%.3f",($is_end ? $t + 0.001 : $t)),
profile_id => $mapping->{profile_id},
network_id => ($mapping->{network_id} ? $mapping->{network_id} : undef),
};
push(@group,$row);
$max_bm_id = $id;
$bm_ids .= '-' . $id;
}
foreach my $row (@group) {
$row->{"last"} = ($max_bm_id == $row->{billing_mapping_id} ? 1 : 0);
}
if ($old_bm_ids ne $bm_ids) {
push(@effective_start_list,@group);
}
$old_bm_ids = $bm_ids;
}
# 4. done (dump the list to db).
# 5. test it with actual billing mapping impl:
my @past_mappings = ();
foreach my $t ($effective_start_list->Keys) {
if ($t <= $contract->{now}) {
push(@past_mappings,$effective_start_list->FETCH($t));
}
}
is(pop(@past_mappings)->{id},$contract->{bm_actual_id},"xxxx");
test_events("perl impl - ",$contract,sub {
my $now = shift;
my $bm_actual_id;
foreach my $row (@effective_start_list) {
next unless $row->{"last"};
last if $row->{effective_start_date} > $now;
$bm_actual_id = $row->{billing_mapping_id};
}
return $bm_actual_id;
},\@effective_start_list);
push(@perl_records,@effective_start_list);
});
SKIP:
if ($schema) {
$schema->storage->dbh_do(sub {
my ($storage, $dbh, @args) = @_;
$dbh->do('use billing');
$dbh->do(<<EOS1
create temporary table tmp_transformed (
contract_id int(11) unsigned,
billing_mapping_id int(11) unsigned,
last tinyint(3),
start_date datetime,
end_date datetime,
effective_start_date decimal(13,3),
profile_id int(11) unsigned,
network_id int(11) unsigned,
key tmp_cid_esd_last_idx (contract_id,effective_start_date,last)
);
EOS1
);
$dbh->do('drop procedure if exists transform_billing_mappings');
#$dbh->do('delimiter ;;');
$dbh->do(<<EOS2
create procedure transform_billing_mappings() begin
declare _contracts_done, _events_done, _mappings_done, _is_end boolean default false;
declare _contract_id, _bm_id, _default_bm_id, _profile_id, _network_id int(11) unsigned;
declare _t, _start_date, _end_date datetime;
declare _effective_start_time decimal(13,3);
declare _bm_ids, _old_bm_ids varchar(65535);
declare contracts_cur cursor for select contract_id
from billing_mappings bm group by contract_id;
declare continue handler for not found set _contracts_done = true;
set _old_bm_ids = "";
open contracts_cur;
contracts_loop: loop
fetch contracts_cur into _contract_id;
if _contracts_done then
leave contracts_loop;
end if;
nested1: begin
declare events_cur cursor for select t,is_end from (
# (select coalesce(bm.start_date,if(c.create_timestamp = "0000-00-00 00:00:00",c.modify_timestamp,c.create_timestamp)) as t, 0 as is_end
(select coalesce(bm.start_date,from_unixtime(0)) as t, 0 as is_end
from billing_mappings bm join contracts c on bm.contract_id = c.id where contract_id = _contract_id)
union all
(select coalesce(end_date,from_unixtime(2147483647) - 0.001) as t, 1 as is_end from billing_mappings where contract_id = _contract_id)
) as events group by t, is_end order by t, is_end;
declare continue handler for not found set _events_done = true;
set _events_done = false;
open events_cur;
events_loop: loop
fetch events_cur into _t, _is_end;
if _events_done then
leave events_loop;
end if;
nested2: begin
declare mappings_cur cursor for select bm1.id, bm1.start_date, bm1.end_date, bm1.billing_profile_id, bm1.network_id from
billing_mappings bm1 where bm1.contract_id = _contract_id and bm1.start_date <=> (select bm2.start_date
from billing_mappings bm2 where
bm2.contract_id = _contract_id
and (bm2.start_date <= _t or bm2.start_date is null)
and (if(_is_end,bm2.end_date > _t,bm2.end_date >= _t) or bm2.end_date is null)
order by bm2.start_date desc limit 1) order by bm1.id asc;
declare continue handler for not found set _mappings_done = true;
set _effective_start_time = (select unix_timestamp(if(_is_end,_t + 0.001,_t)));
set _bm_ids = "";
set _mappings_done = false;
open mappings_cur;
mappings_loop1: loop
fetch mappings_cur into _bm_id, _start_date, _end_date, _profile_id, _network_id;
if _mappings_done then
leave mappings_loop1;
end if;
set _bm_ids = (select concat(_bm_ids,"-",_bm_id));
set _default_bm_id = _bm_id;
end loop mappings_loop1;
close mappings_cur;
if _old_bm_ids != _bm_ids then
set _mappings_done = false;
open mappings_cur;
mappings_loop2: loop
fetch mappings_cur into _bm_id, _start_date, _end_date, _profile_id, _network_id;
if _mappings_done then
leave mappings_loop2;
end if;
#INSERT......
#select _contract_id,_effective_start_time,_profile_id, _network_id;
insert into tmp_transformed values(_contract_id,_bm_id,if(_bm_id = _default_bm_id,1,0),_start_date,_end_date,_effective_start_time,_profile_id,_network_id);
#set _effective_start_time = _effective_start_time + 0.001;
end loop mappings_loop2;
close mappings_cur;
end if;
set _old_bm_ids = _bm_ids;
end nested2;
end loop events_loop;
close events_cur;
end nested1;
end loop contracts_loop;
close contracts_cur;
end;;
EOS2
);
#$dbh->do('delimiter ;');
my $t1 = time();
$dbh->do('call transform_billing_mappings()');
diag("time to transform all billing_mappings: ".sprintf("%.3f secs",time()-$t1));
$dbh->do('drop procedure transform_billing_mappings');
},);
goto SKIP1;
test_contracts(sub {
my $contract = shift;
$schema->storage->dbh_do(sub {
my ($storage, $dbh, @args) = @_;
#my $sth = $dbh->prepare("select from_unixtime(tr.effective_start_date) as effective_start_date_epoch,tr.* from tmp_transformed tr where tr.contract_id = ? order by tr.effective_start_date asc");
my $sth = $dbh->prepare("select * from tmp_transformed tr where tr.contract_id = ? order by tr.effective_start_date asc");
$sth->execute($contract->{contract_id});
my $mappings = $sth->fetchall_arrayref({});
$sth->finish();
test_events("sql impl - ",$contract,sub {
my $now = shift;
my $sth = $dbh->prepare("select max(effective_start_date) from tmp_transformed where contract_id = ? and effective_start_date <= ? and last = 1");
$sth->execute($contract->{contract_id},$now);
my ($effective_start_date) = $sth->fetchrow_array();
$sth = $dbh->prepare("select billing_mapping_id from tmp_transformed where contract_id = ? and effective_start_date = ? and last = 1");
$sth->execute($contract->{contract_id},$effective_start_date);
my ($bm_actual_id) = $sth->fetchrow_array();
$sth->finish();
unless (defined $bm_actual_id) {
$sth = $dbh->prepare("select min(billing_mapping_id) from tmp_transformed where contract_id = ? and last = 1");
$sth->execute($contract->{contract_id});
($bm_actual_id) = $sth->fetchrow_array();
$sth->finish();
}
return $bm_actual_id;
},$mappings);
push(@sql_records,@$mappings);
},);
});
{
is_deeply(\@perl_records,\@sql_records,"compare generated perl and sql effective start date records deeply");
}
SKIP1:
{
my $now = DateTime->now(
time_zone => DateTime::TimeZone->new(name => 'local')
);
my $t1;
my $billing_mappings_actual_new;
$schema->storage->dbh_do(sub {
my ($storage, $dbh, @args) = @_;
$dbh->do("create temporary table tmp_transformed_copy like tmp_transformed");
$dbh->do("insert into tmp_transformed_copy select * from tmp_transformed");
my $sth = $dbh->prepare(<<EOS3
select t2.contract_id,t2.billing_mapping_id from
(select
contract_id,
max(effective_start_date) as effective_start_date
from tmp_transformed
where effective_start_date <= ?
and last = 1
group by contract_id) as t1
join tmp_transformed_copy t2 on t2.contract_id = t1.contract_id and t2.effective_start_date = t1.effective_start_date and t2.last = 1
EOS3
);
$t1 = time();
$sth->execute($now->epoch); #,$contract->{contract_id});
#my $t2 = Time::Hires::time();
$billing_mappings_actual_new = $sth->fetchall_arrayref();
$sth->finish();
diag("new query (".(scalar @$billing_mappings_actual_new)." mappings): ".sprintf("%.3f secs",time()-$t1));
});
my $dtf = $schema->storage->datetime_parser;
$t1 = time();
my @billing_mappings_actual_old = map { my %res = $_->get_inflated_columns; [ @res{qw(contract_id actual_bm_id)} ]; } $schema->resultset('billing_mappings_actual')->search_rs(undef,{
bind => [ ( $dtf->format_datetime($now) ) x 2, ( undef ) x 2 ],
})->all;
diag("old query (".(scalar @$billing_mappings_actual_new)." mappings): ".sprintf("%.3f secs",time()-$t1));
is_deeply($billing_mappings_actual_new,\@billing_mappings_actual_old,"compare actual_billing_mapping table deeply");
}
}
done_testing;
sub create_linked_hash {
@ -93,6 +345,35 @@ sub create_linked_hash {
return tie(%hash, 'Tie::IxHash');
}
sub test_events {
my ($label,$contract,$get_actual_billing_mapping,$mappings) = @_;
my $event_list = create_linked_hash();
foreach my $mapping (@{$contract->{mappings}}) {
my $id = $mapping->{id};
my $s = $mapping->{start_date};
$s = $contract->{contract_create} unless $s;
my $e = $mapping->{end_date};
$e = dt_to_string(DateTime->from_epoch(epoch => 2147483647)) unless $e;
$event_list->Push($s => $id);
$event_list->Push($e => $id);
}
foreach ($event_list->Keys) {
my $dt = dt_from_string($_);
my $i = -1;
foreach my $dt (dt_from_string($_)->subtract(seconds => 1),dt_from_string($_),dt_from_string($_)->add(seconds => 1)) {
unless (is(&$get_actual_billing_mapping($dt->epoch),get_actual_billing_mapping($schema,$contract->{contract_id},$dt),
$label."contract $contract->{contract_id} billing mapping id at t".($i<0?$i:"+$i")." = $dt")) {
foreach my $row (@$mappings) {
print join("\t",(map { "$_=".((not defined $row->{$_}) ? "\t" : $row->{$_}); } sort keys %$row)) . "\n";
#print $mapping[0]."\t".$mapping[1]."\t".$mapping[2]."\t".$mapping->{end_date}."\t".$mapping->{profile_id}."\t".$mapping->{network_id}."\n";
}
}
$i++;
}
}
}
sub test_contracts {
my $code = shift;
@ -102,20 +383,13 @@ sub test_contracts {
my $now = DateTime->now(
time_zone => DateTime::TimeZone->new(name => 'local')
);
my $dtf = $schema->storage->datetime_parser;
#my $dtf = $schema->storage->datetime_parser;
while (my @page = $contract_rs->search_rs(undef,{
page => $page,
rows => 100,
})->all) {
foreach my $contract (@page) {
my $bm_actual_id = $schema->resultset('contracts')->search_rs({
id => $contract->id,
},{
bind => [ ( $dtf->format_datetime($now) ) x 2, ( $contract->id ) x 2 ],
'join' => 'billing_mappings_actual',
'+select' => [ 'billing_mappings_actual.actual_bm_id' ],
'+as' => [ 'billing_mapping_id' ],
})->first->get_column("billing_mapping_id");
my $bm_actual_id = get_actual_billing_mapping($schema,$contract->id,$now);
&$code({
now => $now->epoch,
@ -221,6 +495,20 @@ sub test_contracts {
}
sub get_actual_billing_mapping {
my ($schema, $contract_id, $now) = @_;
my $dtf = $schema->storage->datetime_parser;
return $schema->resultset('contracts')->search_rs({
id => $contract_id,
},{
bind => [ ( $dtf->format_datetime($now) ) x 2, ( $contract_id ) x 2 ],
'join' => 'billing_mappings_actual',
'+select' => [ 'billing_mappings_actual.actual_bm_id' ],
'+as' => [ 'billing_mapping_id' ],
})->first->get_column("billing_mapping_id");
}
sub dt_to_string {
my ($dt) = @_;
return '' unless defined ($dt);
@ -429,8 +717,8 @@ sub dt_from_string {
}
sub intersect {
my ( $self, $start, $end, $sort ) = @_;
$sort = 1 if !defined $sort;
my ( $self, $start, $end ) = @_;
#$sort = 1 if !defined $sort;
my $results = [];
$self->_intersect( $start, $end, $results );
return $results;
@ -453,8 +741,8 @@ sub dt_from_string {
}
sub find {
my ( $self, $t, $sort ) = @_;
$sort = 1 if !defined $sort;
my ( $self, $t ) = @_;
#$sort = 1 if !defined $sort;
my $results = [];
$self->_find( $t, $results );
return $results;

Write Preview
Loading…
Cancel
Save