TT#32972 Fetch location data from redis

Change-Id: Id4edc72e2e7748ff6e9adb4c3370720232065ab6
changes/28/19228/8
Andreas Granig 7 years ago
parent d30caf3d27
commit 3a1d30bee6

@ -87,7 +87,7 @@ sub GET :Allow {
push @embedded, $halitem;
push @links, Data::HAL::Link->new(
relation => 'ngcp:'.$self->resource_name,
href => sprintf('/%s%d', $c->request->path, $item->id),
href => sprintf('/%s%s', $c->request->path, $item->id),
);
}
push @links,

@ -102,7 +102,7 @@ sub PATCH :Allow {
if ('minimal' eq $preference) {
$c->response->status(HTTP_NO_CONTENT);
$c->response->header(Preference_Applied => 'return=minimal');
$c->response->header(Location => sprintf('%s%d', $self->dispatch_path, $item->id));
$c->response->header(Location => sprintf('%s%s', $self->dispatch_path, $item->id));
$c->response->body(q());
} else {
my $hal = $self->hal_from_item($c, $item, $form);
@ -111,7 +111,7 @@ sub PATCH :Allow {
), $hal->as_json);
$c->response->headers($response->headers);
$c->response->header(Preference_Applied => 'return=representation');
$c->response->header(Location => sprintf('%s%d', $self->dispatch_path, $item->id));
$c->response->header(Location => sprintf('%s%s', $self->dispatch_path, $item->id));
$c->response->body($response->content);
}
}
@ -158,7 +158,7 @@ sub PUT :Allow {
if ('minimal' eq $preference) {
$c->response->status(HTTP_NO_CONTENT);
$c->response->header(Preference_Applied => 'return=minimal');
$c->response->header(Location => sprintf('%s%d', $self->dispatch_path, $item->id));
$c->response->header(Location => sprintf('%s%s', $self->dispatch_path, $item->id));
$c->response->body(q());
} else {
my $hal = $self->hal_from_item($c, $item, $form);
@ -167,7 +167,7 @@ sub PUT :Allow {
), $hal->as_json);
$c->response->headers($response->headers);
$c->response->header(Preference_Applied => 'return=representation');
$c->response->header(Location => sprintf('%s%d', $self->dispatch_path, $item->id));
$c->response->header(Location => sprintf('%s%s', $self->dispatch_path, $item->id));
$c->response->body($response->content);
}
}

@ -3608,16 +3608,13 @@ sub ajax_call_details :Chained('master') :PathPart('calls/ajax') :Args(1) {
sub ajax_registered :Chained('master') :PathPart('registered/ajax') :Args(0) {
my ($self, $c) = @_;
my $s = $c->stash->{subscriber}->provisioning_voip_subscriber;
my $reg_rs = $c->model('DB')->resultset('location')->search({
username => $s->username,
});
if($c->config->{features}->{multidomain}) {
$reg_rs = $reg_rs->search({
domain => $s->domain->domain,
});
}
my $reg_rs = NGCP::Panel::Utils::Subscriber::get_subscriber_location_rs(
$c,
{
username => $c->stash->{subscriber}->username,
$c->config->{features}->{multidomain} ? (domain => $c->stash->{subscriber}->domain->domain) : (),
}
);
NGCP::Panel::Utils::Datatables::process($c, $reg_rs, $c->stash->{reg_dt_columns});
$c->detach( $c->view("JSON") );
@ -3862,15 +3859,12 @@ sub registered :Chained('master') :PathPart('registered') :CaptureArgs(1) {
my ($self, $c, $reg_id) = @_;
my $s = $c->stash->{subscriber}->provisioning_voip_subscriber;
my $reg_rs = $c->model('DB')->resultset('location')->search({
id => $reg_id,
username => $s->username,
});
if($c->config->{features}->{multidomain}) {
$reg_rs = $reg_rs->search({
domain => $s->domain->domain,
});
}
$c->log->error("+++++ getting subscriber location rs");
my $reg_rs = NGCP::Panel::Utils::Subscriber::get_subscriber_location_rs(
$c, { id => $reg_id },
);
$c->stash->{registered} = $reg_rs->first;
unless($c->stash->{registered}) {
NGCP::Panel::Utils::Message::error(

@ -11,27 +11,48 @@ use Data::HAL qw();
use Data::HAL::Link qw();
use HTTP::Status qw(:constants);
use NGCP::Panel::Utils::Kamailio;
use NGCP::Panel::Utils::Subscriber;
sub _item_rs {
my ($self, $c) = @_;
my @joins = ();;
if($c->config->{features}->{multidomain}) {
push @joins, 'domain';
}
my $item_rs = $c->model('DB')->resultset('location');
if($c->user->roles eq "admin") {
$item_rs = $item_rs->search({
my $item_rs;
if ($c->config->{redis}->{usrloc}) {
# TODO: will this survive with like 1M records?
my $filter = {};
if ($c->req->param('subscriber_id')) {
my $sub = $c->model('DB')->resultset('voip_subscribers')->find($c->req->param('subscriber_id'));
if ($sub) {
$filter->{username} = $sub->username;
$filter->{domain} = $sub->domain->domain;
}
}
if ($c->user->roles eq "admin") {
} elsif ($c->user->roles eq "reseller") {
$filter->{reseller_id} = $c->user->reseller_id;
}
$item_rs = NGCP::Panel::Utils::Subscriber::get_subscriber_location_rs($c, $filter);
} else {
my @joins = ();
if($c->config->{features}->{multidomain}) {
push @joins, 'domain';
}
$item_rs = $c->model('DB')->resultset('location');
if($c->user->roles eq "admin") {
$item_rs = $item_rs->search({
},{
join => [@joins,'subscriber'],
});
} elsif($c->user->roles eq "reseller") {
$item_rs = $item_rs->search({
'contact.reseller_id' => $c->user->reseller_id
},{
join => [@joins, { 'subscriber' => { 'voip_subscriber' => { 'contract' => 'contact' }}} ],
});
},{
join => [@joins,'subscriber'],
});
} elsif($c->user->roles eq "reseller") {
$item_rs = $item_rs->search({
'contact.reseller_id' => $c->user->reseller_id
},{
join => [@joins, { 'subscriber' => { 'voip_subscriber' => { 'contract' => 'contact' }}} ],
});
}
}
return $item_rs;
}
@ -57,7 +78,7 @@ sub hal_from_item {
),
Data::HAL::Link->new(relation => 'collection', href => sprintf("/api/%s/", $self->resource_name)),
Data::HAL::Link->new(relation => 'profile', href => 'http://purl.org/sipwise/ngcp-api/'),
Data::HAL::Link->new(relation => 'self', href => sprintf("%s%d", $self->dispatch_path, $item->id)),
Data::HAL::Link->new(relation => 'self', href => sprintf("%s%s", $self->dispatch_path, $item->id)),
Data::HAL::Link->new(relation => 'ngcp:subscribers', href => sprintf("/api/subscribers/%d", $resource->{subscriber_id})),
],
relation => 'ngcp:'.$self->resource_name,
@ -72,7 +93,7 @@ sub hal_from_item {
);
$resource->{user_agent} = $user_agent;
$resource->{id} = int($item->id);
$resource->{id} = ($item->id =~ /^\d+$/) ? int($item->id) : $item->id;
$hal->resource($resource);
return $hal;
@ -106,8 +127,9 @@ sub subscriber_from_item {
status => { '!=' => 'terminated' },
});
if($c->config->{features}->{multidomain}) {
my $domain = $c->config->{redis}->{usrloc} ? $item->domain : $item->domain->domain;
$sub_rs = $sub_rs->search({
'domain.domain' => $item->domain->domain,
'domain.domain' => $domain,
}, {
join => 'domain',
});
@ -199,7 +221,7 @@ sub fetch_item {
while ($flush_timeout) {
$item = $self->_item_by_aor($c, $sub, $form->values->{contact});
if ($item && (!$old_item || $item->id != $old_item->id)) {
if ($item && (!$old_item || $item->id ne $old_item->id)) {
last;
}
$item = undef;
@ -216,6 +238,14 @@ sub fetch_item {
return $item;
}
sub valid_id {
my ($self, $id) = @_;
if (defined $id && length $id > 0) {
return 1;
} else {
return;
}
}
1;
# vim: set tabstop=4 expandtab:

@ -0,0 +1,227 @@
package NGCP::Panel::Utils::RedisLocationResultSet;
use Moose;
use TryCatch;
use NGCP::Panel::Utils::RedisLocationResultSource;
use Data::Dumper;
has _c => (
is => 'ro',
isa => 'NGCP::Panel',
);
has _redis => (
is => 'ro',
isa => 'Redis',
);
has _rows => (
is => 'rw',
isa => 'ArrayRef',
default => sub {[]}
);
has _query_done => (
is => 'rw',
isa => 'Int',
default => 0,
);
has _domain_resellers => (
is => 'rw',
isa => 'HashRef',
lazy => 1,
default => sub {
my $self = shift;
my $h = {};
my $domres_rs = $self->_c->model('DB')->resultset('domain_resellers')->search(undef, {
join => 'domain'
});
while ((my $res = $domres_rs->next)) {
$h->{$res->domain->domain} = $res->reseller_id;
}
return $h;
},
);
sub count {
my ($self) = @_;
my $count = @{ $self->_rows };
return $count;
}
sub first {
my ($self) = @_;
return $self->_rows->[0];
}
sub all {
my ($self) = @_;
return @{ $self->_rows };
}
sub find {
my ($self, $filter) = @_;
my $id;
if (ref $filter eq "") {
$id = $filter;
$filter = { id => $id };
} elsif(ref $filter eq "HASH" && exists $filter->{id}) {
$id = $filter->{id};
} else {
$self->_c->log->error("id filter is mandatory for redis find()");
return;
}
my %entry = $self->_redis->hgetall("location:entry::$id");
$entry{id} = $entry{ruid};
if (exists $filter->{reseller_id} && $filter->{reseller_id} != $self->_domain_resellers->{$entry{domain}}) {
return;
}
return NGCP::Panel::Utils::RedisLocationResultSource->new(_data => \%entry);
}
sub search {
my ($self, $filter, $opt) = @_;
$filter //= {};
my $new_rs = $self->meta->clone_object($self);
unless ($new_rs->_query_done) {
if ($filter->{id}) {
push @{ $new_rs->_rows }, $new_rs->find($filter);
} elsif ($filter->{username} && $filter->{domain}) {
push @{ $new_rs->_rows },
@{ $new_rs->_rows_from_mapkey("location:usrdom::" .
$filter->{username} . ":" . $filter->{domain}, $filter) };
} else {
$new_rs->_scan($filter);
}
$new_rs->_query_done(1);
}
$new_rs->_filter($filter);
if ($opt->{order_by}->{'-desc'}) {
my $f = $opt->{order_by}->{'-desc'};
$f =~ s/^me\.//;
$new_rs->_rows([sort { $b->$f cmp $a->$f } @{ $new_rs->_rows }]);
} elsif ($opt->{order_by}->{'-asc'} || ref $opt->{order_by} eq "") {
my $f = $opt->{order_by}->{'-asc'} // $opt->{order_by};
$f =~ s/^me\.//;
$new_rs->_rows([sort { $a->$f cmp $b->$f } @{ $new_rs->_rows }]);
}
$opt->{rows} //= -1;
$opt->{offset} //= 0;
if (!defined $opt->{page} && $opt->{rows} > -1 || $opt->{offset} > 0) {
$new_rs->_rows([ splice @{ $new_rs->_rows }, $opt->{offset}, $opt->{rows} ]);
}
if (defined $opt->{page} && $opt->{rows} > 0) {
$new_rs->_rows([ splice(@{ $new_rs->_rows }, ($opt->{page} - 1 )*$opt->{rows}, $opt->{rows}) ]);
}
return $new_rs;
}
sub _rows_from_mapkey {
my ($self, $mapkey, $filter) = @_;
my @rows = ();
my $keys = $self->_redis->smembers($mapkey);
foreach my $key (@{ $keys }) {
my %entry = $self->_redis->hgetall($key);
$entry{id} = $entry{ruid};
if (exists $filter->{reseller_id} && $filter->{reseller_id} != $self->_domain_resellers->{$entry{domain}}) {
next;
}
my $res = NGCP::Panel::Utils::RedisLocationResultSource->new(_data => \%entry);
push @rows, $res;
}
return \@rows;
}
sub _filter {
my ($self, $filter) = @_;
my @newrows = ();
my $i = 0;
foreach my $row (@{ $self->_rows }) {
my $match = 0;
my $filter_applied = 0;
my %attr = map { $_->name => 1 } $row->meta->get_all_attributes;
foreach my $f (keys %{ $filter }) {
if ($f eq "-and" && ref $filter->{$f} eq "ARRAY") {
foreach my $col (@{ $filter->{$f} }) {
next unless (ref $col eq "ARRAY");
foreach my $innercol (@{ $col }) {
if (ref $innercol eq "HASH") {
foreach my $colname (keys %{ $innercol }) {
my $searchname = $colname;
$colname =~ s/^me\.//;
next if ($colname =~ /\./); # we don't support joined table columns
$filter_applied = 1;
if (ref $innercol->{$searchname} eq "") {
if (!exists $attr{$colname} || lc($row->$colname) ne lc($innercol->{$searchname})) {
} else {
$match = 1;
last;
}
} elsif (ref $innercol->{$searchname} eq "HASH" && exists $innercol->{$searchname}->{like}) {
my $fil = $innercol->{$searchname}->{like};
$fil =~ s/^\%//;
$fil =~ s/\%$//;
if (!exists $attr{$colname} || $row->$colname !~ /$fil/i) {
} else {
$match = 1;
last;
}
}
}
last if ($match);
}
}
}
last if ($match);
}
}
next if ($filter_applied && !$match);
push @newrows, $row;
}
$self->_rows(\@newrows);
}
sub _scan {
my ($self, $filter) = @_;
$filter //= {};
my $match = ($filter->{username} // "") . ":" . ($filter->{domain} // "");
if ($match eq ":") {
$match = "*";
} elsif ($match =~ /:$/) {
$match .= '*';
}
$self->_rows([]);
my $cursor = 0;
do {
my $res = $self->_redis->scan($cursor, MATCH => "location:usrdom::$match", COUNT => 1000);
$cursor = shift @{ $res };
my $mapkeys = shift @{ $res };
foreach my $mapkey (@{ $mapkeys }) {
push @{ $self->_rows }, @{ $self->_rows_from_mapkey($mapkey, $filter) };
}
} while ($cursor);
return 1;
}
sub result_class {
"dummy";
}
sub result_source {
NGCP::Panel::Utils::RedisLocationResultSource->new;
}
1;

@ -0,0 +1,30 @@
package NGCP::Panel::Utils::RedisLocationResultSource;
use Moose;
has _data => (
is => 'ro',
isa => 'HashRef',
default => sub {{}},
);
sub BUILD {
my ($self) = @_;
foreach my $k (keys %{ $self->_data }) {
$self->meta->add_attribute(
$k => (accessor => $k)
);
$self->$k($self->_data->{$k});
}
}
sub get_inflated_columns {
my ($self) = @_;
return %{ $self->_data };
}
sub columns {
}
1;

@ -14,10 +14,13 @@ use NGCP::Panel::Utils::Email;
use NGCP::Panel::Utils::Events;
use NGCP::Panel::Utils::DateTime qw();
use NGCP::Panel::Utils::License;
use NGCP::Panel::Utils::Generic;
use NGCP::Panel::Utils::RedisLocationResultSet;
use UUID qw/generate unparse/;
use JSON qw/decode_json encode_json/;
use IPC::System::Simple qw/capturex/;
use File::Slurp qw/read_file/;
use Redis;
my %LOCK = (
0, 'none',
@ -28,6 +31,40 @@ my %LOCK = (
5, 'ported',
);
sub get_subscriber_location_rs {
my ($c, $filter) = @_;
if ($c->config->{redis}->{usrloc}) {
my $redis;
try {
$redis = Redis->new(
server => $c->config->{redis}->{central_url},
reconnect => 10, every => 500000, # 500ms
cnx_timeout => 3,
);
unless ($redis) {
$c->log->error("Failed to connect to central redis url " . $c->config->{redis}->{central_url});
return;
}
$redis->select($c->config->{redis}->{usrloc_db});
my $rs = NGCP::Panel::Utils::RedisLocationResultSet->new(_redis => $redis, _c => $c);
$rs = $rs->search($filter);
return $rs;
} catch($e) {
$c->log->error("Failed to fetch location information from redis: $e");
return;
}
} else {
my $reg_rs = $c->model('DB')->resultset('location')->search({
username => $filter->{username},
});
if($c->config->{features}->{multidomain}) {
$reg_rs = $reg_rs->search({
domain => $filter->{domain},
});
}
return $reg_rs;
}
}
sub period_as_string {
my $set = shift;

@ -25,7 +25,7 @@ fi
if [ "${SELECT}" = "stable" ] ; then
echo "Test selection: ${SELECT}"
SELECT=$(echo ./t/api-rest/api-{all-links,balanceintervals,bannedips,bannedusers,billingfees,billingnetworks,billingprofiles,billingzones,calllists,calls,cert-auth,cfdestinationsets,contracts,customercontacts,customers,faxes,lnp,ncoslevels,pbxdevicemodels,pbxdevices,peeringgroups,peeringrules,peeringinboundrules,peeringservers,preferences,profilepackages,resellers,rewriterules,rewriterulesets,root,soundsets,subscribers,systemcontacts,threads,topuplogs,trustedsources,valid-patch,vouchers,method-override}.t)
SELECT=$(echo ./t/api-rest/api-{all-links,balanceintervals,bannedips,bannedusers,billingfees,billingnetworks,billingprofiles,billingzones,calllists,calls,cert-auth,cfdestinationsets,contracts,customercontacts,customers,faxes,lnp,ncoslevels,pbxdevicemodels,pbxdevices,peeringgroups,peeringrules,peeringinboundrules,peeringservers,preferences,profilepackages,resellers,rewriterules,rewriterulesets,root,soundsets,subscriberregistrations,subscribers,systemcontacts,threads,topuplogs,trustedsources,valid-patch,vouchers,method-override}.t)
elif [ "${SELECT}" = "fast" ] ; then
echo "Test selection: ${SELECT}"
SELECT=$(echo ./t/api-rest/api-{bannedips,bannedusers,billingnetworks,billingzones,calls,cert-auth,cfdestinationsets,ncoslevels,peeringgroups,peeringrules,peeringinboundrules,peeringservers,resellers,rewriterules,root,soundsets,systemcontacts,valid-patch,vouchers,method-override}.t)

Loading…
Cancel
Save