From d71c223eb2e2063a09c7971e2b91fd581a5935a6 Mon Sep 17 00:00:00 2001 From: Victor Seva Date: Tue, 23 Jun 2020 00:17:02 +0200 Subject: [PATCH] TT#68855 clean_registrations.pl tool that reads info directly from scenario.yml * check.sh: clean some shellcheck warnings * bench.sh: fix group option Change-Id: If736c42729e8fd32e964a5c9db7e47b289e916c8 --- bench.sh | 2 +- bin/check.sh | 90 ++++++------------- bin/clean_registrations.pl | 176 +++++++++++++++++++++++++++++++++++++ debian/control | 10 +++ 4 files changed, 213 insertions(+), 65 deletions(-) create mode 100755 bin/clean_registrations.pl diff --git a/bench.sh b/bench.sh index d05b423d..ad61e0a6 100755 --- a/bench.sh +++ b/bench.sh @@ -42,7 +42,7 @@ export BASE_DIR if ! "${SKIP_CONFIG}" ; then export PERL5LIB="${BASE_DIR}/lib" echo "add configuration for tests" - ./bin/config_debug.pl -g"${GROUP}" on + ./bin/config_debug.pl -g "${GROUP}" on ( cd /etc/ngcp-config || true ngcpcfg apply "k-c-t ${GROUP} on" diff --git a/bin/check.sh b/bin/check.sh index 18a7dbc8..9e78f29b 100755 --- a/bin/check.sh +++ b/bin/check.sh @@ -34,16 +34,6 @@ SKIP_MOVE_JSON_KAM=false CDR=false ERR_FLAG=0 - -# sipwise password for mysql connections -declare -r SIPWISE_EXTRA_CNF="/etc/mysql/sipwise_extra.cnf" - -if [ ! -f "${SIPWISE_EXTRA_CNF}" ]; then - echo "Error: missing DB credentials file '${SIPWISE_EXTRA_CNF}'." >&2 - exit 1 -fi - - # $1 kamailio msg parsed to yml # $2 destination png filename graph() { @@ -246,37 +236,6 @@ create_voip_prefs() { fi } -delete_locations() { - local f - local sub - - for f in ${SCEN_CHECK_DIR}/callee.csv ${SCEN_CHECK_DIR}/caller.csv; do - if ! [ -f "$f" ] ; then - continue - fi - for sub in $(uniq "${f}" | grep "${DOMAIN}" | cut -d\; -f1 | xargs); do - ngcp-kamctl proxy fifo ul.rm location "${sub}@${DOMAIN}" >/dev/null - # delete possible banned user - ngcp-kamcmd lb htable.delete auth "${sub}@${DOMAIN}::auth_count" - done - done - - # check what's in the DDBB - f=$(mysql --defaults-extra-file="${SIPWISE_EXTRA_CNF}" \ - kamailio -e 'select count(*) from location;' -s -r | head) - if [ "${f}" != "0" ]; then - echo "$(date) Cleaning location table" - sub=$(mysql --defaults-extra-file="${SIPWISE_EXTRA_CNF}" \ - -e 'select concat(username, "@", domain) as user from kamailio.location;' \ - -r -s | head| uniq|xargs) - for f in $sub; do - ngcp-kamctl proxy fifo ul.rm location "${f}" >/dev/null - done - mysql --defaults-extra-file="${SIPWISE_EXTRA_CNF}" \ - -e 'delete from kamailio.location;' || true - fi -} - # $1 domain delete_voip() { /usr/bin/ngcp-delete-domain "$1" >/dev/null 2>&1 @@ -332,7 +291,6 @@ delete_voip() { echo "$(date) - Deleting registrations" "${BIN_DIR}/create_registrations.pl" -delete "${SCEN_CHECK_DIR}/registration.yml" fi - delete_locations } release_appearance() { @@ -485,6 +443,8 @@ run_sipp() { echo "$(date) - create ${LOG_DIR}" mkdir -p "${LOG_DIR}" + echo "$(date) - Cleaning registrations" + "${BIN_DIR}/clean_registrations.pl" "${SCEN_CHECK_DIR}/scenario.yml" if [ "${PROFILE}" = "PRO" ] ; then release_appearance fi @@ -595,7 +555,8 @@ run_sipp() { status=1 fi - delete_locations + echo "$(date) - Cleaning registrations" + "${BIN_DIR}/clean_registrations.pl" "${SCEN_CHECK_DIR}/scenario.yml" if [ "${PROFILE}" = "PRO" ] ; then release_appearance fi @@ -681,24 +642,24 @@ cdr_check() { usage() { echo "Usage: check.sh [-hCDRTGgJKm] [-d DOMAIN ] [-p PROFILE ] -s [GROUP] check_name" echo "Options:" - echo -e "\t-C: skip creation of domain and subscribers" - echo -e "\t-R: skip run sipp" - echo -e "\t-D: skip deletion of domain and subscribers as final step" - echo -e "\t-T: skip checks" - echo -e "\t-P: skip parse" - echo -e "\t-G: creation of graphviz image" - echo -e "\t-g: creation of graphviz image only if test fails" - echo -e "\t-r: fix retransmission issues" - echo -e "\t-d: DOMAIN" - echo -e "\t-p: CE|PRO default is CE" - echo -e "\t-J: kamailio json output OFF" - echo -e "\t-M: skip move of kamailio json output to log folder" - echo -e "\t-K: enable tcpdump capture" - echo -e "\t-s: scenario group. Default: scenarios" - echo -e "\t-m: enable memdbg csv" - echo -e "\t-c: enable cdr validation" + echo -e "\\t-C: skip creation of domain and subscribers" + echo -e "\\t-R: skip run sipp" + echo -e "\\t-D: skip deletion of domain and subscribers as final step" + echo -e "\\t-T: skip checks" + echo -e "\\t-P: skip parse" + echo -e "\\t-G: creation of graphviz image" + echo -e "\\t-g: creation of graphviz image only if test fails" + echo -e "\\t-r: fix retransmission issues" + echo -e "\\t-d: DOMAIN" + echo -e "\\t-p: CE|PRO default is CE" + echo -e "\\t-J: kamailio json output OFF" + echo -e "\\t-M: skip move of kamailio json output to log folder" + echo -e "\\t-K: enable tcpdump capture" + echo -e "\\t-s: scenario group. Default: scenarios" + echo -e "\\t-m: enable memdbg csv" + echo -e "\\t-c: enable cdr validation" echo "Arguments:" - echo -e "\tcheck_name. Scenario name to check. This is the name of the directory on GROUP dir." + echo -e "\\tcheck_name. Scenario name to check. This is the name of the directory on GROUP dir." } while getopts 'hCd:p:Rs:DTPGgrcJKMm' opt; do @@ -720,6 +681,7 @@ while getopts 'hCd:p:Rs:DTPGgrcJKMm' opt; do M) SKIP_MOVE_JSON_KAM=true;; m) MEMDBG=true;; c) CDR=true;; + *) usage; exit 1;; esac done shift $((OPTIND - 1)) @@ -838,7 +800,7 @@ if ! "$SKIP_RUNSIPP" ; then if "${FIX_RETRANS}" ; then echo "$(date) - Checking retransmission issues" RETRANS_ISSUE=false - file_find=($(find "${LOG_DIR}" -maxdepth 1 -name '*.json' | sort)) + mapfile -t file_find < <(find "${LOG_DIR}" -maxdepth 1 -name '*.json' | sort) for json_file in "${file_find[@]}" ; do file_find=("${file_find[@]:1}") if ! [ -a "${json_file}" ] ; then @@ -858,12 +820,12 @@ if ! "$SKIP_RUNSIPP" ; then if "${RETRANS_ISSUE}" ; then echo "$(date) - Reordering kamailio json files" - file_find=($(find "${LOG_DIR}" -maxdepth 1 -name '*.json' | sort)) + mapfile -t file_find < <(find "${LOG_DIR}" -maxdepth 1 -name '*.json' | sort) a=1 for json_file in "${file_find[@]}" ; do new_name=$(printf "%04d.json" "${a}") mv -n "${json_file}" "${LOG_DIR}/${new_name}" &> /dev/null - let a=a+1 + ((a=a+1)) done fi @@ -906,7 +868,7 @@ if ! "${SKIP_TESTS}" ; then "${BIN_DIR}/generate_tests.sh" -d \ "${SCEN_CHECK_DIR}" "${LOG_DIR}/scenario_ids.yml" "${PROFILE}" - for t in ${SCEN_CHECK_DIR}/[0-9][0-9][0-9][0-9]_test*.yml; do + for t in "${SCEN_CHECK_DIR}"/[0-9][0-9][0-9][0-9]_test*.yml; do test_filepath "${t}" echo "$(date) - Check test ${t} on ${msg}" dest=${RESULT_DIR}/$(basename "${t}" .yml) diff --git a/bin/clean_registrations.pl b/bin/clean_registrations.pl new file mode 100755 index 00000000..80b09b46 --- /dev/null +++ b/bin/clean_registrations.pl @@ -0,0 +1,176 @@ +#!/usr/bin/perl +# +# Copyright: 2020 Sipwise Development Team +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This package is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# On Debian systems, the complete text of the GNU General +# Public License version 3 can be found in "/usr/share/common-licenses/GPL-3". +# +use strict; +use warnings; + +use English; +use Getopt::Long; +use Cwd 'abs_path'; +use Capture::Tiny qw(capture); +use YAML::XS; +use List::MoreUtils qw(uniq); +use DBI qw(:sql_types); + +my $CONSTANTS = YAML::XS::LoadFile('/etc/ngcp-config/constants.yml'); + +sub usage +{ + my $output = "usage: $PROGRAM_NAME [-h] scenario.yml\n"; + $output .= "Options:\n"; + $output .= "\t-h: this help\n"; + return $output +} + +my $help = 0; +my $del = 0; +GetOptions ("h|help" => \$help) + or die("Error in command line arguments\n".usage()); + +die(usage()) unless (!$help); +die("Wrong number of arguments\n".usage()) unless ($#ARGV == 0); + +my $filename = abs_path($ARGV[0]); +my $cf = YAML::XS::LoadFile($filename); +my $MYSQL_CREDENTIALS = "/etc/mysql/sipwise_extra.cnf"; + +if (! -e ${MYSQL_CREDENTIALS}) { + die("Error: missing DB credentials file '${MYSQL_CREDENTIALS}'.\n") +} + +sub get_mysql_credentials { + return $CONSTANTS->{credentials}->{mysql}->{system}->{u}; +} + +sub get_nodename { + my @lines = capturex( [ 0 ], "/usr/sbin/ngcp-nodename"); + my $l = shift @lines; + chomp $l; + return $l; +} + +sub connect_db { + my ($dbhost, $dbport, $mysql_user, $mysql_pass) = @_; + + my $dsn = "DBI:mysql:database=mysql;host=$dbhost;port=$dbport"; + if ( $mysql_user eq get_mysql_credentials() ) { + $dsn .= ";mysql_read_default_file=$MYSQL_CREDENTIALS"; + } + + my $dbh = DBI->connect($dsn, $mysql_user, $mysql_pass, + { PrintError => 1, mysql_auto_reconnect => 1 }) + or die "Can't connect to MySQL: ". $DBI::errstr; + return $dbh; +} + +sub run_cmd { + my $cmd = shift; + my $rc = 0; + my ($out, $err) = capture { + system($cmd); + $rc = $CHILD_ERROR >> 8; + }; + # error adjustments + if ($err =~ 'Warning: Skipping the data of table mysql\.event') { + $err = ''; + } + if ($err =~ 'Warning: Permanently added') { + $err = ''; + } + + return ($rc, $out, $err); +} + +sub kamailio_ul_lookup +{ + my $subs = shift; + my ($rc, $out, $err) = run_cmd("ngcp-kamcmd proxy ul.lookup location ${subs}"); + if ($err) { + $err = join(", ", split(/\r*\n/, $err)); + die "Error looking up ${subs} from kamailio: $err (errno: $rc)"; + } + print("location ${subs} from kamailio\n${out}\n"); +} + +sub clean_kamailio_ul +{ + my $subs = shift; + my ($rc, undef, $err) = run_cmd("ngcp-kamcmd proxy ul.rm location ${subs}"); + if ($err) { + $err = join(", ", split(/\r*\n/, $err)); + die "Error removing ${subs} from kamailio: $err (errno: $rc)"; + } + print("clean location ${subs} from kamailio\n"); +} + +sub clean_kamailio +{ + my $data = shift; + my @values = (); + foreach (@{$data->{scenarios}}) + { + push @values, $_->{username}."@".$_->{domain}; + foreach (@{$_->{responders}}) + { + $_->{active} = "yes" unless defined($_->{active}); + if($_->{register} eq "yes" && $_->{active} eq "yes") + { + push @values, $_->{username}."@".$_->{domain}; + } + } + } + foreach (uniq @values) { + clean_kamailio_ul($_); + } + return; +} + +sub clean_locations +{ + my $local_host = $CONSTANTS->{database}{local}{dbhost}; + my $local_port = $CONSTANTS->{database}{local}{dbport}; + my $local_user = get_mysql_credentials(); + my $local_pass = undef; + my $local_dbh = connect_db($local_host, $local_port, $local_user, $local_pass); + + my $sht = $local_dbh->prepare(<execute() or die "Cannot get locations: $DBI::errstr"; + my ($num) = $sht->fetchrow_array; + print("${num} in location\n"); + if($num gt 0) { + print("Cleaning location table\n"); + my $aors = $local_dbh->selectall_arrayref(<do(< Build-Depends: debhelper-compat (= 12), devscripts, + libcapture-tiny-perl, libconfig-tiny-perl, + libdbd-mysql-perl, + libdbi-perl, + libfile-slurp-perl, libhash-merge-perl, + libio-compress-perl, libjson-perl, + liblist-moreutils-perl, libtemplate-perl, libtext-csv-perl, libtext-diff-perl, @@ -22,11 +28,15 @@ Package: kamailio-config-tests Architecture: all Depends: curl, + libcapture-tiny-perl, libconfig-tiny-perl, + libdbd-mysql-perl, + libdbi-perl, libfile-slurp-perl, libhash-merge-perl, libio-compress-perl, libjson-perl, + liblist-moreutils-perl, libtemplate-perl, libtext-csv-perl, libtext-diff-perl,