diff --git a/bench.sh b/bench.sh index e16ebaea..8fc753b0 100755 --- a/bench.sh +++ b/bench.sh @@ -5,9 +5,10 @@ GROUP="${GROUP:-scenarios}" CAPTURE=false SINGLE_CAPTURE=false PROV_TYPE=step +CHECK_TYPE=all usage() { - echo "Usage: bench.sh [-p PROFILE] [-C] [num_runs]" + echo "Usage: bench.sh [-hCkK] [-p PROFILE] [-x GROUP] [-P ] [-S ] num_runs" echo "Options:" echo -e "\\t-p CE|PRO default is autodetect" echo -e "\\t-C skips configuration of the environment" @@ -15,11 +16,12 @@ usage() { echo -e "\\t-K capture messages with tcpdump. One big file for all scenarios" echo -e "\\t-x set GROUP scenario. Default: scenarios" echo -e "\\t-P provisioning, default:step" + echo -e "\\t-S check type. Default: all (cfgt, sipp)" echo -e "\\t-h this help" echo -e "num_runs default is 20" } -while getopts 'hCkKP:p:x:' opt; do +while getopts 'hCkKP:p:S:x:' opt; do case $opt in h) usage; exit 0;; C) SKIP_CONFIG=true;; @@ -27,6 +29,7 @@ while getopts 'hCkKP:p:x:' opt; do K) CAPTURE=true;; p) PROFILE=${OPTARG};; P) PROV_TYPE=${OPTARG};; + S) CHECK_TYPE=${OPTARG};; x) GROUP=${OPTARG};; *) echo "Unknown option $opt"; usage; exit 1;; esac @@ -83,7 +86,7 @@ for i in $(seq "$NUM"); do echo "$(date) - ERROR[$status] run_tests $i" break fi - ./get_results.sh -r -x"${GROUP}" | tee /tmp/get_results.log + ./get_results.sh -S"${CHECK_TYPE}" -r -c -x"${GROUP}" | tee /tmp/get_results.log status=$? if [[ $status -ne 0 ]]; then echo "$(date) - ERROR[$status] get_results $i" diff --git a/bin/check.sh b/bin/check.sh index 0a458446..413b24a2 100755 --- a/bin/check.sh +++ b/bin/check.sh @@ -23,7 +23,7 @@ CAPTURE=false SKIP=false MEMDBG=false SKIP_DELDOMAIN=false -SKIP_TESTS=false +CHECK_TYPE=all SKIP_PARSE=false SKIP_RUNSIPP=false FIX_RETRANS=false @@ -103,6 +103,7 @@ function str_check_error() { [[ $(($1 & 4)) -eq 4 ]] && err_type+=("FLOW_VARS") [[ $(($1 & 8)) -eq 8 ]] && err_type+=("SIP_IN") [[ $(($1 & 16)) -eq 16 ]] && err_type+=("SIP_OUT") + [[ $(($1 & 32)) -eq 32 ]] && err_type+=("SIP") echo "${err_type[*]}" } @@ -215,6 +216,39 @@ check_retrans_block() { return 1 } +# $1 sip message test yml +# $2 sipp msg parsed to .msg log file +# $3 destination tap filename +check_sip_test() { + local test_file="${1}" + local msg_file="${2}" + local dest_file="${3}" + + if ! [ -f "$1" ]; then + generate_error_tap "$3" "$1" + ERR_FLAG=1 + return 1 + fi + if ! [ -f "$2" ]; then + generate_error_tap "$3" "$2" + ERR_FLAG=1 + return 1 + fi + + sipp_msg=$(basename "${msg_file}") + if "${BIN_DIR}/check_sip.py" "${test_file}" "${msg_file}" > "${dest_file}" ; then + echo " ok" + test_ok+=("${sipp_msg}") + return 0 + else + err_type=$(str_check_error $?) + echo " NOT ok[${err_type}]" + ERR_FLAG=1 + # we have to add here show_sip.pl funcitoning to produce differences in results + return 1 + fi +} + # $1 unit test yml # $2 kamailio msg parsed to yml # $3 destination tap filename @@ -243,7 +277,7 @@ check_test() { kam_msg=$(basename "$2") echo -n "$(date) - Testing $(basename "$1") against ${kam_msg} -> $(basename "$3")" - if "${BIN_DIR}/check.py" ${kam_type} "$1" "$2" > "$3" ; then + if "${BIN_DIR}/check.py" "${kam_type}" "$1" "$2" > "$3" ; then echo " ok" test_ok+=("${kam_msg}") return 0 @@ -508,6 +542,18 @@ run_sipp() { fi } +test_sip_filepath() { + local msg_name + msg_name=${1/_test.yml/.msg} + msg_name=${msg_name/sip_messages/sipp_scenario} + sip_msg="${LOG_DIR}/$(basename "${msg_name}")" + if ! [ -f "${sip_msg}" ]; then + echo "$(date) - WARNING: no sipp log file found! filename=<${sip_msg}>. Skipping." + return 1 + fi + return 0 +} + test_filepath() { local msg_name @@ -585,13 +631,13 @@ cdr_check() { } usage() { - echo "Usage: check.sh [-hCDRTGgJKm] [-p PROFILE ] -s [GROUP] check_name" + echo "Usage: check.sh [-hCDRGgJKm] [-T ] [-p PROFILE ] [-s GROUP] check_name" echo "Options:" echo -e "\\t-I: SIP_SERVER IP, default:127.0.0.1" echo -e "\\t-C: skip creation of domain and subscribers" echo -e "\\t-R: skip run sipp" echo -e "\\t-D: skip deletion of domain and subscribers as final step" - echo -e "\\t-T: skip checks" + echo -e "\\t-T check type. Default: all" echo -e "\\t-P: skip parse" echo -e "\\t-G: creation of graphviz image" echo -e "\\t-g: creation of graphviz image only if test fails" @@ -607,7 +653,7 @@ usage() { echo -e "\\tcheck_name. Scenario name to check. This is the name of the directory on GROUP dir." } -while getopts 'hI:Cp:Rs:DTPGgrcJKMmw:' opt; do +while getopts 'hI:Cp:Rs:DT:PGgrcJKMmw:' opt; do case $opt in h) usage; exit 0;; I) SIP_SERVER=${OPTARG};; @@ -616,7 +662,7 @@ while getopts 'hI:Cp:Rs:DTPGgrcJKMmw:' opt; do R) SKIP_RUNSIPP=true; SKIP_DELDOMAIN=true;; s) GROUP=${OPTARG};; D) SKIP_DELDOMAIN=true;; - T) SKIP_TESTS=true;; + T) CHECK_TYPE=${OPTARG};; P) SKIP_PARSE=true;; K) CAPTURE=true;; G) GRAPH=true;; @@ -681,6 +727,11 @@ if [ -f "${SCEN_CHECK_DIR}/pro.yml" ] && [ "${PROFILE}" == "CE" ]; then exit 0 fi +case "${CHECK_TYPE}" in + all|sipp|cfgt|none) ;; + *) echo "unknown check type"; exit 1;; +esac + if ! "$SKIP" ; then "${BIN_DIR}/provide_scenario.sh" "${SCEN_CHECK_DIR}" create fi @@ -780,9 +831,8 @@ if ! "${SKIP_PARSE}" ; then fi fi - # let's check the results -if ! "${SKIP_TESTS}" ; then +if [[ ${CHECK_TYPE} != none ]] ; then echo "$(date) - =================================================================================" echo "$(date) - Check [${GROUP}/${PROFILE}]: ${NAME_CHECK}" @@ -800,18 +850,33 @@ if ! "${SKIP_TESTS}" ; then test_ok=() - for t in "${SCEN_CHECK_DIR}"/[0-9][0-9][0-9][0-9]_test*.yml; do - test_filepath "${t}" - echo "$(date) - Check test ${t} on ${msg}" - dest=${RESULT_DIR}/$(basename "${t}" .yml) - check_test "${t}" "${msg}" "${dest}.tap" && result=OK || result=KO - echo "$(date) - $(basename "${t}" .yml) - Done[${result}]" - if "${GRAPH}" ; then - echo "$(date) - Generating flow image: ${dest}.png" - graph "${msg}" "${dest}.png" - echo "$(date) - Done" - fi - done + if [[ ${CHECK_TYPE} != sipp ]] ; then + for t in "${SCEN_CHECK_DIR}"/[0-9][0-9][0-9][0-9]_test*.yml; do + test_filepath "${t}" + echo "$(date) - Check test ${t} on ${msg}" + dest=${RESULT_DIR}/$(basename "${t}" .yml) + check_test "${t}" "${msg}" "${dest}.tap" && result=OK || result=KO + echo "$(date) - $(basename "${t}" .yml) - Done[${result}]" + if "${GRAPH}" ; then + echo "$(date) - Generating flow image: ${dest}.png" + graph "${msg}" "${dest}.png" + echo "$(date) - Done" + fi + done + fi + + if [[ ${CHECK_TYPE} != cfgt ]] ; then + echo "$(date) - ~~~~~" + for t in "${SCEN_CHECK_DIR}"/sip_messages*[0-9][0-9]_test.yml; do + if test_sip_filepath "${t}"; then + echo -n "$(date) - SIP: Check test $(basename "${t}") on ${sip_msg}" + dest=$(basename "${sip_msg}") + dest=${RESULT_DIR}/${dest/.msg/.tap} + check_sip_test "${t}" "$sip_msg" "${dest}" + fi + done + echo "$(date) - ~~~~~" + fi if "${CDR}" ; then echo "$(date) - Validating CDRs" diff --git a/bin/check_sip.py b/bin/check_sip.py new file mode 100755 index 00000000..2a6b4d59 --- /dev/null +++ b/bin/check_sip.py @@ -0,0 +1,260 @@ +#!/usr/bin/python3 +# +# Copyright: 2013-2021 Sipwise Development Team +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This package is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# On Debian systems, the complete text of the GNU General +# Public License version 3 can be found in "/usr/share/common-licenses/GPL-3". +# +import io +import sys +import re +import argparse +import json +import logging +from enum import Flag +from pathlib import Path + +from yaml import load + +try: + from yaml import CLoader as Loader +except ImportError: + from yaml import Loader + + +class Section(Flag): + SIP = 32 + + +class Test: + + """ Class to create TAP output """ + + def __init__(self): + self._step = [] + self._errflag = Section(0) + + def comment(self, msg): + """ Add a comment """ + self._step.append({"result": None, "msg_ok": msg}) + + def ok(self, msg=None): + """ Add a ok result """ + self._step.append({"result": True, "msg_ok": msg}) + + def error(self, section, msg_err): + """ Add an error result""" + self._step.append({"result": False, "msg_err": msg_err}) + self._errflag |= section + + @classmethod + def compare(cls, val0, val1): + logging.debug( + "val0:[%s]:'%s' val1:[%s]:'%s'" + % (type(val0), str(val0), type(val1), str(val1)) + ) + if isinstance(val0, str): + if re.search(val0, str(val1)) is not None: + return True + else: + return False + elif isinstance(val0, int): + try: + result = val0 == int(val1) + except Exception: + result = False + elif isinstance(val0, list) and isinstance(val1, list): + size = len(val0) + if size != len(val1): + return False + result = True + for k in range(size): + try: + result = result and cls.compare(val0[k], val1[k]) + except Exception as e: + logging.debug(e) + return False + else: + result = val0 == val1 + return result + + def test(self, section, value_expected, value, msg_err, msg_ok=None): + """ Test two values and add the result""" + result = Test.compare(value_expected, value) + val = {"result": result, "msg_err": msg_err, "msg_ok": msg_ok} + self._step.append(val) + if not result: + self._errflag |= section + + def isError(self): + return self._errflag.value != 0 + + def _num_tests(self): + """get the num of tests""" + test = 0 + for s in self._step: + if s["result"] is not None: + test = test + 1 + return test + + def __str__(self): + """get the TAP output""" + output = "1..%s\n" % self._num_tests() + test = 1 + for s in self._step: + if s["result"] is None: + output += "# %s\n" % s["msg_ok"] + continue + elif s["result"]: + if s["msg_ok"] is not None: + output += "ok %d - %s\n" % (test, s["msg_ok"]) + else: + output += "ok %d\n" % test + else: + output += "not ok %d - ERROR: %s\n" % (test, s["msg_err"]) + test = test + 1 + return output + + +def check_sip(sec, scen, msg, test): + if isinstance(msg, list): + if len(msg) != 1: + test.error(sec, "sip_in len != 1") + return + else: + msg = msg[0] + for rule in scen: + if rule.startswith("_:NOT:_"): + flag = False + rule = rule[7:] + msg_ok = "%s not match" + msg_ko = "%s match" + else: + flag = True + msg_ok = "%s match" + msg_ko = "%s not match" + result = re.search(rule, msg) + if (result is not None) == flag: + test.ok(msg_ok % rule) + continue + test.comment("result:%s" % result) + test.error(sec, msg_ko % rule) + + +def check_sip_received(sec, scen, msgs, test): + num_msgs = len(msgs) + num_scen = len(scen) + for i in range(num_scen): + test.comment("messages %d" % i) + if i < num_msgs: + check_sip(sec, scen[i], msgs[i], test) + else: + test.error(sec, "messages[%d] does not exist" % i) + if num_scen != num_msgs: + msg = "we expected {} out messages but we have {}" + test.error(sec, msg.format(num_scen, num_msgs)) + + +def load_yaml(filepath): + output = None + with io.open(filepath, "r") as file: + output = load(file, Loader=Loader) + file.close() + return output + + +def load_json(filepath): + output = None + with io.open(filepath, "r") as file: + output = json.load(file) + file.close() + return output + + +def convert_sippmsg_to_json(filepath): + file_content = open(filepath, "r") + c = file_content.read() + + # prepare a list of sipp generated messages (raw text now) + content_list = c.split("-----------------------------------------------") + content_list_received = [] + file_content.close() + + # now pick out only received sipp messages + for each in content_list: + if ("UDP message received" in each) or ( + "TCP message received" in each + ): + # delete all undesired lines from each memeber's raw text + each = each.replace("\n\n\n", "\n").replace("\n\n", "\n") + each = each.replace('"', '\\"') + each = each.split("\n")[2:] + + i, arrLen = 0, len(each) + while i < arrLen: + each[i] = each[i] + "\\r\\n" + i += 1 + + # make a raw text now from list + each = "".join(each) + content_list_received.append(each) + + # add some eventual formatting to implement a JSON syntax + i, arrLen = 0, len(content_list_received) + while i < arrLen: + content_list_received[i] = '"' + content_list_received[i] + '",' + i += 1 + output = "".join(content_list_received)[:-1] + return '{\n "messages": [' + output + "]\n}" + + +def main(args): + + # convert sipp msg into json format, returns only received messages + output = convert_sippmsg_to_json(args.msg_file) + + if args.debug: + logging.basicConfig(level=logging.DEBUG) + + test = Test() + try: + scen = load_yaml(args.test_file) + except Exception: + scen = "message:\n - ['']" + test.error("Error loading yaml file:%s" % args[1]) + + try: + check = json.loads(output) + except Exception: + check = {"messages": ""} + test.error("Error loading json file:%s" % args[1]) + + test.comment("check sip messages") + check_sip_received(Section.SIP, scen["messages"], check["messages"], test) + + print(test) + if test.isError(): + sys.exit(test._errflag.value) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="generate TAP result") + grp = parser.add_mutually_exclusive_group() + parser.add_argument("test_file", help="YAML file with checks") + parser.add_argument("msg_file", help="sipp file") + parser.add_argument("-d", "--debug", action="store_true") + args = parser.parse_args() + main(args) diff --git a/bin/check_sipp.sh b/bin/check_sipp.sh new file mode 100755 index 00000000..1c007171 --- /dev/null +++ b/bin/check_sipp.sh @@ -0,0 +1,165 @@ +#!/bin/bash +# +# Copyright: 2013-2021 Sipwise Development Team +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This package is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# On Debian systems, the complete text of the GNU General +# Public License version 3 can be found in "/usr/share/common-licenses/GPL-3". +# +ERR_FLAG=0 + +# $1 destination tap file +# $2 file path +generate_error_tap() { + local tap_file="$1" + cat < "${tap_file}" +1..1 +not ok 1 - ERROR: File $2 does not exists +EOF +echo "$(date) - $(basename "$2") NOT ok" +} + +test_sip_filepath() { + local msg_name + msg_name=${1/_test.yml/.msg} + msg_name=${msg_name/sip_messages/sipp_scenario} + + msg="${LOG_DIR}/$(basename "${msg_name}")" + if [ -f "${msg}" ]; then + return 0 + fi + echo "$(date) - WARNING: no sipp log file found! filename=<${msg}>. Skipping." + return 1 +} + +# $1 sip message test yml +# $2 sipp msg parsed to .msg log file +# $3 destination tap filename +check_sip_test() { + local test_file="${1}" + local msg_file="${2}" + local dest_file="${3}" + local sipp_msg + + if ! [ -f "$1" ]; then + generate_error_tap "$3" "$1" + ERR_FLAG=1 + return 1 + fi + if ! [ -f "$2" ]; then + generate_error_tap "$3" "$2" + ERR_FLAG=1 + return 1 + fi + + sipp_msg=$(basename "${msg_file}") + if "${BIN_DIR}/check_sip.py" "${test_file}" "${msg_file}" > "${dest_file}" ; then + echo " ok" + test_ok+=("${sipp_msg}") + return 0 + else + echo " NOT ok[SIP]" + ERR_FLAG=1 + # we have to add here show_sip.pl funcitoning to produce differences in results + return 1 + fi +} + +usage() { + echo "Usage: check_sipp.sh [-h] [-p PROFILE ] -s [GROUP] check_name" + echo "Options:" + echo -e "\\t-p: CE|PRO default is CE" + echo -e "\\t-s: scenario group. Default: scenarios" + echo "Arguments:" + echo -e "\\tcheck_name. Scenario name to check. This is the name of the directory on GROUP dir." +} + +while getopts 'hp:s:' opt; do + case $opt in + h) usage; exit 0;; + p) PROFILE=${OPTARG};; + s) GROUP=${OPTARG};; + *) usage; exit 1;; + esac +done +shift $((OPTIND - 1)) + +if [[ $# != 1 ]]; then + echo "Wrong number of arguments" + usage + exit 1 +fi + +GROUP="${GROUP:-scenarios}" +NAME_CHECK="$1" +BASE_DIR="${BASE_DIR:-/usr/share/kamailio-config-tests}" +BIN_DIR="${BASE_DIR}/bin" +LOG_DIR="${BASE_DIR}/log/${GROUP}/${NAME_CHECK}" +RESULT_DIR="${BASE_DIR}/result/${GROUP}/${NAME_CHECK}" +SCEN_DIR="${BASE_DIR}/${GROUP}" +SCEN_CHECK_DIR="${SCEN_DIR}/${NAME_CHECK}" +PROFILE="${PROFILE:-CE}" + +if [ "${PROFILE}" != "CE" ] && [ "${PROFILE}" != "PRO" ]; then + echo "PROFILE ${PROFILE} unknown" + usage + exit 2 +fi + +if ! [ -d "${SCEN_CHECK_DIR}" ]; then + echo "no ${SCEN_CHECK_DIR} found" + usage + exit 3 +fi + +if ! [ -f "${SCEN_CHECK_DIR}/scenario.yml" ]; then + echo "${SCEN_CHECK_DIR}/scenario.yml not found" + exit 14 +fi + +if [ -f "${SCEN_CHECK_DIR}/pro.yml" ] && [ "${PROFILE}" == "CE" ]; then + echo "${SCEN_CHECK_DIR}/pro.yml found but PROFILE ${PROFILE}" + echo "Skipping the tests because not in scope" + exit 0 +fi + +# let's check the results +echo "$(date) - =================================================================================" +echo "$(date) - Check [${GROUP}/${PROFILE}]: ${NAME_CHECK}" + +mkdir -p "${RESULT_DIR}" + +echo "$(date) - Cleaning tests files" +find "${SCEN_CHECK_DIR}" -type f -name '*test.yml' -exec rm {} \; +echo "$(date) - Generating tests files" +"${BIN_DIR}/generate_tests.sh" \ + -f 'sip_messages*test.yml.tt2' \ + -d "${SCEN_CHECK_DIR}" "${LOG_DIR}/scenario_ids.yml" "${PROFILE}" + +test_ok=() + +for t in "${SCEN_CHECK_DIR}"/sip_messages*[0-9][0-9]_test.yml; do + if test_sip_filepath "${t}"; then + echo -n "$(date) - SIP: Check test $(basename "${t}") on ${msg}" + dest=$(basename "${msg}") + dest=${RESULT_DIR}/${dest/.msg/.tap} + check_sip_test "${t}" "$msg" "${dest}" + fi +done + +echo "$(date) - =================================================================================" + +exit ${ERR_FLAG} +#EOF diff --git a/bin/generate_tests.sh b/bin/generate_tests.sh index e2795236..e0d4ac99 100755 --- a/bin/generate_tests.sh +++ b/bin/generate_tests.sh @@ -21,8 +21,8 @@ BASE_DIR="${BASE_DIR:-/usr/share/kamailio-config-tests}" DIR="${BASE_DIR}/scenarios" BIN_DIR="${BASE_DIR}/bin" -DEST_DIR="${DEST_DIR}" - +DEST_DIR="${DEST_DIR:-}" +FILTER="*_test.yml.tt2" error_flag=0 function clean @@ -32,20 +32,23 @@ function clean function usage { - echo "Usage: generate_tests.sh [-h] [-c] [-d directory] scenario_ids.yml profile" + echo "Usage: generate_tests.sh [-h] [-c] [-d directory] [-f filter] scenario_ids.yml profile" echo "Options:" echo -e "\tc: clean. Removes all generated test files" echo -e "\td: directory" + echo -e "\tf: filter. Default: '*_test.yml.tt2'" echo -e "\th: this help" echo "Args:" echo -e "\tprofile: CE|PRO" } -while getopts 'hcd:' opt; do +while getopts 'hcd:f:' opt; do case $opt in h) usage; exit 0;; c) clean; exit 0;; d) DIR=${OPTARG};; + f) FILTER=${OPTARG};; + *) usage; exit 1;; esac done shift $((OPTIND - 1)) @@ -73,7 +76,7 @@ if [ ! -x "${BIN_DIR}/generate_test.pl" ]; then exit 3 fi -for t in $(find "${DIR}" -not -regex '.+customtt.tt2' -type f -name '*_test.yml.tt2' | sort); do +for t in $( find "${DIR}" -not -regex '.+customtt.tt2' -type f -name "${FILTER}" | sort); do origdir="$(dirname "${t}")" template="$(basename "${t}")" if [ -n "${DEST_DIR}" ] ; then @@ -97,4 +100,3 @@ for t in $(find "${DIR}" -not -regex '.+customtt.tt2' -type f -name '*_test.yml. done exit ${error_flag} -#EOF diff --git a/get_results.sh b/get_results.sh index 8d87e372..36f2530e 100755 --- a/get_results.sh +++ b/get_results.sh @@ -6,20 +6,20 @@ PROFILE="${PROFILE:-}" GROUP="${GROUP:-scenarios}" RETRANS="" CDR="" +CHECK_TYPE=all usage() { - echo "Usage: get_results.sh [-p PROFILE] [-h] [-g]" + echo "Usage: get_results.sh [-hcgGr] [-f FILE] [-p PROFILE] [-x GROUP] [-S ] [-R number]" echo -e "\\t-p CE|PRO default is autodetect" echo -e "\\t-g generate png flow graphs if test fails" echo -e "\\t-G generate png all flow graphs" echo -e "\\t-h this help" - echo -e "\\t-P parse only will disable test" - echo -e "\\t-T test only will disable parse" echo -e "\\t-r fix retransmission issues" echo -e "\\t-R choose how many messages before and after to check" echo -e "\\t-c enable cdr validation" echo -e "\\t-x set GROUP scenario. Default: scenarios" echo -e "\\t-f scenarios file" + echo -e "\\t-S check type. Default: all (cfgt, sipp)" echo "BASE_DIR:${BASE_DIR}" echo "BIN_DIR:${BIN_DIR}" } @@ -42,19 +42,18 @@ get_scenarios() { fi } -while getopts 'hf:gGp:TPrR:cx:' opt; do +while getopts 'hf:gGp:rR:cx:S:' opt; do case $opt in h) usage; exit 0;; G) GRAPH="-G";; g) GRAPH="-g";; - P) OPTS="-T";; - T) OPTS="-P";; r) RETRANS="-r";; R) RETRANS="-r -w ${OPTARG}";; c) CDR="-c";; p) PROFILE=${OPTARG};; x) GROUP=${OPTARG};; f) SCEN_FILE=${OPTARG};; + S) CHECK_TYPE=${OPTARG};; *) usage; exit 1;; esac done @@ -85,13 +84,36 @@ fi get_scenarios -echo "${SCEN[@]}" | tr ' ' '\n' \ - | parallel "${BIN_DIR}/check.sh ${GRAPH} -C -R ${OPTS} ${RETRANS} ${CDR} -p ${PROFILE} -s ${GROUP}" -status=$? -echo "$(date) - All done[${status}]" +check_old() { + echo "$(date) - =================================================================================" + echo "$(date) - ----- ${CHECK_TYPE} checks ----- " + echo "${SCEN[@]}" | tr ' ' '\n' \ + | parallel "${BIN_DIR}/check.sh ${GRAPH} -T${CHECK_TYPE} -C -R ${OPTS} ${RETRANS} ${CDR} -p ${PROFILE} -s ${GROUP}" + status=$? + echo "$(date) - All done[${status}]" +} + +check_sipp() { + echo "$(date) - =================================================================================" + echo "$(date) - ---- sipp checks ----- " + echo "${SCEN[@]}" | tr ' ' '\n' \ + | parallel "${BIN_DIR}/check_sipp.sh -p ${PROFILE} -s ${GROUP}" + status=$? + echo "$(date) - All done[${status}]" +} + +case "${CHECK_TYPE}" in + all|cfgt) check_old;; + sipp) check_sipp;; + *) echo "unknown check type"; exit 1;; +esac + tap_cmd=() for t in "${SCEN[@]}" ; do - tap_cmd+=( "result/${GROUP}/${t}/"*tap ) + tap_check=$(find "result/${GROUP}/${t}" -name '*.tap' 2>/dev/null) + if [ -n "${tap_check}" ]; then + tap_cmd+=( "result/${GROUP}/${t}/"*tap ) + fi done prove -f -Q "${tap_cmd[@]}"|tee "result/${GROUP}/prove.log" grep 'Failed:' "result/${GROUP}/prove.log"|awk -F/ '{print $3}'|uniq > "result/${GROUP}/result_failed.txt" diff --git a/run_tests.sh b/run_tests.sh index d70278c1..17679733 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -16,7 +16,7 @@ TMP_LOG_DIR="/tmp" KAM_DIR="/tmp/cfgtest" COREDUMP_DIR="/ngcp-data/coredumps" PROFILE="${PROFILE:-}" -OPTS=(-P -T -M -C) #SKIP_PARSE=true, SKIP_TESTS=true, SKIP_MOVE_JSON_KAM=true, SKIP=true +OPTS=(-P -Tnone -M -C) #SKIP_PARSE=true, SKIP_TESTS=true, SKIP_MOVE_JSON_KAM=true, SKIP=true SHOW_SCENARIOS=false SKIP_CONFIG=false diff --git a/scenarios/invite/sip_messages00_test.yml.tt2 b/scenarios/invite/sip_messages00_test.yml.tt2 new file mode 100644 index 00000000..0ccdaa07 --- /dev/null +++ b/scenarios/invite/sip_messages00_test.yml.tt2 @@ -0,0 +1,39 @@ +messages: + - [ + '^SIP/2.0 100 Trying', + 'CSeq: \d+ INVITE', + 'From: