From 74c16ee5310c768442998c796dc296a2188608db Mon Sep 17 00:00:00 2001 From: Marco Capetta Date: Fri, 12 Jan 2018 15:41:43 +0100 Subject: [PATCH] TT#26452 Improve kamailio-config-test to be re-transmission safe Added option "-r" to run_tests and get_results script to activate retransmission checks. If option is activated: * at the end of the each test, json files will be checked and, if a retransmission is found, the files will be renamed and renumbered. This will help to detect most of the retransmission cases. * In case a retransmission is not detected in the previous step, if a scenario fails, it will be compared with the next json file. This is an additional help to detect retransmissions. * In case the transmission is detected and properly healed, no error code is returned. Otherwise the script will exit 1 as before. Change-Id: I5eaa7bdb920be869099440c295e0bfd75c2bf107 --- bin/check.sh | 112 ++++++++++++++++++++++++++++++++++++++++++++----- get_results.sh | 7 +++- run_tests.sh | 10 ++++- 3 files changed, 115 insertions(+), 14 deletions(-) diff --git a/bin/check.sh b/bin/check.sh index e9cedb90..e604f7a0 100755 --- a/bin/check.sh +++ b/bin/check.sh @@ -26,6 +26,7 @@ SKIP_DELDOMAIN=false SKIP_TESTS=false SKIP_PARSE=false SKIP_RUNSIPP=false +FIX_RETRANS=false GRAPH=false GRAPH_FAIL=false JSON_KAM=false @@ -85,17 +86,48 @@ check_test() { kam_type="--json" fi - echo -n "$(date) - Testing $(basename "$1") againts $(basename "$2") -> $(basename "$3")" - if ! "${BIN_DIR}/check.py" ${kam_type} "$1" "$2" > "$3" ; then - echo " NOT ok" - ERR_FLAG=1 - if ( ! "${GRAPH}" ) && "${GRAPH_FAIL}" ; then - echo "$(date) - Generating flow image: ${dest}.png" - graph "$msg" "${dest}.png" - echo "$(date) - Done" - fi - else + echo -n "$(date) - Testing $(basename "$1") against $(basename "$2") -> $(basename "$3")" + if "${BIN_DIR}/check.py" ${kam_type} "$1" "$2" > "$3" ; then echo " ok" + return + fi + + echo " NOT ok" + + if "${FIX_RETRANS}" ; then + echo "$(date) - Fix retransmissions enabled: try to test the next json file" + local next_msg + local next_tap + next_test_filepath "$1" + next_tap=${3/_test.tap/_test_retry.tap} + + if [ -a "$next_msg" ] ; then + echo -n "$(date) - Testing $(basename "$1") against $(basename "$next_msg") -> $(basename "$next_tap")" + if "${BIN_DIR}/check.py" ${kam_type} "$1" "$next_msg" > "$next_tap" ; then + # Test using the next json file was fine. That means that, with high probability, there was a retransmission. + # Next step is to backup the failed tap test and overwrite it with the working one + mv "${3}" "${3}_retrans" + mv "${next_tap}" "${3}" + echo " ok" + return + fi + + if [ -a "$next_tap" ] ; then + # Test using the next json file was a failure. + # Next step is remove $next_tap file to don't create confusion during the additional checks + rm "${next_tap}" + fi + fi + + echo " NOT ok" + fi + + ERR_FLAG=1 + if ( ! "${GRAPH}" ) && "${GRAPH_FAIL}" ; then + echo "$(date) - Generating flow image: ${dest}.png" + # In any failure case only the graph related to the original json file will be created + graph "$msg" "${dest}.png" + echo "$(date) - Done" fi } @@ -495,6 +527,26 @@ test_filepath() { msg="${LOG_DIR}/$(basename "$msg_name")" } +next_test_filepath() { + local msg_name + local old_json + local new_json + + if ! "${JSON_KAM}" ; then + msg_name=${1/_test.yml/.yml} + else + msg_name=${1/_test.yml/.json} + fi + + msg_name=$(basename "${msg_name}") + old_json="${msg_name:0:4}" + new_json=$(((10#$old_json)+1)) + new_json=$(printf %04d ${new_json}) + msg_name="${new_json}${msg_name:4}" + + next_msg="${LOG_DIR}/${msg_name}" +} + usage() { echo "Usage: check.sh [-hCDRTGgJKm] [-d DOMAIN ] [-p PROFILE ] -s [GROUP] check_name" echo "Options:" @@ -505,6 +557,7 @@ usage() { echo -e "\t-P: skip parse" echo -e "\t-G: creation of graphviz image" echo -e "\t-g: creation of graphviz image only if test fails" + echo -e "\t-r: fix retransmission issues" echo -e "\t-d: DOMAIN" echo -e "\t-p CE|PRO default is CE" echo -e "\t-J kamailio json output ON. PARSE skipped" @@ -515,7 +568,7 @@ usage() { echo -e "\tcheck_name. Scenario name to check. This is the name of the directory on GROUP dir." } -while getopts 'hCd:p:Rs:DTPGgJKm' opt; do +while getopts 'hCd:p:Rs:DTPGgrJKm' opt; do case $opt in h) usage; exit 0;; C) SKIP=true;; @@ -529,6 +582,7 @@ while getopts 'hCd:p:Rs:DTPGgJKm' opt; do K) CAPTURE=true;; G) GRAPH=true;; g) GRAPH_FAIL=true;; + r) FIX_RETRANS=true;; J) JSON_KAM=true;; m) MEMDBG=true;; esac @@ -630,6 +684,42 @@ if ! "$SKIP_RUNSIPP" ; then echo "$(date) - No json files found" fi fi + + if "${FIX_RETRANS}" ; then + echo "$(date) - Checking retransmission issues" + RETRANS_ISSUE=false + file_find=($(find "${LOG_DIR}" -maxdepth 1 -name '*.json' | sort)) + for json_file in "${file_find[@]}" ; do + file_find=("${file_find[@]:1}") + if ! [ -a "${json_file}" ] ; then + continue + fi + for next_json_file in "${file_find[@]}" ; do + if ! [ -a "${next_json_file}" ] ; then + continue + fi + if ( diff -q -u <(tail -n3 "${json_file}") <(tail -n3 "${next_json_file}") &> /dev/null ) ; then + echo "$(basename "${next_json_file}") seems a retransmission of $(basename "${json_file}") ---> renaming the file in ${next_json_file}_retransmission" + mv -f "${next_json_file}" "${next_json_file}_retransmission" + RETRANS_ISSUE=true + fi + done + done + + if "${RETRANS_ISSUE}" ; then + echo "$(date) - Reordering kamailio json files" + file_find=($(find "${LOG_DIR}" -maxdepth 1 -name '*.json' | sort)) + a=1 + for json_file in "${file_find[@]}" ; do + new_name=$(printf "%04d.json" "${a}") + mv -n "${json_file}" "${LOG_DIR}/${new_name}" &> /dev/null + let a=a+1 + done + fi + + echo "$(date) - Done" + fi + fi diff --git a/get_results.sh b/get_results.sh index 7656ceb3..3c58f074 100755 --- a/get_results.sh +++ b/get_results.sh @@ -5,6 +5,7 @@ BIN_DIR="${BASE_DIR}/bin" PROFILE="CE" DOMAIN="spce.test" GROUP="${GROUP:-scenarios}" +RETRANS="" usage() { echo "Usage: get_results.sh [-p PROFILE] [-h] [-g]" @@ -14,6 +15,7 @@ usage() { echo "-h this help" echo "-P parse only will disable test" echo "-T test only will disable parse" + echo "-r fix retransmission issues" echo "-x set GROUP scenario. Default: scenarios" echo "BASE_DIR:${BASE_DIR}" echo "BIN_DIR:${BIN_DIR}" @@ -39,13 +41,14 @@ get_scenarios() { fi } -while getopts 'hgGp:TPx:' opt; do +while getopts 'hgGp:TPrx:' opt; do case $opt in h) usage; exit 0;; G) GRAPH="-G";; g) GRAPH="-g";; P) OPTS="-T";; T) OPTS="-P";; + r) RETRANS="-r";; p) PROFILE=$OPTARG;; x) GROUP=$OPTARG;; esac @@ -67,7 +70,7 @@ fi get_scenarios echo "${SCENARIOS}" | tr ' ' '\n' \ - | parallel "${BIN_DIR}/check.sh ${GRAPH} -J -C -R ${OPTS} -d ${DOMAIN} -p ${PROFILE} -s ${GROUP}" + | parallel "${BIN_DIR}/check.sh ${GRAPH} -J -C -R ${OPTS} ${RETRANS} -d ${DOMAIN} -p ${PROFILE} -s ${GROUP}" status=$? echo "$(date) - All done[$status]" exit $status diff --git a/run_tests.sh b/run_tests.sh index 7f326109..a0b8399a 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -15,6 +15,7 @@ TIMEOUT=${TIMEOUT:-300} SHOW_SCENARIOS=false SKIP=false SKIP_CAPTURE=false +SKIP_RETRANS=false MEMDBG=false error_flag=0 @@ -26,6 +27,7 @@ usage() { echo "-K capture messages with tcpdump" echo "-x set GROUP scenario. Default: scenarios" echo "-t set timeout in secs for pid_watcher.py [PRO]. Default: 300" + echo "-r fix retransmission issues" echo "-h this help" echo "BASE_DIR:${BASE_DIR}" @@ -66,7 +68,7 @@ cfg_debug_off() { fi } -while getopts 'hlcp:Kx:t:m' opt; do +while getopts 'hlcp:Kx:t:rm' opt; do case $opt in h) usage; exit 0;; l) SHOW_SCENARIOS=true;; @@ -75,6 +77,7 @@ while getopts 'hlcp:Kx:t:m' opt; do K) SKIP_CAPTURE=true;; x) GROUP=$OPTARG;; t) TIMEOUT=$OPTARG;; + r) SKIP_RETRANS=true;; m) MEMDBG=true;; esac done @@ -165,6 +168,11 @@ if "${MEMDBG}" ; then OPTS+=(-m) fi +if "${SKIP_RETRANS}" ; then + echo "$(date) - enable skip retransmissions" + OPTS+=(-r) +fi + for t in ${SCENARIOS}; do echo "$(date) - Run[${GROUP}/${PROFILE}]: $t =================================================" log_temp="${LOG_DIR}/${t}"