TT#42150 Improve tests execution script

* Added 5 seconds of sleep time at the end of execution in order to allow tcpdump to capture all the packages and kamailio to write all json files * All json files are collected at the end of all the scenarios * Only one big tcpdump trace is takes to collect all the scenarios Change-Id: Id91835a959785080ecbbaf29a3e4c97bf752934e
7 years ago · 53b3257579
parent f6e69a64ae
commit 53b3257579
2 changed files with 170 additions and 48 deletions
--- a/bin/cdr_extract.sh
+++ b/bin/cdr_extract.sh
@ -21,6 +21,7 @@


 START_TIME=""
+MUTE=false


 # sipwise password for mysql connections
@ -32,15 +33,17 @@ usage() {
  echo "Options:"
  echo -e "\t-t Tests start time in order to better tune the mysql query."
  echo -e "\t-s scenario group. Default: scenarios"
+  echo -e "\t-m mute the console output"
  echo "Arguments:"
  echo -e "\tcheck_name. Scenario name to check. This is the name of the directory on GROUP dir."
 }

-while getopts 'ht:s:' opt; do
+while getopts 'ht:s:m' opt; do
  case $opt in
    h) usage; exit 0;;
    t) START_TIME=$OPTARG;;
    s) GROUP=$OPTARG;;
+    m) MUTE=true;;
  esac
 done
 shift $((OPTIND - 1))
@ -64,11 +67,14 @@ BASE_DIR="${BASE_DIR:-/usr/share/kamailio-config-tests}"
 LOG_DIR="${BASE_DIR}/log/${GROUP}/${NAME_CHECK}"


+if ! "${MUTE}" ; then
  echo "$(date) - Exporting generated CDRs"
+fi
 mysql -usipwise -p"${SIPWISE_DB_PASSWORD}" accounting \
      -e "select * from cdr where call_id like 'NGCP\%${NAME_CHECK}\%%' and start_time > ${START_TIME} order by id desc limit 10\G" > "${LOG_DIR}/cdr.txt" || true
+if ! "${MUTE}" ; then
 echo "$(date) - Done"
-
+fi

 exit 0
 #EOF
--- a/run_tests.sh
+++ b/run_tests.sh
@ -9,34 +9,18 @@ LOG_DIR="${BASE_DIR}/log/${GROUP}"
 MLOG_DIR="${BASE_DIR}/mem"
 KAM_DIR="/tmp/cfgtest"
 PROFILE="CE"
-OPTS=(-J -P -T)
+OPTS=(-P -T)
 DOMAIN="spce.test"
 TIMEOUT=${TIMEOUT:-300}
 SHOW_SCENARIOS=false
-SKIP=false
-SKIP_CAPTURE=false
-SKIP_RETRANS=false
+SKIP_CONFIG=false
+CAPTURE=false
+FIX_RETRANS=false
 MEMDBG=false
 CDR=false
 START_TIME=$(date +%s)
 error_flag=0

-usage() {
-  echo "Usage: run_test.sh [-p PROFILE] [-C] [-t]"
-  echo "-p CE|PRO default is CE"
-  echo "-l print available SCENARIOS in GROUP"
-  echo "-C skips configuration of the environment"
-  echo "-K capture messages with tcpdump"
-  echo "-x set GROUP scenario. Default: scenarios"
-  echo "-t set timeout in secs for pid_watcher.py [PRO]. Default: 300"
-  echo "-r fix retransmission issues"
-  echo "-c export CDRs at the end of the test"
-  echo "-h this help"
-
-  echo "BASE_DIR:${BASE_DIR}"
-  echo "BIN_DIR:${BIN_DIR}"
-}
-
 get_scenarios() {
  local t
  local flag
@ -58,7 +42,7 @@ get_scenarios() {
 }

 cfg_debug_off() {
-  if ! "${SKIP}" ; then
+  if ! "${SKIP_CONFIG}" ; then
    echo "$(date) - Removed apicert.pem"
    rm -f "${BASE_DIR}/apicert.pem"
    echo "$(date) - Setting config debug off"
@ -71,16 +55,137 @@ cfg_debug_off() {
  fi
 }

+capture() {
+  echo "$(date) - Begin capture"
+  datetime=$(date '+%y%m%d_%H%M')
+  for inter in $(ip link | grep '^[0-9]' | cut -d: -f2 | sed 's/ //' | xargs); do
+    tcpdump -i "${inter}" -n -s 65535 -w "${LOG_DIR}/_traces_${inter}_${datetime}.pcap" &
+    capture_pid="$capture_pid ${inter}:$!"
+  done
+}
+
+stop_capture() {
+  local inter=""
+  local temp_pid=""
+  if [ -n "${capture_pid}" ]; then
+    for temp in ${capture_pid}; do
+      inter=$(echo "$temp"|cut -d: -f1)
+      temp_pid=$(echo "$temp"|cut -d: -f2)
+      #echo "inter:${inter} temp_pid:${temp_pid}"
+      if ps -p"${temp_pid}" &> /dev/null ; then
+        echo "$(date) - End ${inter}[$temp_pid] capture"
+        kill -15 "${temp_pid}"
+      fi
+    done
+  fi
+}
+
+move_json_file() {
+  echo "$(date) - ================================================================================="
+  echo "$(date) - Move kamailio json files"
+  for t in ${SCENARIOS}; do
+    echo "$(date) - - Scenarios $t ================================================="
+    json_dir="${KAM_DIR}/${t}"
+    if [ -d "${json_dir}" ] ; then
+      for i in "${json_dir}"/*.json ; do
+        json_size_before=$(stat -c%s "${i}")
+        moved_file="${LOG_DIR}/${t}/$(printf "%04d.json" "$(basename "$i" .json)")"
+        expand -t1 "$i" > "${moved_file}"
+        json_size_after=$(stat -c%s "${moved_file}")
+        echo "$(date) - - - Moved file ${i} with size before: ${json_size_before} and after: ${json_size_after}"
+        rm "$i"
+      done
+      rm -rf "${json_dir}"
+    fi
+  done
+
+  echo "$(date) - Done"
+  echo "$(date) - ================================================================================="
+}
+
+fix_retransmissions() {
+  echo "$(date) - ================================================================================="
+  echo "$(date) - Checking retransmission issues"
+  for t in ${SCENARIOS}; do
+    echo "$(date) - - Scenarios $t ================================================="
+    RETRANS_ISSUE=false
+    file_find=($(find "${LOG_DIR}/${t}" -maxdepth 1 -name '*.json' | sort))
+    for json_file in "${file_find[@]}" ; do
+      file_find=("${file_find[@]:1}")
+      if ! [ -a "${json_file}" ] ; then
+        continue
+      fi
+      for next_json_file in "${file_find[@]}" ; do
+        if ! [ -a "${next_json_file}" ] ; then
+          continue
+        fi
+        if ( diff -q -u <(tail -n3 "${json_file}") <(tail -n3 "${next_json_file}") &> /dev/null ) ; then
+          echo "$(basename "${next_json_file}") seems a retransmission of $(basename "${json_file}") ---> renaming the file in ${next_json_file}_retransmission"
+          mv -f "${next_json_file}" "${next_json_file}_retransmission"
+          RETRANS_ISSUE=true
+        fi
+      done
+    done
+
+    if "${RETRANS_ISSUE}" ; then
+      echo "$(date) - Reordering kamailio json files"
+      file_find=($(find "${LOG_DIR}/${t}" -maxdepth 1 -name '*.json' | sort))
+      a=1
+      for json_file in "${file_find[@]}" ; do
+        new_name=$(printf "%04d.json" "${a}")
+        mv -n "${json_file}" "${LOG_DIR}/${t}/${new_name}" &> /dev/null
+        let a=a+1
+      done
+    fi
+  done
+
+  echo "$(date) - Done"
+  echo "$(date) - ================================================================================="
+}
+
+cdr_export() {
+  echo "$(date) - ================================================================================="
+  echo "$(date) - Extracting CDRs"
+  for t in ${SCENARIOS}; do
+    echo "$(date) - - Scenarios $t ================================================="
+    if ! "${BIN_DIR}/cdr_extract.sh" -m -t "${START_TIME}" -s "${GROUP}" "$t" ; then
+      echo "ERROR: $t"
+      error_flag=1
+    fi
+  done
+
+  echo "$(date) - Done"
+  echo "$(date) - ================================================================================="
+}
+
+usage() {
+  echo "Usage: run_test.sh [-p PROFILE] [-C] [-t]"
+  echo "Options:"
+  echo -e "\t-p CE|PRO default is CE"
+  echo -e "\t-l print available SCENARIOS in GROUP"
+  echo -e "\t-C skips configuration of the environment"
+  echo -e "\t-K capture messages with tcpdump"
+  echo -e "\t-x set GROUP scenario. Default: scenarios"
+  echo -e "\t-t set timeout in secs for pid_watcher.py [PRO]. Default: 300"
+  echo -e "\t-r fix retransmission issues"
+  echo -e "\t-c export CDRs at the end of the test"
+  echo -e "\t-m mem debug"
+  echo -e "\t-h this help"
+
+  echo "BASE_DIR:${BASE_DIR}"
+  echo "BIN_DIR:${BIN_DIR}"
+}
+
 while getopts 'hlCcp:Kx:t:rm' opt; do
  case $opt in
    h) usage; exit 0;;
    l) SHOW_SCENARIOS=true;;
-    C) SKIP=true;;
+    C) SKIP_CONFIG=true;;
    p) PROFILE=$OPTARG;;
-    K) SKIP_CAPTURE=true;;
+    K) CAPTURE=true;;
    x) GROUP=$OPTARG;;
    t) TIMEOUT=$OPTARG;;
-    r) SKIP_RETRANS=true;;
+    r) FIX_RETRANS=true;;
    c) CDR=true;;
    m) MEMDBG=true;;
  esac
@ -115,18 +220,20 @@ else
  PIDWATCH_OPTS=""
 fi

+LOG_DIR="${BASE_DIR}/log/${GROUP}"
+
 echo "$(date) - Create temporary folder for json files"
 rm -rf "${KAM_DIR}"
 mkdir -p "${KAM_DIR}"
 if [ -d "${KAM_DIR}" ]; then
-  chown kamailio "${KAM_DIR}"
+  chown kamailio:kamailio "${KAM_DIR}"
 fi

 echo "$(date) - Clean mem log dir"
 rm -rf "${MLOG_DIR}"
 mkdir -p "${MLOG_DIR}" "${LOG_DIR}"

-if ! "${SKIP}" ; then
+if ! "${SKIP_CONFIG}" ; then
  echo "$(date) - Setting config debug on"
  "${BIN_DIR}/config_debug.pl" -g "${GROUP}" on ${DOMAIN}
  if [ "${PROFILE}" == "PRO" ]; then
@ -162,49 +269,58 @@ fi

 get_scenarios

-if "${SKIP_CAPTURE}" ; then
-  echo "$(date) - enable capture"
-  OPTS+=(-K)
-fi
-
 if "${MEMDBG}" ; then
  echo "$(date) - enable memdbg"
  OPTS+=(-m)
 fi

-if "${SKIP_RETRANS}" ; then
-  echo "$(date) - enable skip retransmissions"
-  OPTS+=(-r)
-fi
-
 if "${CDR}" ; then
  echo "$(date) - enable cdr export at the end of the execution"
 fi

+if "${CAPTURE}" ; then
+  capture
+fi
+
 for t in ${SCENARIOS}; do
  echo "$(date) - Run [${GROUP}/${PROFILE}]: $t ================================================="
+
  log_temp="${LOG_DIR}/${t}"
  if [ -d "${log_temp}" ]; then
    echo "$(date) - Clean log dir"
    rm -rf "${log_temp}"
  fi
-  if ! "${BIN_DIR}/check.sh" "${OPTS[@]}" -d ${DOMAIN} -p "${PROFILE}" -s "${GROUP}" "$t" ; then
+
+  json_temp="${KAM_DIR}/${t}"
+  if [ -d "${json_temp}" ]; then
+    echo "$(date) - Clean json dir"
+    rm -rf "${json_temp}"
+  fi
+
+  if ! "${BIN_DIR}/check.sh" "${OPTS[@]}" -d "${DOMAIN}" -p "${PROFILE}" -s "${GROUP}" "$t" ; then
    echo "ERROR: $t"
    error_flag=1
  fi
+
  echo "$(date) - ================================================================================="
 done

+# Hack to allow tcpdump to capture all the packages and kamailio to write all the json files
+sleep 5
+
+if "${CAPTURE}" ; then
+  stop_capture
+fi
+
+move_json_file
+
+if "${FIX_RETRANS}" ; then
+  fix_retransmissions
+fi
+
 if "${CDR}" ; then
  sleep 2
-  for t in ${SCENARIOS}; do
-    echo "$(date) - Extract CDRs for [${GROUP}/${PROFILE}]: $t ===================================="
-    if ! "${BIN_DIR}/cdr_extract.sh" -t "${START_TIME}" -s "${GROUP}" "$t" ; then
-      echo "ERROR: $t"
-      error_flag=1
-    fi
-    echo "$(date) - ================================================================================="
-  done
+  cdr_export
 fi

 echo "$(date) - Final mem stats"