You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
422 lines
13 KiB
422 lines
13 KiB
#!/bin/bash
|
|
RUN_DIR="$(dirname "$0")"
|
|
export BASE_DIR=${BASE_DIR:-$RUN_DIR}
|
|
# Set up the environment, to use local perl modules
|
|
export PERL5LIB="${BASE_DIR}/lib"
|
|
BIN_DIR="${BASE_DIR}/bin"
|
|
GROUP="${GROUP:-scenarios}"
|
|
LOG_DIR="${BASE_DIR}/log/${GROUP}"
|
|
MLOG_DIR="${BASE_DIR}/mem"
|
|
KAM_LOG=${KAM_LOG:-"/var/log/ngcp/kamailio-proxy.log"}
|
|
KAMLB_LOG=${KAMLB_LOG:-"/var/log/ngcp/kamailio-lb.log"}
|
|
SEMS_LOG=${SEMS_LOG:-"/var/log/ngcp/sems.log"}
|
|
SEMS_PBX_LOG=${SEMS_PBX_LOG:-"/var/log/ngcp/sems-pbx.log"}
|
|
TMP_LOG_DIR="/tmp"
|
|
KAM_DIR="/tmp/cfgtest"
|
|
COREDUMP_DIR="/ngcp-data/coredumps"
|
|
PROFILE="CE"
|
|
OPTS=(-P -T)
|
|
DOMAIN="spce.test"
|
|
TIMEOUT=${TIMEOUT:-300}
|
|
SHOW_SCENARIOS=false
|
|
SKIP_CONFIG=false
|
|
CAPTURE=false
|
|
FIX_RETRANS=false
|
|
MEMDBG=false
|
|
CDR=false
|
|
START_TIME=$(date +%s)
|
|
error_flag=0
|
|
|
|
get_scenarios() {
|
|
local t
|
|
local flag
|
|
flag=0
|
|
if [ -n "${SCENARIOS}" ]; then
|
|
for t in ${SCENARIOS}; do
|
|
if [ ! -d "${BASE_DIR}/${GROUP}/${t}" ]; then
|
|
echo "$(date) - scenario: $t at ${GROUP} not found"
|
|
flag=1
|
|
fi
|
|
done
|
|
if [ ${flag} != 0 ]; then
|
|
exit 1
|
|
fi
|
|
else
|
|
SCENARIOS=$(find "${BASE_DIR}/${GROUP}/" -depth -maxdepth 1 -mindepth 1 \
|
|
-type d -exec basename {} \; | grep -v templates | sort)
|
|
fi
|
|
}
|
|
|
|
cfg_debug_off() {
|
|
if ! "${SKIP_CONFIG}" ; then
|
|
echo "$(date) - Removed apicert.pem"
|
|
rm -f "${BASE_DIR}/apicert.pem"
|
|
echo "$(date) - Setting config debug off"
|
|
"${BIN_DIR}/config_debug.pl" -g "${GROUP}" off ${DOMAIN}
|
|
if ! ngcpcfg apply "config debug off via kamailio-config-tests" ; then
|
|
echo "$(date) - ngcpcfg apply returned $?"
|
|
error_flag=4
|
|
fi
|
|
echo "$(date) - Setting config debug off. Done[${error_flag}]"
|
|
fi
|
|
}
|
|
|
|
copy_logs_to_tmp() {
|
|
# copy the kamailio log
|
|
cp "${KAM_LOG}" "${TMP_LOG_DIR}/tmp_kamailio.log"
|
|
if [ -f "${SEMS_LOG}" ] ; then
|
|
# copy the sems log
|
|
cp "${SEMS_LOG}" "${TMP_LOG_DIR}/tmp_sems.log"
|
|
fi
|
|
if [ -f "${SEMS_PBX_LOG}" ] ; then
|
|
# copy the sems-pbx log
|
|
cp "${SEMS_PBX_LOG}" "${TMP_LOG_DIR}/tmp_sems-pbx.log"
|
|
fi
|
|
# copy the kamailio-lb log
|
|
cp "${KAMLB_LOG}" "${TMP_LOG_DIR}/tmp_kamailio-lb.log"
|
|
}
|
|
|
|
copy_logs_from_tmp() {
|
|
# copy the kamailio log
|
|
cp "${TMP_LOG_DIR}/tmp_kamailio.log" "${KAM_LOG}"
|
|
if [ -f "${SEMS_LOG}" ] ; then
|
|
# copy the sems log
|
|
cp "${TMP_LOG_DIR}/tmp_sems.log" "${SEMS_LOG}"
|
|
fi
|
|
if [ -f "${TMP_LOG_DIR}/tmp_sems-pbx.log" ] ; then
|
|
# copy the sems-pbx log
|
|
cp "${TMP_LOG_DIR}/tmp_sems-pbx.log" "${SEMS_PBX_LOG}"
|
|
fi
|
|
# copy the kamailio-lb log
|
|
cp "${TMP_LOG_DIR}/tmp_kamailio-lb.log" "${KAMLB_LOG}"
|
|
}
|
|
|
|
capture() {
|
|
echo "$(date) - ================================================================================="
|
|
echo "$(date) - Start tcpdump captures"
|
|
datetime=$(date '+%y%m%d_%H%M')
|
|
for inter in $(ip link | grep '^[0-9]' | cut -d: -f2 | sed 's/ //' | xargs); do
|
|
tcpdump -i "${inter}" -n -s 65535 -w "${LOG_DIR}/_traces_${inter}_${datetime}.pcap" &
|
|
capture_pid="${capture_pid} ${inter}:$!"
|
|
done
|
|
echo "$(date) - Done"
|
|
echo "$(date) - ================================================================================="
|
|
}
|
|
|
|
stop_capture() {
|
|
echo "$(date) - ================================================================================="
|
|
echo "$(date) - Stop tcpdump captures"
|
|
local inter=""
|
|
local temp_pid=""
|
|
if [ -n "${capture_pid}" ]; then
|
|
for temp in ${capture_pid}; do
|
|
inter=$(echo "$temp"|cut -d: -f1)
|
|
temp_pid=$(echo "$temp"|cut -d: -f2)
|
|
#echo "inter:${inter} temp_pid:${temp_pid}"
|
|
if ps -p"${temp_pid}" &> /dev/null ; then
|
|
echo "$(date) - End ${inter}[${temp_pid}] capture"
|
|
kill -15 "${temp_pid}"
|
|
fi
|
|
done
|
|
fi
|
|
echo "$(date) - Done"
|
|
echo "$(date) - ================================================================================="
|
|
}
|
|
|
|
move_json_file() {
|
|
echo "$(date) - ================================================================================="
|
|
echo "$(date) - Move kamailio json files"
|
|
for t in ${SCENARIOS}; do
|
|
echo "$(date) - - Scenario: ${t}"
|
|
json_dir="${KAM_DIR}/${t}"
|
|
if [ -d "${json_dir}" ] ; then
|
|
for i in "${json_dir}"/*.json ; do
|
|
json_size_before=$(stat -c%s "${i}")
|
|
moved_file="${LOG_DIR}/${t}/$(printf "%04d.json" "$(basename "$i" .json)")"
|
|
expand -t1 "${i}" > "${moved_file}"
|
|
json_size_after=$(stat -c%s "${moved_file}")
|
|
echo "$(date) - - - Moved file ${i} with size before: ${json_size_before} and after: ${json_size_after}"
|
|
rm "${i}"
|
|
done
|
|
rm -rf "${json_dir}"
|
|
fi
|
|
done
|
|
|
|
echo "$(date) - Done"
|
|
echo "$(date) - ================================================================================="
|
|
}
|
|
|
|
fix_retransmissions() {
|
|
echo "$(date) - ================================================================================="
|
|
echo "$(date) - Checking retransmission issues"
|
|
for t in ${SCENARIOS}; do
|
|
echo "$(date) - - Scenario: ${t}"
|
|
if [ "${t}" == "invite_retrans" ] ; then
|
|
continue
|
|
fi
|
|
RETRANS_ISSUE=false
|
|
file_find=($(find "${LOG_DIR}/${t}" -maxdepth 1 -name '*.json' | sort))
|
|
for json_file in "${file_find[@]}" ; do
|
|
file_find=("${file_find[@]:1}")
|
|
if ! [ -a "${json_file}" ] ; then
|
|
continue
|
|
fi
|
|
for next_json_file in "${file_find[@]}" ; do
|
|
if ! [ -a "${next_json_file}" ] ; then
|
|
continue
|
|
fi
|
|
# Check if both sip_in and Sip_out are equals
|
|
if ( diff -q -u <(tail -n3 "${json_file}") <(tail -n3 "${next_json_file}") &> /dev/null ) ; then
|
|
echo "$(date) - - - $(basename "${next_json_file}") seems a retransmission of $(basename "${json_file}") (case 1) ---> renaming the file in $(basename "${next_json_file}")_retransmission"
|
|
mv -f "${next_json_file}" "${next_json_file}_retransmission"
|
|
RETRANS_ISSUE=true
|
|
continue
|
|
fi
|
|
# Check if only sip_in is equal
|
|
if ( diff -q -u <(tail -n3 "${json_file}" | sed -n 1p) <(tail -n3 "${next_json_file}" | sed -n 1p) &> /dev/null ) ; then
|
|
echo "$(date) - - - $(basename "${next_json_file}") seems a retransmission of $(basename "${json_file}") (case 2) ---> renaming the file in $(basename "${next_json_file}")_retransmission"
|
|
mv -f "${next_json_file}" "${next_json_file}_retransmission"
|
|
RETRANS_ISSUE=true
|
|
continue
|
|
fi
|
|
done
|
|
done
|
|
|
|
if "${RETRANS_ISSUE}" ; then
|
|
echo "$(date) - - - Reordering kamailio json files"
|
|
file_find=($(find "${LOG_DIR}/${t}" -maxdepth 1 -name '*.json' | sort))
|
|
a=1
|
|
for json_file in "${file_find[@]}" ; do
|
|
new_name=$(printf "%04d.json" "${a}")
|
|
mv -n "${json_file}" "${LOG_DIR}/${t}/${new_name}" &> /dev/null
|
|
let a=a+1
|
|
done
|
|
fi
|
|
done
|
|
|
|
echo "$(date) - Done"
|
|
echo "$(date) - ================================================================================="
|
|
}
|
|
|
|
cdr_export() {
|
|
echo "$(date) - ================================================================================="
|
|
echo "$(date) - Extracting CDRs"
|
|
for t in ${SCENARIOS}; do
|
|
echo "$(date) - - Scenario: $t"
|
|
if ! "${BIN_DIR}/cdr_extract.sh" -m -t "${START_TIME}" -s "${GROUP}" "${t}" ; then
|
|
echo "ERROR: ${t}"
|
|
error_flag=1
|
|
fi
|
|
done
|
|
|
|
echo "$(date) - Done"
|
|
echo "$(date) - ================================================================================="
|
|
}
|
|
|
|
usage() {
|
|
echo "Usage: run_test.sh [-p PROFILE] [-C] [-t]"
|
|
echo "Options:"
|
|
echo -e "\t-p CE|PRO default is CE"
|
|
echo -e "\t-l print available SCENARIOS in GROUP"
|
|
echo -e "\t-C skips configuration of the environment"
|
|
echo -e "\t-K capture messages with tcpdump"
|
|
echo -e "\t-x set GROUP scenario. Default: scenarios"
|
|
echo -e "\t-t set timeout in secs for pid_watcher.py [PRO]. Default: 300"
|
|
echo -e "\t-r fix retransmission issues"
|
|
echo -e "\t-c export CDRs at the end of the test"
|
|
echo -e "\t-m mem debug"
|
|
echo -e "\t-h this help"
|
|
|
|
echo "BASE_DIR:${BASE_DIR}"
|
|
echo "BIN_DIR:${BIN_DIR}"
|
|
}
|
|
|
|
while getopts 'hlCcp:Kx:t:rm' opt; do
|
|
case $opt in
|
|
h) usage; exit 0;;
|
|
l) SHOW_SCENARIOS=true;;
|
|
C) SKIP_CONFIG=true;;
|
|
p) PROFILE=${OPTARG};;
|
|
K) CAPTURE=true;;
|
|
x) GROUP=${OPTARG};;
|
|
t) TIMEOUT=${OPTARG};;
|
|
r) FIX_RETRANS=true;;
|
|
c) CDR=true;;
|
|
m) MEMDBG=true;;
|
|
esac
|
|
done
|
|
shift $((OPTIND - 1))
|
|
|
|
if [[ $# -ne 0 ]]; then
|
|
echo "Wrong number or arguments"
|
|
usage
|
|
exit 1
|
|
fi
|
|
|
|
if "${SHOW_SCENARIOS}" ; then
|
|
get_scenarios
|
|
echo "${SCENARIOS}"
|
|
exit 0
|
|
fi
|
|
|
|
if [ "${PROFILE}" != "CE" ] && [ "${PROFILE}" != "PRO" ]; then
|
|
echo "PROFILE ${PROFILE} unknown"
|
|
usage
|
|
exit 2
|
|
fi
|
|
|
|
if [ "${GROUP}" = "scenarios_pbx" ] ; then
|
|
PIDWATCH_OPTS="--pbx"
|
|
# hack for pid_watcher ( sems-pbx was not active )
|
|
mkdir -p /var/run/sems-pbx/
|
|
touch /var/run/sems-pbx/sems-pbx.pid
|
|
chown -R sems-pbx:sems-pbx /var/run/sems-pbx/
|
|
else
|
|
PIDWATCH_OPTS=""
|
|
fi
|
|
|
|
LOG_DIR="${BASE_DIR}/log/${GROUP}"
|
|
|
|
echo "$(date) - Create temporary folder for json files"
|
|
rm -rf "${KAM_DIR}"
|
|
mkdir -p "${KAM_DIR}"
|
|
if [ -d "${KAM_DIR}" ]; then
|
|
chown kamailio:kamailio "${KAM_DIR}"
|
|
fi
|
|
|
|
echo "$(date) - Clean mem log dir"
|
|
rm -rf "${MLOG_DIR}"
|
|
mkdir -p "${MLOG_DIR}" "${LOG_DIR}"
|
|
|
|
if ! "${SKIP_CONFIG}" ; then
|
|
echo "$(date) - Setting config debug on"
|
|
"${BIN_DIR}/config_debug.pl" -g "${GROUP}" on ${DOMAIN}
|
|
if [ "${PROFILE}" == "PRO" ]; then
|
|
echo "$(date) - Exec pid_watcher with timeout[${TIMEOUT}]"
|
|
( timeout "${TIMEOUT}" "${BIN_DIR}/pid_watcher.py" ${PIDWATCH_OPTS} )&
|
|
fi
|
|
if ! ngcpcfg apply "config debug on via kamailio-config-tests" ; then
|
|
echo "$(date) - ngcp apply returned $?"
|
|
echo "$(date) - Done[3]"
|
|
cfg_debug_off
|
|
exit 3
|
|
fi
|
|
if [ "${PROFILE}" == "PRO" ]; then
|
|
echo "$(date) - waiting for pid_watcher[$!] result"
|
|
if ! wait "$!" ; then
|
|
echo "$(date) - error on apply config. Some expected service didn't restart"
|
|
echo "$(date) - check log/pid_watcher.log for details"
|
|
cfg_debug_off
|
|
echo "$(date) - Done[1]"
|
|
exit 1
|
|
fi
|
|
fi
|
|
echo "$(date) - Setting config debug on. Done[${error_flag}]."
|
|
fi
|
|
|
|
echo "$(date) - Initial mem stats"
|
|
VERSION="${PROFILE}_$(cut -f1 -d' '< /etc/ngcp_version)_"
|
|
"${BIN_DIR}/mem_stats.py" --private_file="${MLOG_DIR}/${VERSION}_${GROUP}_initial_pvm.cvs" \
|
|
--share_file="${MLOG_DIR}/${VERSION}_${GROUP}_initial_shm.cvs"
|
|
if "${MEMDBG}" ; then
|
|
ngcp-memdbg-csv /var/log/ngcp/kamailio-proxy.log "${MLOG_DIR}" >/dev/null
|
|
fi
|
|
|
|
get_scenarios
|
|
|
|
if "${MEMDBG}" ; then
|
|
echo "$(date) - enable memdbg"
|
|
OPTS+=(-m)
|
|
fi
|
|
|
|
if "${CDR}" ; then
|
|
echo "$(date) - enable cdr export at the end of the execution"
|
|
fi
|
|
|
|
echo "$(date) - backup kamailio and sems logs to temp files"
|
|
copy_logs_to_tmp
|
|
if ! "${BIN_DIR}/restart_log.sh" ; then
|
|
echo "$(date) - error during initial restart of service logs"
|
|
copy_logs_from_tmp
|
|
exit 4
|
|
fi
|
|
|
|
if "${CAPTURE}" ; then
|
|
capture
|
|
fi
|
|
|
|
for t in ${SCENARIOS}; do
|
|
echo "$(date) - ================================================================================="
|
|
echo "$(date) - Run [${GROUP}/${PROFILE}]: ${t}"
|
|
|
|
log_temp="${LOG_DIR}/${t}"
|
|
if [ -d "${log_temp}" ]; then
|
|
echo "$(date) - Clean log dir"
|
|
rm -rf "${log_temp}"
|
|
fi
|
|
|
|
json_temp="${KAM_DIR}/${t}"
|
|
if [ -d "${json_temp}" ]; then
|
|
echo "$(date) - Clean json dir"
|
|
rm -rf "${json_temp}"
|
|
fi
|
|
|
|
if ! "${BIN_DIR}/check.sh" "${OPTS[@]}" -d "${DOMAIN}" -p "${PROFILE}" -s "${GROUP}" "${t}" ; then
|
|
echo "ERROR: ${t}"
|
|
error_flag=1
|
|
fi
|
|
|
|
echo "$(date) - ================================================================================="
|
|
|
|
# Check if core files have been geneared during the test execution
|
|
coredumps=$(find "${COREDUMP_DIR}" -regex '.*kamailio.*\|.*sems.*\|.*asterisk.*' -print)
|
|
if [ -n "${coredumps}" ]; then
|
|
echo "$(date) - ================================================================================="
|
|
echo "$(date) - One or more coredump files have been generated during the test execution."
|
|
echo "$(date) - Check the following files under the folder '${COREDUMP_DIR}':"
|
|
echo "${coredumps}"
|
|
echo "$(date) - ================================================================================="
|
|
error_flag=1
|
|
break
|
|
fi
|
|
done
|
|
|
|
# Hack to allow tcpdump to capture all the packages and kamailio to write all the json files
|
|
sleep 5
|
|
|
|
if "${CAPTURE}" ; then
|
|
stop_capture
|
|
fi
|
|
|
|
echo "$(date) - restore kamailio and sems logs with the original content"
|
|
copy_logs_from_tmp
|
|
service rsyslog restart
|
|
|
|
move_json_file
|
|
|
|
if "${FIX_RETRANS}" ; then
|
|
fix_retransmissions
|
|
fi
|
|
|
|
if "${CDR}" ; then
|
|
sleep 2
|
|
cdr_export
|
|
fi
|
|
|
|
echo "$(date) - Final mem stats"
|
|
"${BIN_DIR}/mem_stats.py" --private_file="${MLOG_DIR}/${VERSION}_${GROUP}_final_pvm.cvs" \
|
|
--share_file="${MLOG_DIR}/${VERSION}_${GROUP}_final_shm.cvs"
|
|
if [[ ${MEMDBG} = 1 ]] ; then
|
|
ngcp-memdbg-csv /var/log/ngcp/kamailio-proxy.log "${MLOG_DIR}" >/dev/null
|
|
fi
|
|
|
|
if [ -d "${KAM_DIR}" ]; then
|
|
echo "$(date) - Removing temporary json dir"
|
|
rm -rf "${KAM_DIR}"
|
|
fi
|
|
|
|
cfg_debug_off
|
|
|
|
echo "$(date) - Done[${error_flag}]"
|
|
exit ${error_flag}
|