You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
511 lines
16 KiB
511 lines
16 KiB
#!/bin/bash
|
|
RUN_DIR="$(dirname "$0")"
|
|
BASE_DIR=${BASE_DIR:-$RUN_DIR}
|
|
# absolute path
|
|
export BASE_DIR=$(readlink -f "${BASE_DIR}")
|
|
# Set up the environment, to use local perl modules
|
|
export PERL5LIB="${BASE_DIR}/lib"
|
|
BIN_DIR="${BASE_DIR}/bin"
|
|
GROUP="${GROUP:-scenarios}"
|
|
LOG_DIR="${BASE_DIR}/log/${GROUP}"
|
|
MLOG_DIR="${BASE_DIR}/mem"
|
|
KAM_LOG=${KAM_LOG:-"/var/log/ngcp/kamailio-proxy.log"}
|
|
KAMLB_LOG=${KAMLB_LOG:-"/var/log/ngcp/kamailio-lb.log"}
|
|
SEMS_LOG=${SEMS_LOG:-"/var/log/ngcp/sems.log"}
|
|
SEMS_PBX_LOG=${SEMS_PBX_LOG:-"/var/log/ngcp/sems-pbx.log"}
|
|
RTP_LOG=${RTP_LOG:-"/var/log/ngcp/rtp.log"}
|
|
TMP_LOG_DIR="/tmp"
|
|
KAM_DIR="/tmp/cfgtest"
|
|
COREDUMP_DIR="/ngcp-data/coredumps"
|
|
PROFILE="${PROFILE:-}"
|
|
OPTS=(-t -M -C) #SKIP_TESTS=true, SKIP_MOVE_JSON_KAM=true, SKIP=true
|
|
CFG_OPTS=()
|
|
|
|
SHOW_SCENARIOS=false
|
|
SKIP_CONFIG=false
|
|
CAPTURE=false
|
|
SINGLE_CAPTURE=false
|
|
FIX_RETRANS=false
|
|
MEMDBG=false
|
|
CDR=false
|
|
PROV_TYPE="step"
|
|
START_TIME=$(date +%s)
|
|
error_flag=0
|
|
SCEN=()
|
|
CFGT=false
|
|
LOGTYPE="none"
|
|
|
|
get_scenarios() {
|
|
if [ -n "${SCEN_FILE}" ]; then
|
|
echo "$(date) - scenarios from file"
|
|
while read -r t; do
|
|
SCEN+=( "${t}" )
|
|
echo -e "\\t* ${t}"
|
|
done < "${SCEN_FILE}"
|
|
else
|
|
while read -r t; do
|
|
SCEN+=( "${t}" )
|
|
done < <("${BIN_DIR}/get_scenarios.sh" -p "${PROFILE}" -x "${GROUP}")
|
|
fi
|
|
if [[ ${#SCEN[@]} == 0 ]]; then
|
|
echo "$(date) no scenarios found"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
cfg_debug_off() {
|
|
if ! "${SKIP_CONFIG}" ; then
|
|
"${BASE_DIR}/set_config.sh" -c "${CFG_OPTS[@]}"
|
|
fi
|
|
}
|
|
|
|
copy_logs_to_tmp() {
|
|
# copy the kamailio log
|
|
cp -a "${KAM_LOG}" "${TMP_LOG_DIR}/tmp_kamailio.log"
|
|
if [ -f "${SEMS_LOG}" ] ; then
|
|
# copy the sems log
|
|
cp -a "${SEMS_LOG}" "${TMP_LOG_DIR}/tmp_sems.log"
|
|
fi
|
|
if [ -f "${SEMS_PBX_LOG}" ] ; then
|
|
# copy the sems-pbx log
|
|
cp -a "${SEMS_PBX_LOG}" "${TMP_LOG_DIR}/tmp_sems-pbx.log"
|
|
fi
|
|
# copy the kamailio-lb log
|
|
cp -a "${KAMLB_LOG}" "${TMP_LOG_DIR}/tmp_kamailio-lb.log"
|
|
}
|
|
|
|
copy_logs_from_tmp() {
|
|
# copy the kamailio log
|
|
cp -a "${TMP_LOG_DIR}/tmp_kamailio.log" "${KAM_LOG}"
|
|
if [ -f "${SEMS_LOG}" ] ; then
|
|
# copy the sems log
|
|
cp -a "${TMP_LOG_DIR}/tmp_sems.log" "${SEMS_LOG}"
|
|
fi
|
|
if [ -f "${TMP_LOG_DIR}/tmp_sems-pbx.log" ] ; then
|
|
# copy the sems-pbx log
|
|
cp -a "${TMP_LOG_DIR}/tmp_sems-pbx.log" "${SEMS_PBX_LOG}"
|
|
fi
|
|
# copy the kamailio-lb log
|
|
cp -a "${TMP_LOG_DIR}/tmp_kamailio-lb.log" "${KAMLB_LOG}"
|
|
}
|
|
|
|
capture() {
|
|
echo "$(date) - ================================================================================="
|
|
echo "$(date) - Start tcpdump captures"
|
|
datetime=$(date '+%y%m%d_%H%M')
|
|
for inter in $(ip link | grep '^[0-9]' | cut -d: -f2 | sed 's/ //' | xargs); do
|
|
tcpdump -i "${inter}" -n -s 65535 -w "${LOG_DIR}/_traces_${inter}_${datetime}.pcap" &
|
|
capture_pid="${capture_pid} ${inter}:$!"
|
|
done
|
|
echo "$(date) - Done"
|
|
echo "$(date) - ================================================================================="
|
|
}
|
|
|
|
stop_capture() {
|
|
echo "$(date) - ================================================================================="
|
|
echo "$(date) - Stop tcpdump captures"
|
|
local inter=""
|
|
local temp_pid=""
|
|
if [ -n "${capture_pid}" ]; then
|
|
for temp in ${capture_pid}; do
|
|
inter=$(echo "$temp"|cut -d: -f1)
|
|
temp_pid=$(echo "$temp"|cut -d: -f2)
|
|
#echo "inter:${inter} temp_pid:${temp_pid}"
|
|
if ps -p"${temp_pid}" &> /dev/null ; then
|
|
echo "$(date) - End ${inter}[${temp_pid}] capture"
|
|
kill -15 "${temp_pid}"
|
|
fi
|
|
done
|
|
fi
|
|
echo "$(date) - Done"
|
|
echo "$(date) - ================================================================================="
|
|
}
|
|
|
|
move_json_file() {
|
|
echo "$(date) - ================================================================================="
|
|
echo "$(date) - Move kamailio json files"
|
|
for t in "${SCEN[@]}"; do
|
|
echo "$(date) - - Scenario: ${t}"
|
|
json_dir="${KAM_DIR}/${t}"
|
|
if [ -d "${json_dir}" ] ; then
|
|
for i in "${json_dir}"/*.json ; do
|
|
json_size_before=$(stat -c%s "${i}")
|
|
moved_file="${LOG_DIR}/${t}/$(printf "%04d.json" "$(basename "$i" .json)")"
|
|
expand -t1 "${i}" > "${moved_file}"
|
|
json_size_after=$(stat -c%s "${moved_file}")
|
|
if [ "${json_size_before}" -ne "${json_size_after}" ] ; then
|
|
echo "$(date) - - - Moved file ${i} with size before: ${json_size_before} and after: ${json_size_after}"
|
|
fi
|
|
rm "${i}"
|
|
done
|
|
rm -rf "${json_dir}"
|
|
fi
|
|
done
|
|
|
|
echo "$(date) - Done"
|
|
echo "$(date) - ================================================================================="
|
|
}
|
|
|
|
fix_retransmissions() {
|
|
echo "$(date) - ================================================================================="
|
|
echo "$(date) - Checking retransmission issues"
|
|
for t in "${SCEN[@]}"; do
|
|
echo "$(date) - - Scenario: ${t}"
|
|
if [ "${t}" == "invite_retrans" ] ; then
|
|
continue
|
|
fi
|
|
RETRANS_ISSUE=false
|
|
mapfile -t file_find < <(find "${LOG_DIR}/${t}" -maxdepth 1 -name '*.json' | sort)
|
|
for json_file in "${file_find[@]}" ; do
|
|
file_find=("${file_find[@]:1}")
|
|
if ! [ -a "${json_file}" ] ; then
|
|
continue
|
|
fi
|
|
for next_json_file in "${file_find[@]}" ; do
|
|
if ! [ -a "${next_json_file}" ] ; then
|
|
continue
|
|
fi
|
|
# Check if both sip_in and Sip_out are equals
|
|
if ( diff -q -u <(tail -n3 "${json_file}") <(tail -n3 "${next_json_file}") &> /dev/null ) ; then
|
|
echo "$(date) - - - $(basename "${next_json_file}") seems a retransmission of $(basename "${json_file}") (case 1) ---> renaming the file in $(basename "${next_json_file}")_retransmission"
|
|
mv -f "${next_json_file}" "${next_json_file}_retransmission"
|
|
RETRANS_ISSUE=true
|
|
continue
|
|
fi
|
|
# Check if only sip_in is equal
|
|
if ( diff -q -u <(tail -n3 "${json_file}" | sed -n 1p) <(tail -n3 "${next_json_file}" | sed -n 1p) &> /dev/null ) ; then
|
|
echo "$(date) - - - $(basename "${next_json_file}") seems a retransmission of $(basename "${json_file}") (case 2) ---> renaming the file in $(basename "${next_json_file}")_retransmission"
|
|
mv -f "${next_json_file}" "${next_json_file}_retransmission"
|
|
RETRANS_ISSUE=true
|
|
continue
|
|
fi
|
|
done
|
|
done
|
|
|
|
if "${RETRANS_ISSUE}" ; then
|
|
echo "$(date) - - - Reordering kamailio json files"
|
|
mapfile -t file_find < <(find "${LOG_DIR}/${t}" -maxdepth 1 -name '*.json' | sort)
|
|
a=1
|
|
for json_file in "${file_find[@]}" ; do
|
|
new_name=$(printf "%04d.json" "${a}")
|
|
mv -n "${json_file}" "${LOG_DIR}/${t}/${new_name}" &> /dev/null
|
|
((a++))
|
|
done
|
|
fi
|
|
done
|
|
|
|
echo "$(date) - Done"
|
|
echo "$(date) - ================================================================================="
|
|
}
|
|
|
|
cdr_export() {
|
|
echo "$(date) - ================================================================================="
|
|
echo "$(date) - Extracting CDRs"
|
|
for t in "${SCEN[@]}"; do
|
|
echo "$(date) - - Scenario: $t"
|
|
if ! "${BIN_DIR}/cdr_extract.sh" -m -t "${START_TIME}" -s "${GROUP}" "${t}" ; then
|
|
echo "ERROR: ${t}"
|
|
error_flag=1
|
|
fi
|
|
done
|
|
|
|
echo "$(date) - Done"
|
|
echo "$(date) - ================================================================================="
|
|
}
|
|
|
|
get_config() {
|
|
local data=()
|
|
if [ ! -f "${BASE_DIR}/config.yml" ]; then
|
|
echo "can't read ${BASE_DIR}/config.yml file" >&2
|
|
exit 4
|
|
fi
|
|
mapfile -t data < <("${BIN_DIR}/get_config.pl" "${BASE_DIR}/config.yml")
|
|
# not used here
|
|
#IP=${data[0]}
|
|
#PORT=${data[1]}
|
|
#MPORT=${data[2]}
|
|
#PEER_IP=${data[3]}
|
|
#PEER_PORT=${data[4]}
|
|
#PEER_MPORT=${data[5]}
|
|
#PHONE_CC=${data[6]}
|
|
#PHONE_AC=${data[7]}
|
|
#PHONE_SN=${data[8]}
|
|
OPTS+=(-I"${data[9]}")
|
|
}
|
|
|
|
usage() {
|
|
echo "Usage: run_test.sh [-x GROUP] [-p PROFILE] [-f file] [-hlCckKrTm] [-P <full|step|none>]"
|
|
echo "Options:"
|
|
echo -e "\\t-p CE|PRO default is autodetect"
|
|
echo -e "\\t-l print available SCENARIOS in GROUP"
|
|
echo -e "\\t-C skips configuration of the environment"
|
|
echo -e "\\t-k capture messages with tcpdump, per scenario"
|
|
echo -e "\\t-K capture messages with tcpdump. One big file for all scenarios"
|
|
echo -e "\\t-x set GROUP scenario. Default: scenarios"
|
|
echo -e "\\t-r fix retransmission issues"
|
|
echo -e "\\t-c export CDRs at the end of the test"
|
|
echo -e "\\t-m mem debug"
|
|
echo -e "\\t-P provisioning, default:step"
|
|
echo -e "\\t\\tfull: provision all scenarios in one step"
|
|
echo -e "\\t\\tstep: provision scenario one by one before execution"
|
|
echo -e "\\t\\tnone: skip any provision"
|
|
echo -e "\\t-f scenarios file"
|
|
echo -e "\\t-T enable cfgt"
|
|
echo -e "\\t-L [caller|responder|all|none]: produce sipp logfile for <log> directive inside <action> (default none)"
|
|
echo -e "\\t-h this help"
|
|
|
|
echo "BASE_DIR:${BASE_DIR}"
|
|
echo "BIN_DIR:${BIN_DIR}"
|
|
}
|
|
|
|
while getopts 'f:hlCcP:p:kKx:rTmL:' opt; do
|
|
case $opt in
|
|
h) usage; exit 0;;
|
|
l) SHOW_SCENARIOS=true;;
|
|
C) SKIP_CONFIG=true;;
|
|
p) PROFILE=${OPTARG};;
|
|
P) PROV_TYPE=${OPTARG};;
|
|
k) SINGLE_CAPTURE=true;;
|
|
K) CAPTURE=true;;
|
|
x) GROUP=${OPTARG};;
|
|
r) FIX_RETRANS=true;;
|
|
c) CDR=true;;
|
|
m) MEMDBG=true;;
|
|
f) SCEN_FILE=${OPTARG};;
|
|
T) CFGT=true; CFG_OPTS+=(-T); OPTS+=(-Tall);;
|
|
L) case $OPTARG in "none" | "caller" | "responders" | "all")
|
|
LOGTYPE=${OPTARG};;
|
|
*) echo "unknown LOGTYPE '$OPTARG'"; usage; exit 0 ;;
|
|
esac
|
|
;;
|
|
*) usage; exit 1;;
|
|
esac
|
|
done
|
|
shift $((OPTIND - 1))
|
|
|
|
if [[ $# -ne 0 ]]; then
|
|
echo "Wrong number or arguments"
|
|
usage
|
|
exit 1
|
|
fi
|
|
|
|
if "${SHOW_SCENARIOS}" ; then
|
|
get_scenarios
|
|
echo "${SCEN[@]}"
|
|
exit 0
|
|
fi
|
|
|
|
if [ -z "${PROFILE}" ]; then
|
|
ngcp_type=$(command -v ngcp-type)
|
|
if [ -n "${ngcp_type}" ]; then
|
|
case $(${ngcp_type}) in
|
|
sppro|carrier) PROFILE=PRO;;
|
|
spce) PROFILE=CE;;
|
|
*) ;;
|
|
esac
|
|
echo "ngcp-type: profile ${PROFILE}"
|
|
fi
|
|
fi
|
|
if [ "${PROFILE}" != "CE" ] && [ "${PROFILE}" != "PRO" ]; then
|
|
echo "PROFILE ${PROFILE} unknown"
|
|
usage
|
|
exit 2
|
|
fi
|
|
|
|
case "${PROV_TYPE}" in
|
|
full|step|none) ;;
|
|
*) echo "provisioning type:${PROV_TYPE} unknown" >&2; exit 2
|
|
esac
|
|
|
|
LOG_DIR="${BASE_DIR}/log/${GROUP}"
|
|
|
|
if ${CFGT} ; then
|
|
if ! [ -d "${KAM_DIR}" ]; then
|
|
echo "$(date) - Create temporary folder for json files"
|
|
mkdir -p "${KAM_DIR}"
|
|
chown kamailio:root "${KAM_DIR}"
|
|
chmod 0770 "${KAM_DIR}"
|
|
else
|
|
echo "$(date) - Clean temporary folder for json files"
|
|
rm -rf "${KAM_DIR:?}/*"
|
|
fi
|
|
fi
|
|
|
|
echo "$(date) - Clean mem log dir"
|
|
rm -rf "${MLOG_DIR}"
|
|
mkdir -p "${MLOG_DIR}" "${LOG_DIR}"
|
|
|
|
if ! "${SKIP_CONFIG}" ; then
|
|
CFG_OPTS+=( -x "${GROUP}" -p "${PROFILE}" )
|
|
"${BASE_DIR}/set_config.sh" "${CFG_OPTS[@]}"
|
|
fi
|
|
|
|
echo "$(date) - Initial mem stats"
|
|
VERSION="${PROFILE}_$(cut -f1 -d' '< /etc/ngcp_version)_"
|
|
"${BIN_DIR}/mem_stats.py" --private_file="${MLOG_DIR}/${VERSION}_${GROUP}_initial_pvm.cvs" \
|
|
--share_file="${MLOG_DIR}/${VERSION}_${GROUP}_initial_shm.cvs"
|
|
if "${MEMDBG}" ; then
|
|
ngcp-memdbg-csv /var/log/ngcp/kamailio-proxy.log "${MLOG_DIR}" >/dev/null
|
|
fi
|
|
|
|
get_scenarios
|
|
|
|
if "${MEMDBG}" ; then
|
|
echo "$(date) - enable memdbg"
|
|
OPTS+=(-m)
|
|
fi
|
|
|
|
if "${CDR}" ; then
|
|
echo "$(date) - enable cdr export at the end of the execution"
|
|
fi
|
|
|
|
echo "$(date) - backup kamailio and sems logs to temp files"
|
|
copy_logs_to_tmp
|
|
if ! "${BIN_DIR}/restart_log.sh" ; then
|
|
echo "$(date) - error during initial restart of service logs"
|
|
copy_logs_from_tmp
|
|
exit 4
|
|
fi
|
|
|
|
if "${CAPTURE}" ; then
|
|
capture
|
|
elif "${SINGLE_CAPTURE}" ; then
|
|
OPTS+=(-K)
|
|
fi
|
|
|
|
if [[ "${PROV_TYPE}" == "full" ]] ; then
|
|
echo "$(date) - Provide all scenarios"
|
|
SCENARIOS="${SCEN[*]}" "${BIN_DIR}/provide_scenarios.sh" \
|
|
-f "${BASE_DIR}/config.yml" -x "${GROUP}" create || error_flag=1
|
|
fi
|
|
|
|
# find SERVER_IP
|
|
get_config
|
|
|
|
failed=()
|
|
for t in "${SCEN[@]}"; do
|
|
echo "$(date) - ================================================================================="
|
|
echo "$(date) - Run [${GROUP}/${PROFILE}]: ${t}"
|
|
|
|
log_temp="${LOG_DIR}/${t}"
|
|
if [ -d "${log_temp}" ]; then
|
|
echo "$(date) - Clean log dir"
|
|
rm -rf "${log_temp}"
|
|
fi
|
|
|
|
json_temp="${KAM_DIR}/${t}"
|
|
if [ -d "${json_temp}" ]; then
|
|
echo "$(date) - Clean json dir"
|
|
rm -rf "${json_temp}"
|
|
fi
|
|
|
|
if [[ "${PROV_TYPE}" == "step" ]] ; then
|
|
SCENARIOS="${t}" "${BIN_DIR}/provide_scenarios.sh" \
|
|
-f "${BASE_DIR}/config.yml" -x "${GROUP}" create
|
|
fi
|
|
|
|
if ! "${BIN_DIR}/check.sh" "${OPTS[@]}" -L "${LOGTYPE}" -p "${PROFILE}" -s "${GROUP}" "${t}" ; then
|
|
echo "ERROR: ${t}"
|
|
failed+=( "${t}" )
|
|
error_flag=1
|
|
fi
|
|
|
|
if [[ "${PROV_TYPE}" == "step" ]] ; then
|
|
SCENARIOS="${t}" "${BIN_DIR}/provide_scenarios.sh" \
|
|
-f "${BASE_DIR}/config.yml" -x "${GROUP}" delete
|
|
fi
|
|
|
|
echo "$(date) - ================================================================================="
|
|
|
|
# Check if core files have been geneared during the test execution
|
|
coredumps=$(find "${COREDUMP_DIR}" -regex '.*kamailio.*\|.*sems.*\|.*asterisk.*' -print)
|
|
if [ -n "${coredumps}" ]; then
|
|
echo "$(date) - ================================================================================="
|
|
echo "$(date) - One or more coredump files have been generated during the test execution."
|
|
echo "$(date) - Check the following files under the folder '${COREDUMP_DIR}':"
|
|
echo "${coredumps}"
|
|
echo "$(date) - ================================================================================="
|
|
error_flag=1
|
|
break
|
|
fi
|
|
done
|
|
|
|
rm -f "${LOG_DIR}/run_failed.txt"
|
|
for t in ${failed[*]}; do
|
|
echo "$t" >> "${LOG_DIR}/run_failed.txt"
|
|
done
|
|
|
|
if [[ "${PROV_TYPE}" == "full" ]] ; then
|
|
echo "$(date) - Delete provided scenarios"
|
|
SCENARIOS="${SCEN[*]}" "${BIN_DIR}/provide_scenarios.sh" \
|
|
-f "${BASE_DIR}/config.yml" -x "${GROUP}" delete
|
|
fi
|
|
|
|
# Hack to allow:
|
|
# - tcpdump to capture all the packages
|
|
# - kamailio to write all the json files
|
|
# - rtpengine to close rtp ports
|
|
# - write all the CDRs
|
|
sleep 6
|
|
|
|
# Check if there are still some rtp port open after tests execution
|
|
rtpengine_ctl_ip=$(grep 'listen-cli' /etc/rtpengine/rtpengine.conf |awk '{print $3}')
|
|
rtp_ports=$(rtpengine-ctl -ip "${rtpengine_ctl_ip}" list interfaces |grep "Ports used" | awk '{print $3}')
|
|
if [ -n "${rtp_ports}" ]; then
|
|
while read -r i; do
|
|
if [ "${i}" -gt 0 ]; then
|
|
echo "$(date) - ================================================================================="
|
|
echo "$(date) - There are still some rtp ports open, please check the following output"
|
|
rtpengine-ctl -ip "${rtpengine_ctl_ip}" list interfaces
|
|
rtpengine-ctl -ip "${rtpengine_ctl_ip}" list sessions all
|
|
echo "$(date) - copy rtpengine log to ${LOG_DIR}"
|
|
cp -a "${RTP_LOG}" "${LOG_DIR}"
|
|
echo "$(date) - ================================================================================="
|
|
error_flag=1
|
|
fi
|
|
done <<< "${rtp_ports}"
|
|
fi
|
|
|
|
if "${CAPTURE}" ; then
|
|
stop_capture
|
|
fi
|
|
|
|
echo "$(date) - restore kamailio and sems logs with the original content"
|
|
copy_logs_from_tmp
|
|
service rsyslog restart
|
|
|
|
if ${CFGT} ; then
|
|
move_json_file
|
|
|
|
if "${FIX_RETRANS}" ; then
|
|
fix_retransmissions
|
|
fi
|
|
fi
|
|
|
|
if "${CDR}" ; then
|
|
cdr_export
|
|
fi
|
|
|
|
echo "$(date) - Final mem stats"
|
|
"${BIN_DIR}/mem_stats.py" --private_file="${MLOG_DIR}/${VERSION}_${GROUP}_final_pvm.cvs" \
|
|
--share_file="${MLOG_DIR}/${VERSION}_${GROUP}_final_shm.cvs"
|
|
if [[ ${MEMDBG} = 1 ]] ; then
|
|
ngcp-memdbg-csv /var/log/ngcp/kamailio-proxy.log "${MLOG_DIR}" >/dev/null
|
|
fi
|
|
|
|
if ${CFGT} && [ -d "${KAM_DIR}" ]; then
|
|
echo "$(date) - clean cfgt scenarios"
|
|
ngcp-kamcmd proxy cfgt.clean all
|
|
echo "$(date) - Removing temporary json dir"
|
|
rm -rf "${KAM_DIR}"
|
|
fi
|
|
|
|
cfg_debug_off
|
|
|
|
echo "$(date) - Done[${error_flag}]"
|
|
if [ -f "${LOG_DIR}/run_failed.txt" ]; then
|
|
echo "$(date) - Failed scenarios:"
|
|
cat "${LOG_DIR}/run_failed.txt"
|
|
fi
|
|
exit ${error_flag}
|