CI: Allow runUnittests to use 'expect' to run the tests

There seems to be a race condition between starting the asterisk
daemon and attempting to use 'asterisk -r' that can cause the
control socket file to not be created.  Since all of the Jenkins
slaves have 'expect' installed, the runUnittests script can use
it to start asterisk in the forground and issue the commands
interactively.  This is much more reliable and it can also make
startup errors more visible since they'll be in the Jenkins console
output.

If 'expect' isn't installed, the original daemon/asterisk -r
process is used.

Also added a "core show settings" before running the tests
and added "notice,warning,error" to the console log.

Change-Id: Idd656085f854afede813ac241b9e312b31358160

tests/CI/runUnittests.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 CIDIR=$(dirname $(readlink -fn $0))
+NO_EXPECT=0
 source $CIDIR/ci.functions
 ASTETCDIR=$DESTDIR/etc/asterisk
 
@@ -14,9 +15,54 @@ asterisk_corefile_glob() {
 	fi
 }
 
+run_tests_expect() {
+$EXPECT <<-EOF
+	spawn sudo $ASTERISK ${USER_GROUP:+-U ${USER_GROUP%%:*} -G ${USER_GROUP##*:}} -fcng -C $CONFFILE
+	match_max 512
+	set timeout 600
+	expect -notransfer "Asterisk Ready."
+	send "core show settings\r"
+	expect -notransfer "CLI>"
+	send "${UNITTEST_COMMAND:-test execute all}\r"
+	expect -notransfer -ex "Test(s) Executed"
+	expect -notransfer "CLI>"
+	send "test show results failed\r"
+	expect -notransfer "CLI>"
+	send "test generate results xml ${OUTPUTFILE}\r"
+	expect -notransfer "CLI>"
+	send "core stop now\r"
+	expect -notransfer "Executing last minute cleanups"
+	wait
+EOF
+}
+
+run_tests_socket() {
+	sudo $ASTERISK ${USER_GROUP:+-U ${USER_GROUP%%:*} -G ${USER_GROUP##*:}} -gn -C $CONFFILE
+	for n in {1..5} ; do
+		sleep 3
+		$ASTERISK -rx "core waitfullybooted" -C $CONFFILE && break
+	done
+	sleep 1
+	$ASTERISK -rx "core show settings" -C $CONFFILE
+	$ASTERISK -rx "${UNITTEST_COMMAND:-test execute all}" -C $CONFFILE
+	$ASTERISK -rx "test show results failed" -C $CONFFILE
+	$ASTERISK -rx "test generate results xml $OUTPUTFILE" -C $CONFFILE
+	$ASTERISK -rx "core stop now" -C $CONFFILE
+}
+
+# If DESTDIR is used to install and run asterisk from non standard locations,
+# the directory entries in asterisk.conf need to be munged to prepend DESTDIR.
+ALTERED=$(head -10 ../tmp/DESTDIR/etc/asterisk/asterisk.conf | grep -q "DESTDIR" && echo yes)
+if [ x"$ALTERED" = x ] ; then
+	# In the section that starts with [directories and ends with a blank line,
+	# replace "=> " with "=> ${DESTDIR}"
+	sed -i -r -e "/^\[directories/,/^$/ s@=>\s+@=> ${DESTDIR}@" "$ASTETCDIR/asterisk.conf"
+fi
+
 cat <<-EOF > "$ASTETCDIR/logger.conf"
 	[logfiles]
 	full => notice,warning,error,debug,verbose
+	console => notice,warning,error
 EOF
 
 echo "[default]" > "$ASTETCDIR/extensions.conf"
@@ -58,6 +104,7 @@ ASTERISK="$DESTDIR/usr/sbin/asterisk"
 CONFFILE=$ASTETCDIR/asterisk.conf
 OUTPUTDIR=${OUTPUT_DIR:-tests/CI/output/}
 OUTPUTFILE=${OUTPUT_XML:-${OUTPUTDIR}/unittests-results.xml}
+EXPECT="$(which expect 2>/dev/null || : )"
 
 [ ! -d ${OUTPUTDIR} ] && mkdir -p $OUTPUTDIR
 [ x"$USER_GROUP" != x ] && sudo chown -R $USER_GROUP $OUTPUTDIR
@@ -65,16 +112,14 @@ OUTPUTFILE=${OUTPUT_XML:-${OUTPUTDIR}/unittests-results.xml}
 rm -rf $ASTETCDIR/extensions.{ael,lua} || :
 
 set -x
-sudo $ASTERISK ${USER_GROUP:+-U ${USER_GROUP%%:*} -G ${USER_GROUP##*:}} -gn -C $CONFFILE
-for n in {1..5} ; do
-	sleep 3
-	$ASTERISK -rx "core waitfullybooted" -C $CONFFILE && break
-done
-sleep 1
-$ASTERISK -rx "${UNITTEST_COMMAND:-test execute all}" -C $CONFFILE
-$ASTERISK -rx "test show results failed" -C $CONFFILE
-$ASTERISK -rx "test generate results xml $OUTPUTFILE" -C $CONFFILE
-$ASTERISK -rx "core stop now" -C $CONFFILE
+if [ x"$EXPECT" != x -a $NO_EXPECT -eq 0 ] ; then
+	run_tests_expect
+else
+	run_tests_socket
+fi
+
+# Cleanup "just in case"
+sudo killall -qe -ABRT $ASTERISK 
 
 runner rsync -vaH $DESTDIR/var/log/asterisk/. $OUTPUTDIR
 set +x
@@ -86,6 +131,7 @@ do
 	if [ -f $core ]
 	then
 		echo "*** Found a core file ($core) after running unit tests ***"
+		set -x
 		sudo OUTPUTDIR=$OUTPUTDIR $DESTDIR/var/lib/asterisk/scripts/ast_coredumper --no-default-search $core
 	fi
 done