diff --git a/Makefile b/Makefile index 1aa5c9fd..9cf78a6a 100644 --- a/Makefile +++ b/Makefile @@ -1,27 +1,23 @@ +SCENARIOS:=$(shell find -maxdepth 1 -type d -name 'scenarios*'|sed 's_\./__g') +TESTS=$(addprefix test_,$(SCENARIOS)) + +RESULTS ?= reports + # do nothing as default all: -.ONESHELL: -SHELL = /bin/bash -venv: requirements.txt - virtualenv --python=python2.7 venv - source ./venv/bin/activate && \ - pip install -r ./requirements.txt >install.log +# python-junitxml 0.6 has this bug +# https://bugs.launchpad.net/pyjunitxml/+bug/892293 +$(TESTS): + @mkdir -p $(RESULTS) + $(eval SCEN_DIR := $(@:test_%=%)) + ./tests/do_test_yaml_format.sh $(SCEN_DIR) -test_check: venv tests/test_check.py - mkdir -p reports - source ./venv/bin/activate && \ - ./tests/test_check.py > reports/$(@).xml +test_check: tests/test_check.py + mkdir -p $(RESULTS) + ./tests/test_check.py > $(RESULTS)/$(@).xml # run this in parallel!! -j is your friend -test: test_check - -# get rid of test files -clean: - rm -rf install.log - -# also get rid of pip environment -dist-clean: clean - rm -rf venv +test: $(TESTS) test_check -.PHONY: all \ No newline at end of file +.PHONY: all $(TESTS) diff --git a/bin/generate_tests.sh b/bin/generate_tests.sh index 0968d763..717be1d9 100755 --- a/bin/generate_tests.sh +++ b/bin/generate_tests.sh @@ -21,6 +21,8 @@ BASE_DIR="${BASE_DIR:-/usr/share/kamailio-config-tests}" DIR="${BASE_DIR}/scenarios" BIN_DIR="${BASE_DIR}/bin" +DEST_DIR="${DEST_DIR}" + error_flag=0 function clean @@ -72,18 +74,24 @@ if [ ! -x "${BIN_DIR}/generate_test.pl" ]; then fi for t in $(find "${DIR}" -not -regex '.+customtt.tt2' -type f -name '*.tt2' | sort); do + origdir="$(dirname "$t")" template="$(basename "$t")" - destdir="$(dirname "$t")" + if [ -n "$DEST_DIR" ] ; then + destdir="$DEST_DIR/$(dirname "$t")" + mkdir -p "${destdir}" + else + destdir="$(dirname "$t")" + fi destfile="$(basename "$t" .tt2)" custom_template="$(basename "$t" .tt2).customtt.tt2" - if [ -f "${destdir}/${custom_template}" ]; then + if [ -f "${origdir}/${custom_template}" ]; then echo "Custom detected" template=${custom_template} fi echo "generating: ${destdir}/${destfile}" # shellcheck disable=SC2086 - if ! "${BIN_DIR}/generate_test.pl" ${ARGS} "${destdir}/${template}" ${IDS} > "${destdir}/${destfile}" ; then + if ! "${BIN_DIR}/generate_test.pl" ${ARGS} "${origdir}/${template}" ${IDS} > "${destdir}/${destfile}" ; then error_flag=1 fi done diff --git a/scenarios_disabled/invite_peerout_emergency/0001_test.yml.tt2 b/disabled/invite_peerout_emergency/0001_test.yml.tt2 similarity index 100% rename from scenarios_disabled/invite_peerout_emergency/0001_test.yml.tt2 rename to disabled/invite_peerout_emergency/0001_test.yml.tt2 diff --git a/scenarios_disabled/invite_peerout_emergency/0003_test.yml.tt2 b/disabled/invite_peerout_emergency/0003_test.yml.tt2 similarity index 100% rename from scenarios_disabled/invite_peerout_emergency/0003_test.yml.tt2 rename to disabled/invite_peerout_emergency/0003_test.yml.tt2 diff --git a/scenarios_disabled/invite_peerout_emergency/0007_test.yml.tt2 b/disabled/invite_peerout_emergency/0007_test.yml.tt2 similarity index 100% rename from scenarios_disabled/invite_peerout_emergency/0007_test.yml.tt2 rename to disabled/invite_peerout_emergency/0007_test.yml.tt2 diff --git a/scenarios_disabled/invite_peerout_emergency/0008_test.yml.tt2 b/disabled/invite_peerout_emergency/0008_test.yml.tt2 similarity index 100% rename from scenarios_disabled/invite_peerout_emergency/0008_test.yml.tt2 rename to disabled/invite_peerout_emergency/0008_test.yml.tt2 diff --git a/scenarios_disabled/invite_peerout_emergency/peer.yml b/disabled/invite_peerout_emergency/peer.yml similarity index 100% rename from scenarios_disabled/invite_peerout_emergency/peer.yml rename to disabled/invite_peerout_emergency/peer.yml diff --git a/scenarios_disabled/invite_peerout_emergency/prefs.json b/disabled/invite_peerout_emergency/prefs.json similarity index 100% rename from scenarios_disabled/invite_peerout_emergency/prefs.json rename to disabled/invite_peerout_emergency/prefs.json diff --git a/scenarios_disabled/invite_peerout_emergency/rewrite.yml b/disabled/invite_peerout_emergency/rewrite.yml similarity index 100% rename from scenarios_disabled/invite_peerout_emergency/rewrite.yml rename to disabled/invite_peerout_emergency/rewrite.yml diff --git a/scenarios_disabled/invite_peerout_emergency/scenario.yml b/disabled/invite_peerout_emergency/scenario.yml similarity index 100% rename from scenarios_disabled/invite_peerout_emergency/scenario.yml rename to disabled/invite_peerout_emergency/scenario.yml diff --git a/scenarios_disabled/invite_peerout_emergency/sipp_scenario00.xml b/disabled/invite_peerout_emergency/sipp_scenario00.xml similarity index 100% rename from scenarios_disabled/invite_peerout_emergency/sipp_scenario00.xml rename to disabled/invite_peerout_emergency/sipp_scenario00.xml diff --git a/scenarios_disabled/invite_peerout_emergency/sipp_scenario_responder00.xml b/disabled/invite_peerout_emergency/sipp_scenario_responder00.xml similarity index 100% rename from scenarios_disabled/invite_peerout_emergency/sipp_scenario_responder00.xml rename to disabled/invite_peerout_emergency/sipp_scenario_responder00.xml diff --git a/scenarios_disabled/invite_speeddial/0005_test.yml.tt2 b/disabled/invite_speeddial/0005_test.yml.tt2 similarity index 100% rename from scenarios_disabled/invite_speeddial/0005_test.yml.tt2 rename to disabled/invite_speeddial/0005_test.yml.tt2 diff --git a/scenarios_disabled/invite_speeddial/0007_test.yml.tt2 b/disabled/invite_speeddial/0007_test.yml.tt2 similarity index 100% rename from scenarios_disabled/invite_speeddial/0007_test.yml.tt2 rename to disabled/invite_speeddial/0007_test.yml.tt2 diff --git a/scenarios_disabled/invite_speeddial/0009_test.yml.tt2 b/disabled/invite_speeddial/0009_test.yml.tt2 similarity index 100% rename from scenarios_disabled/invite_speeddial/0009_test.yml.tt2 rename to disabled/invite_speeddial/0009_test.yml.tt2 diff --git a/scenarios_disabled/invite_speeddial/0010_test.yml.tt2 b/disabled/invite_speeddial/0010_test.yml.tt2 similarity index 100% rename from scenarios_disabled/invite_speeddial/0010_test.yml.tt2 rename to disabled/invite_speeddial/0010_test.yml.tt2 diff --git a/scenarios_disabled/invite_speeddial/0011_test.yml.tt2 b/disabled/invite_speeddial/0011_test.yml.tt2 similarity index 100% rename from scenarios_disabled/invite_speeddial/0011_test.yml.tt2 rename to disabled/invite_speeddial/0011_test.yml.tt2 diff --git a/scenarios_disabled/invite_speeddial/0012_test.yml.tt2 b/disabled/invite_speeddial/0012_test.yml.tt2 similarity index 100% rename from scenarios_disabled/invite_speeddial/0012_test.yml.tt2 rename to disabled/invite_speeddial/0012_test.yml.tt2 diff --git a/scenarios_disabled/invite_speeddial/0013_test.yml.tt2 b/disabled/invite_speeddial/0013_test.yml.tt2 similarity index 100% rename from scenarios_disabled/invite_speeddial/0013_test.yml.tt2 rename to disabled/invite_speeddial/0013_test.yml.tt2 diff --git a/scenarios_disabled/invite_speeddial/0014_test.yml.tt2 b/disabled/invite_speeddial/0014_test.yml.tt2 similarity index 100% rename from scenarios_disabled/invite_speeddial/0014_test.yml.tt2 rename to disabled/invite_speeddial/0014_test.yml.tt2 diff --git a/scenarios_disabled/invite_speeddial/0016_test.yml.tt2 b/disabled/invite_speeddial/0016_test.yml.tt2 similarity index 100% rename from scenarios_disabled/invite_speeddial/0016_test.yml.tt2 rename to disabled/invite_speeddial/0016_test.yml.tt2 diff --git a/scenarios_disabled/invite_speeddial/0018_test.yml.tt2 b/disabled/invite_speeddial/0018_test.yml.tt2 similarity index 100% rename from scenarios_disabled/invite_speeddial/0018_test.yml.tt2 rename to disabled/invite_speeddial/0018_test.yml.tt2 diff --git a/scenarios_disabled/invite_speeddial/0019_test.yml.tt2 b/disabled/invite_speeddial/0019_test.yml.tt2 similarity index 100% rename from scenarios_disabled/invite_speeddial/0019_test.yml.tt2 rename to disabled/invite_speeddial/0019_test.yml.tt2 diff --git a/scenarios_disabled/invite_speeddial/0020_test.yml.tt2 b/disabled/invite_speeddial/0020_test.yml.tt2 similarity index 100% rename from scenarios_disabled/invite_speeddial/0020_test.yml.tt2 rename to disabled/invite_speeddial/0020_test.yml.tt2 diff --git a/scenarios_disabled/invite_speeddial/0021_test.yml.tt2 b/disabled/invite_speeddial/0021_test.yml.tt2 similarity index 100% rename from scenarios_disabled/invite_speeddial/0021_test.yml.tt2 rename to disabled/invite_speeddial/0021_test.yml.tt2 diff --git a/scenarios_disabled/invite_speeddial/0022_test.yml.tt2 b/disabled/invite_speeddial/0022_test.yml.tt2 similarity index 100% rename from scenarios_disabled/invite_speeddial/0022_test.yml.tt2 rename to disabled/invite_speeddial/0022_test.yml.tt2 diff --git a/scenarios_disabled/invite_speeddial/0023_test.yml.tt2 b/disabled/invite_speeddial/0023_test.yml.tt2 similarity index 100% rename from scenarios_disabled/invite_speeddial/0023_test.yml.tt2 rename to disabled/invite_speeddial/0023_test.yml.tt2 diff --git a/scenarios_disabled/invite_speeddial/0025_test.yml.tt2 b/disabled/invite_speeddial/0025_test.yml.tt2 similarity index 100% rename from scenarios_disabled/invite_speeddial/0025_test.yml.tt2 rename to disabled/invite_speeddial/0025_test.yml.tt2 diff --git a/scenarios_disabled/invite_speeddial/0027_test.yml.tt2 b/disabled/invite_speeddial/0027_test.yml.tt2 similarity index 100% rename from scenarios_disabled/invite_speeddial/0027_test.yml.tt2 rename to disabled/invite_speeddial/0027_test.yml.tt2 diff --git a/scenarios_disabled/invite_speeddial/0029_test.yml.tt2 b/disabled/invite_speeddial/0029_test.yml.tt2 similarity index 100% rename from scenarios_disabled/invite_speeddial/0029_test.yml.tt2 rename to disabled/invite_speeddial/0029_test.yml.tt2 diff --git a/scenarios_disabled/invite_speeddial/scenario.yml b/disabled/invite_speeddial/scenario.yml similarity index 100% rename from scenarios_disabled/invite_speeddial/scenario.yml rename to disabled/invite_speeddial/scenario.yml diff --git a/scenarios_disabled/invite_speeddial/sipp_scenario00.xml b/disabled/invite_speeddial/sipp_scenario00.xml similarity index 100% rename from scenarios_disabled/invite_speeddial/sipp_scenario00.xml rename to disabled/invite_speeddial/sipp_scenario00.xml diff --git a/scenarios_disabled/invite_speeddial/sipp_scenario01.xml b/disabled/invite_speeddial/sipp_scenario01.xml similarity index 100% rename from scenarios_disabled/invite_speeddial/sipp_scenario01.xml rename to disabled/invite_speeddial/sipp_scenario01.xml diff --git a/scenarios_disabled/invite_speeddial/sipp_scenario02.xml b/disabled/invite_speeddial/sipp_scenario02.xml similarity index 100% rename from scenarios_disabled/invite_speeddial/sipp_scenario02.xml rename to disabled/invite_speeddial/sipp_scenario02.xml diff --git a/scenarios_disabled/invite_speeddial/sipp_scenario03.xml b/disabled/invite_speeddial/sipp_scenario03.xml similarity index 100% rename from scenarios_disabled/invite_speeddial/sipp_scenario03.xml rename to disabled/invite_speeddial/sipp_scenario03.xml diff --git a/scenarios_disabled/invite_speeddial/sipp_scenario_responder00.xml b/disabled/invite_speeddial/sipp_scenario_responder00.xml similarity index 100% rename from scenarios_disabled/invite_speeddial/sipp_scenario_responder00.xml rename to disabled/invite_speeddial/sipp_scenario_responder00.xml diff --git a/scenarios_disabled/invite_speeddial/sipp_scenario_responder01.xml b/disabled/invite_speeddial/sipp_scenario_responder01.xml similarity index 100% rename from scenarios_disabled/invite_speeddial/sipp_scenario_responder01.xml rename to disabled/invite_speeddial/sipp_scenario_responder01.xml diff --git a/scenarios_disabled/invite_speeddial/speeddial.yml b/disabled/invite_speeddial/speeddial.yml similarity index 100% rename from scenarios_disabled/invite_speeddial/speeddial.yml rename to disabled/invite_speeddial/speeddial.yml diff --git a/scenarios_disabled/presence/0003_test.yml.tt2 b/disabled/presence/0003_test.yml.tt2 similarity index 100% rename from scenarios_disabled/presence/0003_test.yml.tt2 rename to disabled/presence/0003_test.yml.tt2 diff --git a/scenarios_disabled/presence/0005_test.yml.tt2 b/disabled/presence/0005_test.yml.tt2 similarity index 100% rename from scenarios_disabled/presence/0005_test.yml.tt2 rename to disabled/presence/0005_test.yml.tt2 diff --git a/scenarios_disabled/presence/0007_test.yml.tt2 b/disabled/presence/0007_test.yml.tt2 similarity index 100% rename from scenarios_disabled/presence/0007_test.yml.tt2 rename to disabled/presence/0007_test.yml.tt2 diff --git a/scenarios_disabled/presence/0009_test.yml.tt2 b/disabled/presence/0009_test.yml.tt2 similarity index 100% rename from scenarios_disabled/presence/0009_test.yml.tt2 rename to disabled/presence/0009_test.yml.tt2 diff --git a/scenarios_disabled/presence/0011_test.yml.tt2 b/disabled/presence/0011_test.yml.tt2 similarity index 100% rename from scenarios_disabled/presence/0011_test.yml.tt2 rename to disabled/presence/0011_test.yml.tt2 diff --git a/scenarios_disabled/presence/0013_test.yml.tt2 b/disabled/presence/0013_test.yml.tt2 similarity index 100% rename from scenarios_disabled/presence/0013_test.yml.tt2 rename to disabled/presence/0013_test.yml.tt2 diff --git a/scenarios_disabled/presence/scenario.yml b/disabled/presence/scenario.yml similarity index 100% rename from scenarios_disabled/presence/scenario.yml rename to disabled/presence/scenario.yml diff --git a/scenarios_disabled/presence/sipp_scenario00.xml b/disabled/presence/sipp_scenario00.xml similarity index 100% rename from scenarios_disabled/presence/sipp_scenario00.xml rename to disabled/presence/sipp_scenario00.xml diff --git a/scenarios_disabled/presence/sipp_scenario_responder01.xml b/disabled/presence/sipp_scenario_responder01.xml similarity index 100% rename from scenarios_disabled/presence/sipp_scenario_responder01.xml rename to disabled/presence/sipp_scenario_responder01.xml diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 1e240c92..00000000 --- a/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -unittest-xml-reporting -pyaml diff --git a/scenarios/invite_peerout/0005_test.yml.tt2 b/scenarios/invite_peerout/0005_test.yml.tt2 index c760e7e8..365aa769 100644 --- a/scenarios/invite_peerout/0005_test.yml.tt2 +++ b/scenarios/invite_peerout/0005_test.yml.tt2 @@ -11,5 +11,5 @@ sip_out: '^SIP/2.0 180 Ringing', 'CSeq: 2 INVITE', 'From: >/root/.bash_history + +WORKDIR /code/ + +################################################################################ +# Instructions for usage +# ---------------------- +# When you want to build the base image from scratch (jump to the next section if you don't want to build yourself!): +# % docker build --tag="kamailio-config-tests-jessie" . +# % docker run --rm -i -t -v $(pwd)/..:/code:rw kamailio-config-tests-jessie:latest bash +# +# Use the existing docker image: +# % docker pull docker.mgm.sipwise.com/kamailio-config-tests-jessie +# % docker run --rm -i -t -v $(pwd)/:/code:rw docker.mgm.sipwise.com/kamailio-config-tests-jessie:latest bash +# +# Inside docker (the command is in history, just press UP button): +# ./t/testrunner +# +################################################################################ diff --git a/t/testrunner b/t/testrunner new file mode 100755 index 00000000..47a324b2 --- /dev/null +++ b/t/testrunner @@ -0,0 +1,19 @@ +#!/bin/bash +# This script is used for running the tests with proper arguments +# from within Jenkins + +set -e +set -u + +if [ -d /results ] ; then + # Running from Jenkins (RW) + RESULTS="/results" + + cd "/code" +else + # Running locally in Docker + RESULTS="./results" + mkdir -p "${RESULTS}" +fi + +RESULTS=${RESULTS} make -j test diff --git a/tests/do_test_yaml_format.sh b/tests/do_test_yaml_format.sh new file mode 100755 index 00000000..c18b3ed2 --- /dev/null +++ b/tests/do_test_yaml_format.sh @@ -0,0 +1,10 @@ +#!/bin/sh +export BASE_DIR=$(pwd) DEST_DIR=${RESULTS} + +./bin/generate_tests.sh -d "$1" tests/fixtures/scenario_ids.yml PRO +find "${RESULTS}/$1" -type f -name '*_test.yml' | while read -r i; do + ./tests/test_yaml_format.py "$i" | sed -e 's/skip>/skipped>/g' > "$i.xml" +done + +# exclude templates +rm -rf "${RESULTS}/$1/templates" diff --git a/tests/fixtures/scenario_ids.yml b/tests/fixtures/scenario_ids.yml new file mode 100644 index 00000000..2864fbb3 --- /dev/null +++ b/tests/fixtures/scenario_ids.yml @@ -0,0 +1,14 @@ +--- +customer_test: + id: 7 +peer_00_host0: + id: 31 +peer_00_host1: + id: 32 +spce_test: + testuser1001: + uuid: UUID1001 + testuser1002: + uuid: UUID1002 + testuser1003: + uuid: UUID1003 diff --git a/tests/test_check.py b/tests/test_check.py index 29870ffc..2cb0010a 100755 --- a/tests/test_check.py +++ b/tests/test_check.py @@ -20,8 +20,8 @@ # import os import sys +import junitxml import unittest -import xmlrunner import re lib_path = os.path.abspath('bin') sys.path.append(lib_path) @@ -197,8 +197,17 @@ class TestJson(unittest.TestCase): if __name__ == '__main__': - unittest.main( - testRunner=xmlrunner.XMLTestRunner(output=sys.stdout), - # these make sure that some options that are not applicable - # remain hidden from the help menu. - failfast=False, buffer=False, catchbreak=False) + + suite = unittest.TestSuite() + suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(TestXAvp)) + suite.addTest( + unittest.defaultTestLoader.loadTestsFromTestCase(TestCheckFlowVars)) + suite.addTest( + unittest.defaultTestLoader.loadTestsFromTestCase(TestCheckSipIn)) + suite.addTest( + unittest.defaultTestLoader.loadTestsFromTestCase(TestCheckSipOut)) + suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(TestJson)) + result = junitxml.JUnitXmlResult(sys.stdout) + result.startTestRun() + suite.run(result) + result.stopTestRun() diff --git a/tests/test_yaml_format.py b/tests/test_yaml_format.py new file mode 100755 index 00000000..135dac63 --- /dev/null +++ b/tests/test_yaml_format.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python +# +# Copyright: 2013-2015 Sipwise Development Team +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This package is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# On Debian systems, the complete text of the GNU General +# Public License version 3 can be found in "/usr/share/common-licenses/GPL-3". +# +from yaml import load +import junitxml +import os +import sys +import unittest +import fnmatch +try: + from yaml import CLoader as Loader +except ImportError: + from yaml import Loader + + +class ParametrizedTestCase(unittest.TestCase): + + """ TestCase classes that want to be parametrized should + inherit from this class. + http://eli.thegreenplace.net/ + 2011/08/02/python-unit-testing-parametrized-test-cases + """ + + def __init__(self, methodName='runTest', param=None): + super(ParametrizedTestCase, self).__init__(methodName) + self.param = param + self.scenario = os.path.dirname(self.param) + + def id(self): + return "%s_%s" % (super(ParametrizedTestCase, self).id(), + self.scenario) + + @staticmethod + def parametrize(testcase_klass, param=None): + """ Create a suite containing all tests taken from the given + subclass, passing them the parameter 'param'. + """ + testloader = unittest.TestLoader() + testnames = testloader.getTestCaseNames(testcase_klass) + suite = unittest.TestSuite() + for name in testnames: + suite.addTest(testcase_klass(name, param=param)) + return suite + + +class TestYmlLint(ParametrizedTestCase): + + def setUp(self): + self.yaml = load(file(self.param, 'r')) + + def testFlow(self): + self.assertTrue('flow' in self.yaml) + self.assertIsInstance(self.yaml['flow'], list) + + def testSipIn(self): + self.assertTrue('sip_in' in self.yaml) + self.assertIsInstance(self.yaml['sip_in'], list) + + def testSipOut(self): + self.assertTrue('sip_out' in self.yaml) + self.assertIsInstance(self.yaml['sip_out'], list) + + +if __name__ == '__main__': + assert len(sys.argv) == 2 + assert os.path.exists(sys.argv[1]) + suite = unittest.TestSuite() + suite.addTest( + ParametrizedTestCase.parametrize(TestYmlLint, + param=sys.argv[1])) + result = junitxml.JUnitXmlResult(sys.stdout) + result.startTestRun() + suite.run(result) + result.stopTestRun()