TT#5412 test: add yaml format test for XXXX_test.yml.tt2 files

* migrate to Docker
* tests/test_check.py: use junitxml
* bin/generate_tests.sh: support DEST_DIR
  will use DEST_DIR as base dir destination
* rename scenarios_disabled -> disabled
* fix \t at XXXX_test.yml.tt2 templates

Change-Id: I845588ee4692f157f370a000abab75a05e701839

Makefile

@@ -1,27 +1,23 @@
+SCENARIOS:=$(shell find -maxdepth 1 -type d -name 'scenarios*'|sed 's_\./__g')
+TESTS=$(addprefix test_,$(SCENARIOS))
+
+RESULTS ?= reports
+
 # do nothing as default
 all:
 
-.ONESHELL:
+# python-junitxml 0.6 has this bug
-SHELL = /bin/bash
+# https://bugs.launchpad.net/pyjunitxml/+bug/892293
-venv: requirements.txt
+$(TESTS):
-	virtualenv --python=python2.7 venv
+	@mkdir -p $(RESULTS)
-	source ./venv/bin/activate && \
+	$(eval SCEN_DIR := $(@:test_%=%))
-		pip install -r ./requirements.txt >install.log
+	./tests/do_test_yaml_format.sh $(SCEN_DIR)
 
-test_check: venv tests/test_check.py
+test_check: tests/test_check.py
-	mkdir -p reports
+	mkdir -p $(RESULTS)
-	source ./venv/bin/activate && \
+	./tests/test_check.py > $(RESULTS)/$(@).xml
-		./tests/test_check.py > reports/$(@).xml
 
 # run this in parallel!! -j is your friend
-test: test_check
+test: $(TESTS) test_check
-
-# get rid of test files
-clean:
-	rm -rf install.log
-
-# also get rid of pip environment
-dist-clean: clean
-	rm -rf venv
 
-.PHONY: all
+.PHONY: all $(TESTS)

bin/generate_tests.sh

@@ -21,6 +21,8 @@
 BASE_DIR="${BASE_DIR:-/usr/share/kamailio-config-tests}"
 DIR="${BASE_DIR}/scenarios"
 BIN_DIR="${BASE_DIR}/bin"
+DEST_DIR="${DEST_DIR}"
+
 error_flag=0
 
 function clean
@@ -72,18 +74,24 @@ if [ ! -x "${BIN_DIR}/generate_test.pl" ]; then
 fi
 
 for t in $(find "${DIR}" -not -regex '.+customtt.tt2' -type f -name '*.tt2' | sort); do
+  origdir="$(dirname "$t")"
   template="$(basename "$t")"
-  destdir="$(dirname "$t")"
+  if [ -n "$DEST_DIR" ] ; then
+    destdir="$DEST_DIR/$(dirname "$t")"
+    mkdir -p "${destdir}"
+  else
+    destdir="$(dirname "$t")"
+  fi
   destfile="$(basename "$t" .tt2)"
   custom_template="$(basename "$t" .tt2).customtt.tt2"
 
-  if [ -f "${destdir}/${custom_template}" ]; then
+  if [ -f "${origdir}/${custom_template}" ]; then
     echo "Custom detected"
     template=${custom_template}
   fi
   echo "generating: ${destdir}/${destfile}"
   # shellcheck disable=SC2086
-  if ! "${BIN_DIR}/generate_test.pl" ${ARGS} "${destdir}/${template}" ${IDS} > "${destdir}/${destfile}" ; then
+  if ! "${BIN_DIR}/generate_test.pl" ${ARGS} "${origdir}/${template}" ${IDS} > "${destdir}/${destfile}" ; then
     error_flag=1
   fi
 done

requirements.txt

@@ -1,2 +0,0 @@
-unittest-xml-reporting
-pyaml

scenarios/invite_peerout/0005_test.yml.tt2

@@ -11,5 +11,5 @@ sip_out:
       '^SIP/2.0 180 Ringing',
       'CSeq: 2 INVITE',
       'From: <sip:testuser1002@',
-  	  'To: <sip:004321001@',
+      'To: <sip:004321001@',
     ]

scenarios/invite_peerout_rewrite/0005_test.yml.tt2

@@ -11,5 +11,5 @@ sip_out:
       '^SIP/2.0 180 Ringing',
       'CSeq: 2 INVITE',
       'From: <sip:testuser1002@',
-  	  'To: <sip:004321001@',
+      'To: <sip:004321001@',
     ]

scenarios/invite_peerout_rewrite_pv_regex/0005_test.yml.tt2

@@ -11,5 +11,5 @@ sip_out:
       '^SIP/2.0 180 Ringing',
       'CSeq: 2 INVITE',
       'From: <sip:testuser1002@',
-  	  'To: <sip:004321001@',
+      'To: <sip:004321001@',
     ]

scenarios/invite_redirect/0011_test.yml.tt2

@@ -11,5 +11,5 @@ sip_out:
       '^SIP/2.0 180 Ringing',
       'CSeq: 2 INVITE',
       'From: <sip:testuser1002@',
-  	  'To: <sip:testuser1003@',
+      'To: <sip:testuser1003@',
     ]

scenarios_lnp/incoming_peer/0005_test.yml.tt2

@@ -11,5 +11,5 @@ sip_out:
       '^SIP/2.0 180 Ringing',
       'CSeq: 10 INVITE',
       'From: <sip:004321001@',
-  	  'To: <sip:4311001@',
+      'To: <sip:4311001@',
     ]

scenarios_lnp/invite/0005_test.yml.tt2

@@ -11,5 +11,5 @@ sip_out:
       '^SIP/2.0 180 Ringing',
       'CSeq: 2 INVITE',
       'From: <sip:testuser1002@',
-  	  'To: <sip:4311001@',
+      'To: <sip:4311001@',
     ]

scenarios_lnp/invite_ncos_black/0013_test.yml.tt2

@@ -11,5 +11,5 @@ sip_out:
       '^SIP/2.0 180 Ringing',
       'CSeq: 2 INVITE',
       'From: <sip:testuser1002@',
-  	  'To: <sip:43999300@',
+      'To: <sip:43999300@',
     ]

scenarios_lnp/invite_ncos_white/0005_test.yml.tt2

@@ -11,5 +11,5 @@ sip_out:
       '^SIP/2.0 180 Ringing',
       'CSeq: 2 INVITE',
       'From: <sip:testuser1002@',
-  	  'To: <sip:43999100@',
+      'To: <sip:43999100@',
     ]

scenarios_lnp/invite_ncos_white/0014_test.yml.tt2

@@ -11,5 +11,5 @@ sip_out:
       '^SIP/2.0 180 Ringing',
       'CSeq: 2 INVITE',
       'From: <sip:testuser1002@',
-  	  'To: <sip:43999200@',
+      'To: <sip:43999200@',
     ]

scenarios_lnp/invite_peerout/0005_test.yml.tt2

@@ -11,5 +11,5 @@ sip_out:
       '^SIP/2.0 180 Ringing',
       'CSeq: 2 INVITE',
       'From: <sip:testuser1002@',
-  	  'To: <sip:004321001@',
+      'To: <sip:004321001@',
     ]

scenarios_lnp/invite_peerout_rn/0005_test.yml.tt2

@@ -11,5 +11,5 @@ sip_out:
       '^SIP/2.0 180 Ringing',
       'CSeq: 2 INVITE',
       'From: <sip:testuser1002@',
-  	  'To: <sip:004321001@',
+      'To: <sip:004321001@',
     ]

t/Dockerfile

@@ -0,0 +1,32 @@
+# DOCKER_NAME=kamailio-config-tests-jessie
+FROM docker.mgm.sipwise.com/sipwise-jessie:latest
+
+# Important! Update this no-op ENV variable when this Dockerfile
+# is updated with the current date. It will force refresh of all
+# of the base images and things like `apt-get update` won't be using
+# old cached versions when the Dockerfile is built.
+ENV REFRESHED_AT 2016-11-04
+
+RUN apt-get update
+RUN apt-get install --assume-yes python-yaml python-junitxml \
+	make libtemplate-perl libyaml-perl
+
+RUN echo './t/testrunner' >>/root/.bash_history
+
+WORKDIR /code/
+
+################################################################################
+# Instructions for usage
+# ----------------------
+# When you want to build the base image from scratch (jump to the next section if you don't want to build yourself!):
+# % docker build --tag="kamailio-config-tests-jessie" .
+# % docker run --rm -i -t -v $(pwd)/..:/code:rw kamailio-config-tests-jessie:latest bash
+#
+# Use the existing docker image:
+# % docker pull docker.mgm.sipwise.com/kamailio-config-tests-jessie
+# % docker run --rm -i -t -v $(pwd)/:/code:rw docker.mgm.sipwise.com/kamailio-config-tests-jessie:latest bash
+#
+# Inside docker (the command is in history, just press UP button):
+#   ./t/testrunner
+#
+################################################################################

t/testrunner

@@ -0,0 +1,19 @@
+#!/bin/bash
+# This script is used for running the tests with proper arguments
+# from within Jenkins
+
+set -e
+set -u
+
+if [ -d /results ] ; then
+  # Running from Jenkins (RW)
+  RESULTS="/results"
+
+  cd "/code"
+else
+  # Running locally in Docker
+  RESULTS="./results"
+  mkdir -p "${RESULTS}"
+fi
+
+RESULTS=${RESULTS} make -j test

tests/do_test_yaml_format.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+export BASE_DIR=$(pwd) DEST_DIR=${RESULTS}
+
+./bin/generate_tests.sh -d "$1" tests/fixtures/scenario_ids.yml PRO
+find "${RESULTS}/$1" -type f -name '*_test.yml' | while read -r i; do
+  ./tests/test_yaml_format.py "$i" | sed -e 's/skip>/skipped>/g' > "$i.xml"
+done
+
+# exclude templates
+rm -rf "${RESULTS}/$1/templates"

tests/fixtures/scenario_ids.yml

@@ -0,0 +1,14 @@
+---
+customer_test:
+  id: 7
+peer_00_host0:
+  id: 31
+peer_00_host1:
+  id: 32
+spce_test:
+  testuser1001:
+    uuid: UUID1001
+    testuser1002:
+    uuid: UUID1002
+  testuser1003:
+    uuid: UUID1003

tests/test_check.py

@@ -20,8 +20,8 @@
 #
 import os
 import sys
+import junitxml
 import unittest
-import xmlrunner
 import re
 lib_path = os.path.abspath('bin')
 sys.path.append(lib_path)
@@ -197,8 +197,17 @@ class TestJson(unittest.TestCase):
 
 
 if __name__ == '__main__':
-    unittest.main(
+
-        testRunner=xmlrunner.XMLTestRunner(output=sys.stdout),
+    suite = unittest.TestSuite()
-        # these make sure that some options that are not applicable
+    suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(TestXAvp))
-        # remain hidden from the help menu.
+    suite.addTest(
-        failfast=False, buffer=False, catchbreak=False)
+        unittest.defaultTestLoader.loadTestsFromTestCase(TestCheckFlowVars))
+    suite.addTest(
+        unittest.defaultTestLoader.loadTestsFromTestCase(TestCheckSipIn))
+    suite.addTest(
+        unittest.defaultTestLoader.loadTestsFromTestCase(TestCheckSipOut))
+    suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(TestJson))
+    result = junitxml.JUnitXmlResult(sys.stdout)
+    result.startTestRun()
+    suite.run(result)
+    result.stopTestRun()

tests/test_yaml_format.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python
+#
+# Copyright: 2013-2015 Sipwise Development Team <support@sipwise.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This package is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+# On Debian systems, the complete text of the GNU General
+# Public License version 3 can be found in "/usr/share/common-licenses/GPL-3".
+#
+from yaml import load
+import junitxml
+import os
+import sys
+import unittest
+import fnmatch
+try:
+    from yaml import CLoader as Loader
+except ImportError:
+    from yaml import Loader
+
+
+class ParametrizedTestCase(unittest.TestCase):
+
+    """ TestCase classes that want to be parametrized should
+        inherit from this class.
+        http://eli.thegreenplace.net/
+        2011/08/02/python-unit-testing-parametrized-test-cases
+    """
+
+    def __init__(self, methodName='runTest', param=None):
+        super(ParametrizedTestCase, self).__init__(methodName)
+        self.param = param
+        self.scenario = os.path.dirname(self.param)
+
+    def id(self):
+        return "%s_%s" % (super(ParametrizedTestCase, self).id(),
+                          self.scenario)
+
+    @staticmethod
+    def parametrize(testcase_klass, param=None):
+        """ Create a suite containing all tests taken from the given
+            subclass, passing them the parameter 'param'.
+        """
+        testloader = unittest.TestLoader()
+        testnames = testloader.getTestCaseNames(testcase_klass)
+        suite = unittest.TestSuite()
+        for name in testnames:
+            suite.addTest(testcase_klass(name, param=param))
+        return suite
+
+
+class TestYmlLint(ParametrizedTestCase):
+
+    def setUp(self):
+        self.yaml = load(file(self.param, 'r'))
+
+    def testFlow(self):
+        self.assertTrue('flow' in self.yaml)
+        self.assertIsInstance(self.yaml['flow'], list)
+
+    def testSipIn(self):
+        self.assertTrue('sip_in' in self.yaml)
+        self.assertIsInstance(self.yaml['sip_in'], list)
+
+    def testSipOut(self):
+        self.assertTrue('sip_out' in self.yaml)
+        self.assertIsInstance(self.yaml['sip_out'], list)
+
+
+if __name__ == '__main__':
+    assert len(sys.argv) == 2
+    assert os.path.exists(sys.argv[1])
+    suite = unittest.TestSuite()
+    suite.addTest(
+        ParametrizedTestCase.parametrize(TestYmlLint,
+                                         param=sys.argv[1]))
+    result = junitxml.JUnitXmlResult(sys.stdout)
+    result.startTestRun()
+    suite.run(result)
+    result.stopTestRun()