TT#15305 build: continue building if failure jobs are not in build_deps

Change-Id: Ibaf2360e388ad57551bd2691c310b8ad0e2f3896

build/fixtures/test_failed_builds.yaml

@@ -0,0 +1,13 @@
+-   model: build.buildrelease
+    pk: 1
+    fields:
+        uuid: 3f25fa3f-7ffd-464c-97e1-4b0ce1568ef5
+        start_date: 2020-03-09 11:18:02.265202+00:00
+        tag: mr8.1.2.1
+        branch: mr8.1.2
+        release: release-mr8.1.2
+        distribution: buster
+        projects: asterisk-voicemail,backup-tools,bulk-processor,bulk-processor-projects,bootenv,cdr-exporter,cfg-schema,check-tools,cleanup-tools,cloudpbx-sources,comx-fileshare-service,csta-testsuite,data-hal,db-schema,deployment-iso,dhtest,documentation,faxserver,heartbeat,installer,janus-admin,janus-client,kamailio,kamailio-config-tests,keyring,klish,libinewrate,libswrate,libtcap,license-client,lnpd,lua-ngcp-kamailio,mediator,megacli,metapackages,monitoring-tools,ngcp-api-tools,ngcp-csc-ui,ngcp-exporter,ngcp-fauditd,ngcp-inventory,ngcp-logfs,ngcp-klish-config,ngcp-panel,ngcp-prompts,ngcp-rtcengine,ngcp-schema,ngcp-status,ngcp-sudo-plugin,ngcp-support,ngcp-user-framework,ngcpcfg,ngcpcfg-api,ngcpcfg-ha,ngrep-sip,prosody,pushd,rate-o-mat,reminder,rtpengine,sems,sems-ha,sems-modules,sems-pbx,sipwise-base,snmp-agent,system-tests,system-tools,templates,upgrade,vmnotify,voisniff-ng,websocket
+        built_projects: release-copy-debs-yml,data-hal,libinewrate,libswrate,libtcap,sipwise-base,check-tools
+        failed_projects: templates
+        pool_size: 1

build/models.py

@@ -315,7 +315,7 @@ class BuildRelease(models.Model):
             return "tag/{}".format(self.tag)
         return "branch/{}".format(self.branch)
 
-    def _next(self):
+    def _next(self, exclude=[]):
         log = logger.bind(release=self)
         if self.built_projects is None:
             return self.build_deps[0][0]
@@ -339,6 +339,8 @@ class BuildRelease(models.Model):
                     )
                     return None
         for prj in self.projects_list:
+            if prj in exclude:
+                continue
             if prj not in built_list and prj not in t_list:
                 return prj
 
@@ -352,15 +354,10 @@ class BuildRelease(models.Model):
                 failed_projects=failed_projects,
             )
             return
-        res = self._next()
-        if res is not None:
-            if res in failed_projects:
-                log.error(
-                    "project marked as failed, stop sending jobs",
-                    project=res,
-                )
-            else:
-                return res
+        prj = self._next(exclude=failed_projects)
+        if prj in failed_projects and self.config.is_build_dep(prj):
+            return None
+        return prj
 
     @property
     def build_deps(self) -> list:

build/test/test_release.py

@@ -19,35 +19,80 @@ from build.models import BuildRelease
 from repoapi.test.base import BaseTest
 
 
-class BuildReleaseStepsTest(BaseTest):
-    fixtures = [
-        "test_change_build_deps",
-    ]
-
+class StepsTest(BaseTest):
     def setUp(self):
+        super().setUp()
         self.br = BuildRelease.objects.get(pk=1)
         self.built_projects = deepcopy(self.br.built_projects)
+        self.failed_projects = deepcopy(self.br.failed_projects)
         self.jbi = MagicMock()
         self.jbi.result = "SUCCESS"
 
-    def test_stopped(self):
-        self.test_append_built()
-        self.assertIsNone(self.br.next)
-
-    def test_append_built(self):
-        self.jbi.projectname = "templates"
-        self.jbi.jobname = "templates-repos"
+    def append_built(self, projectname):
+        self.jbi.projectname = projectname
+        self.jbi.jobname = f"{projectname}-repos"
         self.assertTrue(self.br.append_built(self.jbi))
-        self.built_projects += ",templates"
+        self.built_projects += f",{projectname}"
         self.assertEqual(self.br.built_projects, self.built_projects)
         self.assertEqual(self.br.pool_size, 0)
 
+    def append_built_failed(self, projectname):
+        self.jbi.projectname = projectname
+        self.jbi.jobname = f"{projectname}-repos"
+        self.jbi.result = "FAILURE"
+        self.assertTrue(self.br.append_built(self.jbi))
+        if self.failed_projects:
+            self.failed_projects += f",{projectname}"
+        else:
+            self.failed_projects = f"{projectname}"
+        self.assertEqual(self.br.failed_projects, self.failed_projects)
+
+
+class BuildReleaseStepsTest(StepsTest):
+    """template was not build_dep and bootenv failed"""
+
+    fixtures = [
+        "test_change_build_deps",
+    ]
+
+    def test_status(self):
+        self.assertTrue(self.br.config.is_build_dep("templates"))
+        self.assertEqual(self.br.failed_projects, "bootenv")
+
     def test_next_build_deps_changed(self):
         self.assertEqual(self.br.next, "templates")
 
-    def test_next_retrigger(self):
-        self.test_append_built()
-        self.jbi.projectname = "bootenv"
-        self.jbi.jobname = "{}-repos".format(self.jbi.projectname)
-        self.assertTrue(self.br.append_built(self.jbi))
-        self.assertIsNotNone(self.br.next)
+    def test_next_build_deps_fixed(self):
+        self.append_built("templates")
+        self.assertEqual(self.br.next, "cdr-exporter")
+
+
+class BuildReleaseFailureStepsTest(StepsTest):
+    """failed build dep... no more builds"""
+
+    fixtures = [
+        "test_failed_builds",
+    ]
+
+    def test_status(self):
+        self.assertTrue(self.br.config.is_build_dep("templates"))
+        self.assertTrue(self.br.config.is_build_dep("ngcp-schema"))
+        self.assertEqual(self.br.failed_projects, "templates")
+
+    def test_stopped(self):
+        self.assertIsNone(self.br.next)
+
+    def test_next_fixed(self):
+        self.append_built("templates")
+        self.assertEqual(self.br.failed_projects, None)
+        self.failed_projects = None
+        self.assertEqual(self.br.next, "ngcp-schema")
+
+    def test_failed_non_build_dep(self):
+        self.test_next_fixed()
+        self.append_built("ngcp-schema")
+        self.assertEqual(self.br.next, "asterisk-voicemail")
+        self.assertFalse(self.br.config.is_build_dep("asterisk-voicemail"))
+        self.append_built_failed("asterisk-voicemail")
+        # next will continue
+        self.assertEqual(self.br.next, "backup-tools")

build/test/test_tasks.py

@@ -346,6 +346,15 @@ class WeeklyTest(BaseTest):
             buildnumber=1,
             result="FAILURE",
         )
+        params = {
+            "project": "ngcp-prompts-get-code",
+            "release_uuid": br.uuid,
+            "trigger_release": br.release,
+            "trigger_branch_or_tag": br.branch_or_tag,
+            "trigger_distribution": br.distribution,
+        }
+        tb.assert_called_once_with(**params)
+        tb.reset_mock()
         JenkinsBuildInfo.objects.create(
             job_url="http://fake.local/job/ngcp-prompts-repos/",
             projectname="ngcp-prompts",

build/test/test_utils.py

@@ -205,6 +205,11 @@ class ReleaseConfigTestCase(SimpleTestCase):
         rc = ReleaseConfig("trunk")
         self.assertListEqual(list(rc.build_deps.keys()), self.build_deps)
 
+    def test_is_build_dep(self):
+        rc = ReleaseConfig("trunk")
+        self.assertTrue(rc.is_build_dep("data-hal"))
+        self.assertFalse(rc.is_build_dep("templates"))
+
     def test_build_deps_iter_step_1(self):
         rc = ReleaseConfig("trunk")
         build_deps = [

build/utils.py

@@ -309,6 +309,9 @@ class ReleaseConfig(object):
             raise err.NoDistrisInfo(msg.format(self.config_file))
         self.check_circular_dependencies()
 
+    def is_build_dep(self, prj: str) -> bool:
+        return prj in self.build_deps.keys()
+
     @property
     def build_deps(self) -> dict:
         return self.jenkins_jobs.get("build_deps", dict())