TT#15305 build: improve refresh_projects

When refreshing list of projects, remove every info related to
any old project

We had this scenario when releasing mr10.5, ngcp-cve-scanner was
wrongly added as a project and we couldn't resume the build until
I removed the related info

Change-Id: I58ee97f3256bb63a0a07a6ed606ede0c233cfb84

build/fixtures/config/trunk-weekly.yml

@@ -156,6 +156,7 @@ jenkins-jobs:
     - ngcp-admin-ui
     - ngcp-api-tools
     - ngcp-csc-ui
+    - ngcp-cve-scanner
     - ngcp-exporter
     - ngcp-fauditd
     - ngcp-inventory
@@ -210,6 +211,7 @@ release_mirror:
       - ngcp-check-tools
       - ngcp-cleanup-tools
       - ngcp-csc-ui
+      - ngcp-cve-scanner
       - ngcp-db-schema
       - ngcp-deployment-iso
       - ngcp-dhtest

build/models.py

@@ -26,6 +26,7 @@ from .exceptions import BuildReleaseUnique
 from .exceptions import PreviousBuildNotDone
 from .utils import get_simple_release
 from .utils import ReleaseConfig
+from .utils import remove_from_textlist
 from repoapi.models import JenkinsBuildInfo
 
 logger = structlog.get_logger(__name__)
@@ -157,8 +158,21 @@ class BuildRelease(models.Model):
         return "%s[%s]" % (self.release, self.uuid)
 
     def refresh_projects(self):
+        old_projects = set(self.projects_list)
         self.projects = ",".join(self.config.projects)
+        new_list = set(self.projects_list)
+        append_list = [item for item in new_list if item not in old_projects]
+        removed_list = [item for item in old_projects if item not in new_list]
+        for project in removed_list:
+            remove_from_textlist(self, "built_projects", project)
+            remove_from_textlist(self, "triggered_projects", project)
+            remove_from_textlist(self, "failed_projects", project)
+        JenkinsBuildInfo.objects.filter(
+            projectname__in=removed_list,
+            param_release_uuid=self.uuid,
+        ).delete()
         self.save()
+        return (append_list, removed_list)
 
     def resume(self):
         if not self.done:
@@ -274,16 +288,7 @@ class BuildRelease(models.Model):
         return True
 
     def remove_triggered(self, jbi):
-        value = jbi.projectname
-        triggered_list = self.triggered_projects_list
-        if value in triggered_list:
-            triggered_list.remove(value)
-            tl = ",".join(triggered_list)
-            if len(tl) > 0:
-                self.triggered_projects = tl
-            else:
-                self.triggered_projects = None
-            self.save()
+        remove_from_textlist(self, "triggered_projects", jbi.projectname)
 
     def append_built(self, jbi):
         jobname = jbi.jobname

build/test/test_models.py

@@ -556,3 +556,70 @@ class BuildReleaseRetriggerTest(BaseTest):
         self.assertTrue(self.br.append_built(self.jbi))
         self.assertEqual(self.br.built_projects, "data-hal,libinewrate")
         self.assertIsNone(self.br.failed_projects)
+
+
+@override_settings(JBI_ALLOWED_HOSTS=["fake.local"])
+class RefreshProjects(BaseTest):
+    fixtures = [
+        "test_weekly",
+    ]
+    release = "release-trunk-weekly"
+    release_uuid = "dbe569f7-eab6-4532-a6d1-d31fb559649b"
+
+    def setUp(self):
+        self.br = BuildRelease.objects.get(uuid=self.release_uuid)
+
+    def test_refresh_append(self):
+        self.assertNotIn("ngcp-cve-scanner", self.br.projects_list)
+        self.assertIn("ngcp-cve-scanner", self.br.config.projects)
+        _append, _removed = self.br.refresh_projects()
+        self.assertEqual(_append, ["ngcp-cve-scanner"])
+        self.assertEqual(_removed, [])
+        self.assertIn("ngcp-cve-scanner", self.br.projects_list)
+
+    @patch("repoapi.utils.dlfile")
+    @patch("build.signals.build_resume")
+    def test_refresh_remove(self, build_resume, dl):
+        self.br.projects += ",fake-project"
+        self.assertIn("fake-project", self.br.projects_list)
+        self.br.save()
+        JenkinsBuildInfo.objects.create(
+            job_url="http://fake.local/job/fake-project-repos/",
+            projectname="fake-project",
+            jobname="fake-project-repos",
+            param_tag="UUIDA",
+            param_release=self.release,
+            param_release_uuid=self.release_uuid,
+            buildnumber=1,
+            result="SUCCESS",
+        )
+        JenkinsBuildInfo.objects.create(
+            job_url="http://fake.local/job/data-hal-source/",
+            projectname="data-hal",
+            jobname="data-hal-source",
+            param_tag="UUIDA",
+            param_release=self.release,
+            param_release_uuid=self.release_uuid,
+            buildnumber=1,
+            result="FAILURE",
+        )
+        self.br = BuildRelease.objects.get(uuid=self.release_uuid)
+        self.assertIn("fake-project", self.br.built_projects_list)
+        self.assertIn("data-hal", self.br.failed_projects_list)
+        _append, _removed = self.br.refresh_projects()
+        self.assertNotIn("fake-project", self.br.projects_list)
+        self.assertEqual(_append, ["ngcp-cve-scanner"])
+        self.assertEqual(
+            _removed,
+            [
+                "fake-project",
+            ],
+        )
+        self.assertNotIn("fake-project", self.br.projects_list)
+        self.assertNotIn("fake-project", self.br.built_projects_list)
+        self.assertNotIn("fake-project", self.br.triggered_projects_list)
+        self.assertNotIn("fake-project", self.br.failed_projects_list)
+        res = JenkinsBuildInfo.objects.filter(
+            param_release_uuid=self.release_uuid
+        ).count()
+        self.assertEqual(1, res)

build/test/test_rest.py

@@ -74,7 +74,7 @@ class TestRest(APIAuthenticatedTestCase):
         self.assertEqual(response.data["branch"], "master")
         self.assertEqual(response.data["distribution"], "bullseye")
         projects = response.data["projects"].split(",")
-        self.assertEqual(len(projects), 71)
+        self.assertEqual(len(projects), 72)
 
     def test_mrXX(self):
         data = {

build/test/test_utils.py

@@ -25,6 +25,7 @@ from build.utils import get_common_release
 from build.utils import get_simple_release
 from build.utils import is_release_trunk
 from build.utils import ReleaseConfig
+from build.utils import remove_from_textlist
 from build.utils import trigger_build
 from build.utils import trigger_build_matrix
 from build.utils import trigger_copy_deps
@@ -381,3 +382,21 @@ class TriggerBuildMatrix(BaseTest):
         res = trigger_build_matrix(br)
         self.assertIsNone(res)
         openurl.assert_not_called()
+
+
+@override_settings(DEBUG=False)
+class RemoveList(BaseTest):
+    fixtures = [
+        "test_weekly",
+    ]
+    release = "release-trunk-weekly"
+    release_uuid = "dbe569f7-eab6-4532-a6d1-d31fb559649b"
+
+    def setUp(self):
+        BuildRelease = apps.get_model("build", "BuildRelease")
+        self.br = BuildRelease.objects.get(uuid=self.release_uuid)
+
+    def test_remove_from_textlist(self):
+        self.br.triggered_projects = "fake-project"
+        remove_from_textlist(self.br, "triggered_projects", "fake-project")
+        self.assertIsNone(self.br.triggered_projects)

build/utils.py

@@ -41,6 +41,18 @@ re_release_common = re.compile(r"^(release-)?(mr[0-9]+\.[0-9]+)(\.[0-9]+)?$")
 re_release_trunk = re.compile(r"^release-trunk-(\w+)$")
 
 
+def remove_from_textlist(br, orig, value):
+    _list = getattr(br, f"{orig}_list")
+    if value in _list:
+        _list.remove(value)
+        tl = ",".join(_list)
+        if len(tl) > 0:
+            setattr(br, orig, tl)
+        else:
+            setattr(br, orig, None)
+        br.save()
+
+
 def is_release_trunk(version):
     match = re_release_trunk.search(version)
     if match: