MT#59039 jbi: move related jbi_files to JBI_ARCHIVE on delete

We are storing some build artifact files indefinitely.
Implement removal of stored build artifacts on JenkinsBuildInfo
record deletion. So when a record in the database gets removed
the related files of that build will get relocated from JBI_FILES
to JBI_ARCHIVE

Change-Id: Ia1fcba2948bb3d0b50209fe29c1daa3f293cf59e

repoapi/models/jbi.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2022 The Sipwise Team - http://sipwise.com
+# Copyright (C) 2015-2024 The Sipwise Team - http://sipwise.com
 #
 # This program is free software: you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by the Free
@@ -18,6 +18,7 @@ import re
 from collections import OrderedDict
 from datetime import datetime
 from datetime import timedelta
+from pathlib import Path
 from urllib.parse import urlparse
 
 import structlog
@@ -254,7 +255,7 @@ class JenkinsBuildInfo(models.Model):
         return self.param_ppa not in ["$ppa", None]
 
     @property
-    def build_path(self):
+    def build_path(self) -> Path:
         return settings.JBI_BASEDIR.joinpath(
             self.jobname, str(self.buildnumber)
         )

repoapi/settings/dev.py

@@ -32,6 +32,7 @@ BROKER_BACKEND = "amqp"
 CELERY_TASK_ALWAYS_EAGER = False
 CELERY_BROKER_URL = "amqp://guest:guest@rabbit"
 JBI_BASEDIR = BASE_DIR / "jbi_files"  # noqa
+JBI_ARCHIVE = BASE_DIR / "jbi_archive"  # noqa
 
 # Enable access when not accessing from localhost:
 ALLOWED_HOSTS = [

repoapi/settings/prod.py

@@ -171,6 +171,7 @@ CELERY_TIMEZONE = "UTC"
 FLOWER_URL_PREFIX = "flower"
 
 JBI_BASEDIR = VAR_DIR / "jbi_files"
+JBI_ARCHIVE = Path("/srv/repoapi_archive")
 JBI_ARTIFACT_JOBS = [
     "release-tools-runner",
 ]

repoapi/signals.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2022-2023 The Sipwise Team - http://sipwise.com
+# Copyright (C) 2022-2024 The Sipwise Team - http://sipwise.com
 #
 # This program is free software: you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by the Free
@@ -15,10 +15,12 @@
 import structlog
 from django.apps import apps
 from django.db.models.signals import post_save
+from django.db.models.signals import pre_delete
 from django.dispatch import receiver
 
 from .models.wni import NoteInfo
 from .tasks import get_jbi_files
+from .tasks import jbi_files_cleanup
 from .utils import get_next_release
 from .utils import regex_mrXXX
 from release_dashboard.utils.build import is_ngcp_project
@@ -38,6 +40,13 @@ def jbi_manage(sender, **kwargs):
             )
 
 
+@receiver(
+    pre_delete, sender="repoapi.JenkinsBuildInfo", dispatch_uid="jbi_cleanup"
+)
+def jbi_cleanup(sender, **kwargs):
+    jbi_files_cleanup.delay(kwargs["instance"].pk)
+
+
 def gerrit_repo_add(instance):
     if not instance.has_ppa:
         logger.warn("ppa unset, skip removal")

repoapi/tasks.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2016-2022 The Sipwise Team - http://sipwise.com
+# Copyright (C) 2016-2024 The Sipwise Team - http://sipwise.com
 #
 # This program is free software: you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by the Free
@@ -23,6 +23,7 @@ from .celery import jbi_parse_buildinfo
 from .celery import jbi_parse_hotfix
 from .celery import process_result
 from .conf import settings
+from .utils import cleanup_build
 from .utils import is_download_artifacts
 from .utils import jenkins_get_artifact
 from .utils import jenkins_get_build
@@ -91,3 +92,12 @@ def jbi_purge(release, weeks):
     JenkinsBuildInfo = apps.get_model("repoapi", "JenkinsBuildInfo")
     JenkinsBuildInfo.objects.purge_release(release, timedelta(weeks=weeks))
     logger.info(f"purged release {release} jbi older than {weeks} weeks")
+
+
+@shared_task(ignore_result=True)
+def jbi_files_cleanup(jbi_id):
+    JenkinsBuildInfo = apps.get_model("repoapi", "JenkinsBuildInfo")
+    jbi = JenkinsBuildInfo.objects.get(id=jbi_id)
+    build_path = jbi.build_path
+    dst_path = settings.JBI_ARCHIVE / jbi.jobname
+    cleanup_build(build_path, dst_path)

repoapi/test/base.py

@@ -24,29 +24,34 @@ from rest_framework.test import APITestCase
 from rest_framework_api_key.models import APIKey
 
 JBI_BASEDIR = Path(mkdtemp(dir=os.environ.get("RESULTS")))
+JBI_ARCHIVE = Path(mkdtemp(dir=os.environ.get("RESULTS")))
 
 
-@override_settings(DEBUG=True, JBI_BASEDIR=JBI_BASEDIR)
+@override_settings(
+    DEBUG=True, JBI_BASEDIR=JBI_BASEDIR, JBI_ARCHIVE=JBI_ARCHIVE
+)
 class BaseTest(TestCase):
     @classmethod
     def setUpTestData(cls):
         from repoapi.conf import settings
 
         cls.path = Path(settings.JBI_BASEDIR)
+        cls.archive_path = Path(settings.JBI_ARCHIVE)
 
     def setUp(self):
         RepoAPIConfig = apps.get_app_config("repoapi")
         RepoAPIConfig.ready()
         super(BaseTest, self).setUp()
-        self.path.mkdir(parents=True, exist_ok=True)
+        for path in [self.path, self.archive_path]:
+            path.mkdir(parents=True, exist_ok=True)
 
     def tearDown(self):
-        if self.path.exists():
+        for path in [self.path, self.archive_path]:
-            shutil.rmtree(self.path)
+            if path.exists():
+                shutil.rmtree(path)
 
 
 class APIAuthenticatedTestCase(BaseTest, APITestCase):
-
     APP_NAME = "Project Tests"
 
     def setUp(self):

repoapi/test/test_jbi_info.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2022 The Sipwise Team - http://sipwise.com
+# Copyright (C) 2015-2024 The Sipwise Team - http://sipwise.com
 #
 # This program is free software: you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by the Free
@@ -137,6 +137,29 @@ class TestJBICelery(BaseTest):
         dlfile.assert_any_call(url, path)
 
 
+class TestJBICleanUP(BaseTest):
+    fixtures = ["test_model_queries"]
+
+    def setUp(self):
+        super().setUp()
+        self.jbi = JenkinsBuildInfo.objects.get(id=1)
+        self.jbi.build_path.mkdir(parents=True, exist_ok=True)
+
+    @patch("repoapi.utils.shutil")
+    def test_jbi_cleanup(self, sh):
+        build_path = self.jbi.build_path
+        dst_path = settings.JBI_ARCHIVE / self.jbi.jobname
+        self.assertTrue(build_path.exists())
+        self.jbi.delete()
+        sh.move.assert_called_with(build_path, dst_path)
+
+    @patch("repoapi.signals.jbi_files_cleanup")
+    def test_jbi_cleanup_called(self, jfc):
+        jbi_id = self.jbi.id
+        self.jbi.delete()
+        jfc.delay.assert_called_with(jbi_id)
+
+
 class TestJBIReleaseChangedCelery(BaseTest):
     @patch("builtins.open", mock_open(read_data=artifacts_json))
     @patch("repoapi.utils.dlfile")

repoapi/test/test_tasks.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2017-2023 The Sipwise Team - http://sipwise.com
+# Copyright (C) 2017-2024 The Sipwise Team - http://sipwise.com
 #
 # This program is free software: you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by the Free
@@ -55,6 +55,12 @@ class TasksTestCase(BaseTest):
         tasks.jbi_purge.delay(None, 3)
         self.assertEqual(JenkinsBuildInfo.objects.count(), prev_count - 1)
 
+    @patch("repoapi.signals.jbi_files_cleanup")
+    def test_jbi_files_cleanup(self, jfc):
+        jbi = JenkinsBuildInfo.objects.get(pk=1)
+        jbi.delete()
+        jfc.delay.assert_called_once_with(1)
+
 
 @override_settings(JBI_BASEDIR=FIXTURES_PATH)
 @patch("repoapi.tasks.jenkins_remove_project_ppa")

repoapi/test/test_utils.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2017-2022 The Sipwise Team - http://sipwise.com
+# Copyright (C) 2017-2024 The Sipwise Team - http://sipwise.com
 #
 # This program is free software: you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by the Free
@@ -12,6 +12,7 @@
 #
 # You should have received a copy of the GNU General Public License along
 # with this program.  If not, see <http://www.gnu.org/licenses/>.
+from unittest.mock import Mock
 from unittest.mock import patch
 
 from django.test import override_settings
@@ -55,3 +56,32 @@ class UtilsTestCase(BaseTest):
             utils.is_download_artifacts("fake-release-tools-runner")
         )
         self.assertTrue(utils.is_download_artifacts("whatever-repos"))
+
+    @patch("repoapi.utils.shutil")
+    def test_cleanup_build_ko(self, sh):
+        dst_path = Mock()
+        dst_path.exists.return_value = True
+        build_path = Mock()
+        build_path.exists.return_value = False
+        utils.cleanup_build(build_path, dst_path)
+        sh.move.assert_not_called()
+
+    @patch("repoapi.utils.shutil")
+    def test_cleanup_build_no_dst(self, sh):
+        dst_path = Mock()
+        dst_path.exists.return_value = False
+        build_path = Mock()
+        build_path.exists.return_value = True
+        utils.cleanup_build(build_path, dst_path)
+        dst_path.mkdir.assert_called_once()
+        sh.move.assert_called_once_with(build_path, dst_path)
+
+    @patch("repoapi.utils.shutil")
+    def test_cleanup_build(self, sh):
+        dst_path = Mock()
+        dst_path.exists.return_value = True
+        build_path = Mock()
+        build_path.exists.return_value = True
+        utils.cleanup_build(build_path, dst_path)
+        dst_path.mkdir.assert_not_called()
+        sh.move.assert_called_once_with(build_path, dst_path)

repoapi/utils.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2023 The Sipwise Team - http://sipwise.com
+# Copyright (C) 2015-2024 The Sipwise Team - http://sipwise.com
 #
 # This program is free software: you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by the Free
@@ -13,6 +13,7 @@
 # You should have received a copy of the GNU General Public License along
 # with this program.  If not, see <http://www.gnu.org/licenses/>.
 import re
+import shutil
 import subprocess
 from pathlib import Path
 
@@ -189,3 +190,11 @@ def is_download_artifacts(jobname):
         if re.search(check, jobname) is not None:
             return True
     return False
+
+
+def cleanup_build(build_path: Path, dst_path: Path):
+    if not dst_path.exists():
+        dst_path.mkdir(parents=True, exist_ok=True)
+    if build_path.exists():
+        shutil.move(build_path, dst_path)
+        logger.info(f"{build_path} stored at {dst_path}")