diff --git a/repoapi/models/jbi.py b/repoapi/models/jbi.py
index f752a0b..e39b59f 100644
--- a/repoapi/models/jbi.py
+++ b/repoapi/models/jbi.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2022 The Sipwise Team - http://sipwise.com
+# Copyright (C) 2015-2024 The Sipwise Team - http://sipwise.com
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free
@@ -18,6 +18,7 @@ import re
from collections import OrderedDict
from datetime import datetime
from datetime import timedelta
+from pathlib import Path
from urllib.parse import urlparse
import structlog
@@ -254,7 +255,7 @@ class JenkinsBuildInfo(models.Model):
return self.param_ppa not in ["$ppa", None]
@property
- def build_path(self):
+ def build_path(self) -> Path:
return settings.JBI_BASEDIR.joinpath(
self.jobname, str(self.buildnumber)
)
diff --git a/repoapi/settings/dev.py b/repoapi/settings/dev.py
index fd8e7e0..955599f 100644
--- a/repoapi/settings/dev.py
+++ b/repoapi/settings/dev.py
@@ -32,6 +32,7 @@ BROKER_BACKEND = "amqp"
CELERY_TASK_ALWAYS_EAGER = False
CELERY_BROKER_URL = "amqp://guest:guest@rabbit"
JBI_BASEDIR = BASE_DIR / "jbi_files" # noqa
+JBI_ARCHIVE = BASE_DIR / "jbi_archive" # noqa
# Enable access when not accessing from localhost:
ALLOWED_HOSTS = [
diff --git a/repoapi/settings/prod.py b/repoapi/settings/prod.py
index c3bfa69..b75bdc5 100644
--- a/repoapi/settings/prod.py
+++ b/repoapi/settings/prod.py
@@ -171,6 +171,7 @@ CELERY_TIMEZONE = "UTC"
FLOWER_URL_PREFIX = "flower"
JBI_BASEDIR = VAR_DIR / "jbi_files"
+JBI_ARCHIVE = Path("/srv/repoapi_archive")
JBI_ARTIFACT_JOBS = [
"release-tools-runner",
]
diff --git a/repoapi/signals.py b/repoapi/signals.py
index 58a0edf..7f1b1f2 100644
--- a/repoapi/signals.py
+++ b/repoapi/signals.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2022-2023 The Sipwise Team - http://sipwise.com
+# Copyright (C) 2022-2024 The Sipwise Team - http://sipwise.com
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free
@@ -15,10 +15,12 @@
import structlog
from django.apps import apps
from django.db.models.signals import post_save
+from django.db.models.signals import pre_delete
from django.dispatch import receiver
from .models.wni import NoteInfo
from .tasks import get_jbi_files
+from .tasks import jbi_files_cleanup
from .utils import get_next_release
from .utils import regex_mrXXX
from release_dashboard.utils.build import is_ngcp_project
@@ -38,6 +40,13 @@ def jbi_manage(sender, **kwargs):
)
+@receiver(
+ pre_delete, sender="repoapi.JenkinsBuildInfo", dispatch_uid="jbi_cleanup"
+)
+def jbi_cleanup(sender, **kwargs):
+ jbi_files_cleanup.delay(kwargs["instance"].pk)
+
+
def gerrit_repo_add(instance):
if not instance.has_ppa:
logger.warn("ppa unset, skip removal")
diff --git a/repoapi/tasks.py b/repoapi/tasks.py
index 567c49b..67975e0 100644
--- a/repoapi/tasks.py
+++ b/repoapi/tasks.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2016-2022 The Sipwise Team - http://sipwise.com
+# Copyright (C) 2016-2024 The Sipwise Team - http://sipwise.com
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free
@@ -23,6 +23,7 @@ from .celery import jbi_parse_buildinfo
from .celery import jbi_parse_hotfix
from .celery import process_result
from .conf import settings
+from .utils import cleanup_build
from .utils import is_download_artifacts
from .utils import jenkins_get_artifact
from .utils import jenkins_get_build
@@ -91,3 +92,12 @@ def jbi_purge(release, weeks):
JenkinsBuildInfo = apps.get_model("repoapi", "JenkinsBuildInfo")
JenkinsBuildInfo.objects.purge_release(release, timedelta(weeks=weeks))
logger.info(f"purged release {release} jbi older than {weeks} weeks")
+
+
+@shared_task(ignore_result=True)
+def jbi_files_cleanup(jbi_id):
+ JenkinsBuildInfo = apps.get_model("repoapi", "JenkinsBuildInfo")
+ jbi = JenkinsBuildInfo.objects.get(id=jbi_id)
+ build_path = jbi.build_path
+ dst_path = settings.JBI_ARCHIVE / jbi.jobname
+ cleanup_build(build_path, dst_path)
diff --git a/repoapi/test/base.py b/repoapi/test/base.py
index 927c14f..a0cb1cb 100644
--- a/repoapi/test/base.py
+++ b/repoapi/test/base.py
@@ -24,29 +24,34 @@ from rest_framework.test import APITestCase
from rest_framework_api_key.models import APIKey
JBI_BASEDIR = Path(mkdtemp(dir=os.environ.get("RESULTS")))
+JBI_ARCHIVE = Path(mkdtemp(dir=os.environ.get("RESULTS")))
-@override_settings(DEBUG=True, JBI_BASEDIR=JBI_BASEDIR)
+@override_settings(
+ DEBUG=True, JBI_BASEDIR=JBI_BASEDIR, JBI_ARCHIVE=JBI_ARCHIVE
+)
class BaseTest(TestCase):
@classmethod
def setUpTestData(cls):
from repoapi.conf import settings
cls.path = Path(settings.JBI_BASEDIR)
+ cls.archive_path = Path(settings.JBI_ARCHIVE)
def setUp(self):
RepoAPIConfig = apps.get_app_config("repoapi")
RepoAPIConfig.ready()
super(BaseTest, self).setUp()
- self.path.mkdir(parents=True, exist_ok=True)
+ for path in [self.path, self.archive_path]:
+ path.mkdir(parents=True, exist_ok=True)
def tearDown(self):
- if self.path.exists():
- shutil.rmtree(self.path)
+ for path in [self.path, self.archive_path]:
+ if path.exists():
+ shutil.rmtree(path)
class APIAuthenticatedTestCase(BaseTest, APITestCase):
-
APP_NAME = "Project Tests"
def setUp(self):
diff --git a/repoapi/test/test_jbi_info.py b/repoapi/test/test_jbi_info.py
index 7fb5840..ff25ae7 100644
--- a/repoapi/test/test_jbi_info.py
+++ b/repoapi/test/test_jbi_info.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2022 The Sipwise Team - http://sipwise.com
+# Copyright (C) 2015-2024 The Sipwise Team - http://sipwise.com
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free
@@ -137,6 +137,29 @@ class TestJBICelery(BaseTest):
dlfile.assert_any_call(url, path)
+class TestJBICleanUP(BaseTest):
+ fixtures = ["test_model_queries"]
+
+ def setUp(self):
+ super().setUp()
+ self.jbi = JenkinsBuildInfo.objects.get(id=1)
+ self.jbi.build_path.mkdir(parents=True, exist_ok=True)
+
+ @patch("repoapi.utils.shutil")
+ def test_jbi_cleanup(self, sh):
+ build_path = self.jbi.build_path
+ dst_path = settings.JBI_ARCHIVE / self.jbi.jobname
+ self.assertTrue(build_path.exists())
+ self.jbi.delete()
+ sh.move.assert_called_with(build_path, dst_path)
+
+ @patch("repoapi.signals.jbi_files_cleanup")
+ def test_jbi_cleanup_called(self, jfc):
+ jbi_id = self.jbi.id
+ self.jbi.delete()
+ jfc.delay.assert_called_with(jbi_id)
+
+
class TestJBIReleaseChangedCelery(BaseTest):
@patch("builtins.open", mock_open(read_data=artifacts_json))
@patch("repoapi.utils.dlfile")
diff --git a/repoapi/test/test_tasks.py b/repoapi/test/test_tasks.py
index afcfd8a..ffc78be 100644
--- a/repoapi/test/test_tasks.py
+++ b/repoapi/test/test_tasks.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2017-2023 The Sipwise Team - http://sipwise.com
+# Copyright (C) 2017-2024 The Sipwise Team - http://sipwise.com
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free
@@ -55,6 +55,12 @@ class TasksTestCase(BaseTest):
tasks.jbi_purge.delay(None, 3)
self.assertEqual(JenkinsBuildInfo.objects.count(), prev_count - 1)
+ @patch("repoapi.signals.jbi_files_cleanup")
+ def test_jbi_files_cleanup(self, jfc):
+ jbi = JenkinsBuildInfo.objects.get(pk=1)
+ jbi.delete()
+ jfc.delay.assert_called_once_with(1)
+
@override_settings(JBI_BASEDIR=FIXTURES_PATH)
@patch("repoapi.tasks.jenkins_remove_project_ppa")
diff --git a/repoapi/test/test_utils.py b/repoapi/test/test_utils.py
index b3cc3e3..791b990 100644
--- a/repoapi/test/test_utils.py
+++ b/repoapi/test/test_utils.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2017-2022 The Sipwise Team - http://sipwise.com
+# Copyright (C) 2017-2024 The Sipwise Team - http://sipwise.com
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free
@@ -12,6 +12,7 @@
#
# You should have received a copy of the GNU General Public License along
# with this program. If not, see .
+from unittest.mock import Mock
from unittest.mock import patch
from django.test import override_settings
@@ -55,3 +56,32 @@ class UtilsTestCase(BaseTest):
utils.is_download_artifacts("fake-release-tools-runner")
)
self.assertTrue(utils.is_download_artifacts("whatever-repos"))
+
+ @patch("repoapi.utils.shutil")
+ def test_cleanup_build_ko(self, sh):
+ dst_path = Mock()
+ dst_path.exists.return_value = True
+ build_path = Mock()
+ build_path.exists.return_value = False
+ utils.cleanup_build(build_path, dst_path)
+ sh.move.assert_not_called()
+
+ @patch("repoapi.utils.shutil")
+ def test_cleanup_build_no_dst(self, sh):
+ dst_path = Mock()
+ dst_path.exists.return_value = False
+ build_path = Mock()
+ build_path.exists.return_value = True
+ utils.cleanup_build(build_path, dst_path)
+ dst_path.mkdir.assert_called_once()
+ sh.move.assert_called_once_with(build_path, dst_path)
+
+ @patch("repoapi.utils.shutil")
+ def test_cleanup_build(self, sh):
+ dst_path = Mock()
+ dst_path.exists.return_value = True
+ build_path = Mock()
+ build_path.exists.return_value = True
+ utils.cleanup_build(build_path, dst_path)
+ dst_path.mkdir.assert_not_called()
+ sh.move.assert_called_once_with(build_path, dst_path)
diff --git a/repoapi/utils.py b/repoapi/utils.py
index 86181ac..8f1115e 100644
--- a/repoapi/utils.py
+++ b/repoapi/utils.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2023 The Sipwise Team - http://sipwise.com
+# Copyright (C) 2015-2024 The Sipwise Team - http://sipwise.com
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free
@@ -13,6 +13,7 @@
# You should have received a copy of the GNU General Public License along
# with this program. If not, see .
import re
+import shutil
import subprocess
from pathlib import Path
@@ -189,3 +190,11 @@ def is_download_artifacts(jobname):
if re.search(check, jobname) is not None:
return True
return False
+
+
+def cleanup_build(build_path: Path, dst_path: Path):
+ if not dst_path.exists():
+ dst_path.mkdir(parents=True, exist_ok=True)
+ if build_path.exists():
+ shutil.move(build_path, dst_path)
+ logger.info(f"{build_path} stored at {dst_path}")