diff --git a/repoapi/models/jbi.py b/repoapi/models/jbi.py index f752a0b..e39b59f 100644 --- a/repoapi/models/jbi.py +++ b/repoapi/models/jbi.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2022 The Sipwise Team - http://sipwise.com +# Copyright (C) 2015-2024 The Sipwise Team - http://sipwise.com # # This program is free software: you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the Free @@ -18,6 +18,7 @@ import re from collections import OrderedDict from datetime import datetime from datetime import timedelta +from pathlib import Path from urllib.parse import urlparse import structlog @@ -254,7 +255,7 @@ class JenkinsBuildInfo(models.Model): return self.param_ppa not in ["$ppa", None] @property - def build_path(self): + def build_path(self) -> Path: return settings.JBI_BASEDIR.joinpath( self.jobname, str(self.buildnumber) ) diff --git a/repoapi/settings/dev.py b/repoapi/settings/dev.py index fd8e7e0..955599f 100644 --- a/repoapi/settings/dev.py +++ b/repoapi/settings/dev.py @@ -32,6 +32,7 @@ BROKER_BACKEND = "amqp" CELERY_TASK_ALWAYS_EAGER = False CELERY_BROKER_URL = "amqp://guest:guest@rabbit" JBI_BASEDIR = BASE_DIR / "jbi_files" # noqa +JBI_ARCHIVE = BASE_DIR / "jbi_archive" # noqa # Enable access when not accessing from localhost: ALLOWED_HOSTS = [ diff --git a/repoapi/settings/prod.py b/repoapi/settings/prod.py index c3bfa69..b75bdc5 100644 --- a/repoapi/settings/prod.py +++ b/repoapi/settings/prod.py @@ -171,6 +171,7 @@ CELERY_TIMEZONE = "UTC" FLOWER_URL_PREFIX = "flower" JBI_BASEDIR = VAR_DIR / "jbi_files" +JBI_ARCHIVE = Path("/srv/repoapi_archive") JBI_ARTIFACT_JOBS = [ "release-tools-runner", ] diff --git a/repoapi/signals.py b/repoapi/signals.py index 58a0edf..7f1b1f2 100644 --- a/repoapi/signals.py +++ b/repoapi/signals.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022-2023 The Sipwise Team - http://sipwise.com +# Copyright (C) 2022-2024 The Sipwise Team - http://sipwise.com # # This program is free software: you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the Free @@ -15,10 +15,12 @@ import structlog from django.apps import apps from django.db.models.signals import post_save +from django.db.models.signals import pre_delete from django.dispatch import receiver from .models.wni import NoteInfo from .tasks import get_jbi_files +from .tasks import jbi_files_cleanup from .utils import get_next_release from .utils import regex_mrXXX from release_dashboard.utils.build import is_ngcp_project @@ -38,6 +40,13 @@ def jbi_manage(sender, **kwargs): ) +@receiver( + pre_delete, sender="repoapi.JenkinsBuildInfo", dispatch_uid="jbi_cleanup" +) +def jbi_cleanup(sender, **kwargs): + jbi_files_cleanup.delay(kwargs["instance"].pk) + + def gerrit_repo_add(instance): if not instance.has_ppa: logger.warn("ppa unset, skip removal") diff --git a/repoapi/tasks.py b/repoapi/tasks.py index 567c49b..67975e0 100644 --- a/repoapi/tasks.py +++ b/repoapi/tasks.py @@ -1,4 +1,4 @@ -# Copyright (C) 2016-2022 The Sipwise Team - http://sipwise.com +# Copyright (C) 2016-2024 The Sipwise Team - http://sipwise.com # # This program is free software: you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the Free @@ -23,6 +23,7 @@ from .celery import jbi_parse_buildinfo from .celery import jbi_parse_hotfix from .celery import process_result from .conf import settings +from .utils import cleanup_build from .utils import is_download_artifacts from .utils import jenkins_get_artifact from .utils import jenkins_get_build @@ -91,3 +92,12 @@ def jbi_purge(release, weeks): JenkinsBuildInfo = apps.get_model("repoapi", "JenkinsBuildInfo") JenkinsBuildInfo.objects.purge_release(release, timedelta(weeks=weeks)) logger.info(f"purged release {release} jbi older than {weeks} weeks") + + +@shared_task(ignore_result=True) +def jbi_files_cleanup(jbi_id): + JenkinsBuildInfo = apps.get_model("repoapi", "JenkinsBuildInfo") + jbi = JenkinsBuildInfo.objects.get(id=jbi_id) + build_path = jbi.build_path + dst_path = settings.JBI_ARCHIVE / jbi.jobname + cleanup_build(build_path, dst_path) diff --git a/repoapi/test/base.py b/repoapi/test/base.py index 927c14f..a0cb1cb 100644 --- a/repoapi/test/base.py +++ b/repoapi/test/base.py @@ -24,29 +24,34 @@ from rest_framework.test import APITestCase from rest_framework_api_key.models import APIKey JBI_BASEDIR = Path(mkdtemp(dir=os.environ.get("RESULTS"))) +JBI_ARCHIVE = Path(mkdtemp(dir=os.environ.get("RESULTS"))) -@override_settings(DEBUG=True, JBI_BASEDIR=JBI_BASEDIR) +@override_settings( + DEBUG=True, JBI_BASEDIR=JBI_BASEDIR, JBI_ARCHIVE=JBI_ARCHIVE +) class BaseTest(TestCase): @classmethod def setUpTestData(cls): from repoapi.conf import settings cls.path = Path(settings.JBI_BASEDIR) + cls.archive_path = Path(settings.JBI_ARCHIVE) def setUp(self): RepoAPIConfig = apps.get_app_config("repoapi") RepoAPIConfig.ready() super(BaseTest, self).setUp() - self.path.mkdir(parents=True, exist_ok=True) + for path in [self.path, self.archive_path]: + path.mkdir(parents=True, exist_ok=True) def tearDown(self): - if self.path.exists(): - shutil.rmtree(self.path) + for path in [self.path, self.archive_path]: + if path.exists(): + shutil.rmtree(path) class APIAuthenticatedTestCase(BaseTest, APITestCase): - APP_NAME = "Project Tests" def setUp(self): diff --git a/repoapi/test/test_jbi_info.py b/repoapi/test/test_jbi_info.py index 7fb5840..ff25ae7 100644 --- a/repoapi/test/test_jbi_info.py +++ b/repoapi/test/test_jbi_info.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2022 The Sipwise Team - http://sipwise.com +# Copyright (C) 2015-2024 The Sipwise Team - http://sipwise.com # # This program is free software: you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the Free @@ -137,6 +137,29 @@ class TestJBICelery(BaseTest): dlfile.assert_any_call(url, path) +class TestJBICleanUP(BaseTest): + fixtures = ["test_model_queries"] + + def setUp(self): + super().setUp() + self.jbi = JenkinsBuildInfo.objects.get(id=1) + self.jbi.build_path.mkdir(parents=True, exist_ok=True) + + @patch("repoapi.utils.shutil") + def test_jbi_cleanup(self, sh): + build_path = self.jbi.build_path + dst_path = settings.JBI_ARCHIVE / self.jbi.jobname + self.assertTrue(build_path.exists()) + self.jbi.delete() + sh.move.assert_called_with(build_path, dst_path) + + @patch("repoapi.signals.jbi_files_cleanup") + def test_jbi_cleanup_called(self, jfc): + jbi_id = self.jbi.id + self.jbi.delete() + jfc.delay.assert_called_with(jbi_id) + + class TestJBIReleaseChangedCelery(BaseTest): @patch("builtins.open", mock_open(read_data=artifacts_json)) @patch("repoapi.utils.dlfile") diff --git a/repoapi/test/test_tasks.py b/repoapi/test/test_tasks.py index afcfd8a..ffc78be 100644 --- a/repoapi/test/test_tasks.py +++ b/repoapi/test/test_tasks.py @@ -1,4 +1,4 @@ -# Copyright (C) 2017-2023 The Sipwise Team - http://sipwise.com +# Copyright (C) 2017-2024 The Sipwise Team - http://sipwise.com # # This program is free software: you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the Free @@ -55,6 +55,12 @@ class TasksTestCase(BaseTest): tasks.jbi_purge.delay(None, 3) self.assertEqual(JenkinsBuildInfo.objects.count(), prev_count - 1) + @patch("repoapi.signals.jbi_files_cleanup") + def test_jbi_files_cleanup(self, jfc): + jbi = JenkinsBuildInfo.objects.get(pk=1) + jbi.delete() + jfc.delay.assert_called_once_with(1) + @override_settings(JBI_BASEDIR=FIXTURES_PATH) @patch("repoapi.tasks.jenkins_remove_project_ppa") diff --git a/repoapi/test/test_utils.py b/repoapi/test/test_utils.py index b3cc3e3..791b990 100644 --- a/repoapi/test/test_utils.py +++ b/repoapi/test/test_utils.py @@ -1,4 +1,4 @@ -# Copyright (C) 2017-2022 The Sipwise Team - http://sipwise.com +# Copyright (C) 2017-2024 The Sipwise Team - http://sipwise.com # # This program is free software: you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the Free @@ -12,6 +12,7 @@ # # You should have received a copy of the GNU General Public License along # with this program. If not, see . +from unittest.mock import Mock from unittest.mock import patch from django.test import override_settings @@ -55,3 +56,32 @@ class UtilsTestCase(BaseTest): utils.is_download_artifacts("fake-release-tools-runner") ) self.assertTrue(utils.is_download_artifacts("whatever-repos")) + + @patch("repoapi.utils.shutil") + def test_cleanup_build_ko(self, sh): + dst_path = Mock() + dst_path.exists.return_value = True + build_path = Mock() + build_path.exists.return_value = False + utils.cleanup_build(build_path, dst_path) + sh.move.assert_not_called() + + @patch("repoapi.utils.shutil") + def test_cleanup_build_no_dst(self, sh): + dst_path = Mock() + dst_path.exists.return_value = False + build_path = Mock() + build_path.exists.return_value = True + utils.cleanup_build(build_path, dst_path) + dst_path.mkdir.assert_called_once() + sh.move.assert_called_once_with(build_path, dst_path) + + @patch("repoapi.utils.shutil") + def test_cleanup_build(self, sh): + dst_path = Mock() + dst_path.exists.return_value = True + build_path = Mock() + build_path.exists.return_value = True + utils.cleanup_build(build_path, dst_path) + dst_path.mkdir.assert_not_called() + sh.move.assert_called_once_with(build_path, dst_path) diff --git a/repoapi/utils.py b/repoapi/utils.py index 86181ac..8f1115e 100644 --- a/repoapi/utils.py +++ b/repoapi/utils.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2023 The Sipwise Team - http://sipwise.com +# Copyright (C) 2015-2024 The Sipwise Team - http://sipwise.com # # This program is free software: you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the Free @@ -13,6 +13,7 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . import re +import shutil import subprocess from pathlib import Path @@ -189,3 +190,11 @@ def is_download_artifacts(jobname): if re.search(check, jobname) is not None: return True return False + + +def cleanup_build(build_path: Path, dst_path: Path): + if not dst_path.exists(): + dst_path.mkdir(parents=True, exist_ok=True) + if build_path.exists(): + shutil.move(build_path, dst_path) + logger.info(f"{build_path} stored at {dst_path}")