diff --git a/repoapi/fixtures/test_model_queries.json b/repoapi/fixtures/test_model_queries.json index 8b8ae38..333b0ce 100644 --- a/repoapi/fixtures/test_model_queries.json +++ b/repoapi/fixtures/test_model_queries.json @@ -70,7 +70,7 @@ }, { "fields": { - "buildnumber": 1, + "buildnumber": 2, "date": "2015-05-04T17:04:57.802Z", "gerrit_change": null, "gerrit_eventtype": null, diff --git a/repoapi/management/commands/jbi_files.py b/repoapi/management/commands/jbi_files.py new file mode 100644 index 0000000..c4c6bc1 --- /dev/null +++ b/repoapi/management/commands/jbi_files.py @@ -0,0 +1,72 @@ +# Copyright (C) 2024 The Sipwise Team - http://sipwise.com +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +from os import scandir +from pathlib import Path + +from django.conf import settings +from django.core.management.base import BaseCommand + +from repoapi.models import JenkinsBuildInfo +from repoapi.utils import cleanup_build + + +class Command(BaseCommand): + help = "jbi_files related actions" + + def get_missing_builds(self, jobname, path: Path): + qs = JenkinsBuildInfo.objects.job_builds(jobname) + known_builds = [str(x) for x in qs] + missing_builds = set() + with scandir(path) as it: + for item in it: + if item.is_dir() and item.name not in known_builds: + missing_builds.add(item.name) + count = len(missing_builds) + if count > 0: + self.stdout.write( + self.style.NOTICE( + f"detected {count} missing builds files for {jobname}" + ) + ) + return missing_builds + + def add_arguments(self, parser): + parser.add_argument("action", choices=["cleanup"]) + parser.add_argument("--dry-run", action="store_true") + + def cleanup(self, *args, **options): + jobnames = [] + with scandir(settings.JBI_BASEDIR) as it: + for item in it: + if item.is_dir(): + jobnames.append(item.name) + for jobname in jobnames: + path = settings.JBI_BASEDIR / jobname + dst_path = settings.JBI_ARCHIVE / jobname + missing_builds = self.get_missing_builds(jobname, path) + if options["dry_run"]: + continue + for build in missing_builds: + cleanup_build(path / build, dst_path) + if len(missing_builds) > 0: + self.stdout.write( + self.style.SUCCESS( + f"archived {missing_builds} dirs at {path}" + ) + ) + + def handle(self, *args, **options): + action = getattr(self, options["action"]) + action(*args, **options) diff --git a/repoapi/models/jbi.py b/repoapi/models/jbi.py index e39b59f..8c0662c 100644 --- a/repoapi/models/jbi.py +++ b/repoapi/models/jbi.py @@ -202,6 +202,15 @@ class JenkinsBuildInfoManager(models.Manager): param_release=release, date__date__lt=_date ).delete() + def job_builds(self, jobname): + res = ( + self.get_queryset() + .filter(jobname=jobname) + .values_list("buildnumber", flat=True) + .order_by("buildnumber") + ) + return res + class JenkinsBuildInfo(models.Model): tag = models.CharField(max_length=64, null=True) diff --git a/repoapi/test/test_commands.py b/repoapi/test/test_commands.py index 851004a..c997014 100644 --- a/repoapi/test/test_commands.py +++ b/repoapi/test/test_commands.py @@ -1,4 +1,4 @@ -# Copyright (C) 2022 The Sipwise Team - http://sipwise.com +# Copyright (C) 2022-2024 The Sipwise Team - http://sipwise.com # # This program is free software: you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the Free @@ -12,6 +12,8 @@ # # You should have received a copy of the GNU General Public License along # with this program. If not, see . +import io +import shutil from tempfile import NamedTemporaryFile from django.core.management import call_command @@ -95,3 +97,108 @@ class apikeyTest(BaseTest): self.cmd.append("--value") self.cmd.append(self.key) call_command(*self.cmd) + + +class JBIFilesTest(BaseTest): + def setUp(self): + super().setUp() + self.cmd = [ + "jbi_files", + ] + + def prepare_jbi_files(self, jobname): + shutil.copytree( + FIXTURES_PATH / "jbi_files", self.path, dirs_exist_ok=True + ) + self.assertTrue((self.path / jobname).exists()) + + def test_cleanup(self): + jobname = "lua-ngcp-kamailio-repos" + self.prepare_jbi_files(jobname) + qs = JenkinsBuildInfo.objects.filter( + jobname=jobname, + ) + self.assertEqual(qs.count(), 0) + self.assertTrue((self.path / jobname / "605").exists()) + self.cmd.append("cleanup") + with io.StringIO() as out: + call_command(*self.cmd, stdout=out) + stdout = out.getvalue() + print(stdout) + self.assertIn(f"detected 4 missing builds files for {jobname}", stdout) + self.assertFalse( + (self.path / "lua-ngcp-kamailio-repos" / "605").exists() + ) + self.assertTrue( + (self.archive_path / "lua-ngcp-kamailio-repos" / "605").exists() + ) + + def test_cleanup_dry_run(self): + jobname = "lua-ngcp-kamailio-repos" + self.prepare_jbi_files(jobname) + qs = JenkinsBuildInfo.objects.filter( + jobname=jobname, + ) + self.assertEqual(qs.count(), 0) + self.assertTrue((self.path / jobname / "605").exists()) + self.cmd.append("cleanup") + self.cmd.append("--dry-run") + with io.StringIO() as out: + call_command(*self.cmd, stdout=out) + stdout = out.getvalue() + print(stdout) + self.assertIn(f"detected 4 missing builds files for {jobname}", stdout) + self.assertTrue( + (self.path / "lua-ngcp-kamailio-repos" / "605").exists() + ) + self.assertFalse( + (self.archive_path / "lua-ngcp-kamailio-repos" / "605").exists() + ) + + def test_cleanup_ok(self): + jobname = "lua-ngcp-kamailio-repos" + self.prepare_jbi_files(jobname) + params = { + "gerrit_patchset": "44323", + "gerrit_change": "44323", + "gerrit_eventtype": "patchset-created", + "tag": "de13c0b6-2e70-4c9d-b3a5-3a476149d2d1", + "projectname": "lua-ngcp-kamailio", + "git_commit_msg": "TT#95650 mocks/pv: ", + "job_url": "https://fake/job/lua-ngcp-kamailio-repos/", + "buildnumber": 605, + "jobname": "lua-ngcp-kamailio-repos", + "result": "SUCCESS", + "param_tag": "none", + "param_branch": "master", + "param_release": "none", + "param_release_uuid": "", + "param_distribution": "buster", + "param_ppa": "gerrit_vseva_95650", + } + JenkinsBuildInfo.objects.create(**params) + qs = JenkinsBuildInfo.objects.filter( + jobname=jobname, + ) + self.assertEqual(qs.count(), 1) + self.assertTrue((self.path / jobname / "605").exists()) + self.cmd.append("cleanup") + with io.StringIO() as out: + call_command(*self.cmd, stdout=out) + stdout = out.getvalue() + print(stdout) + self.assertIn(f"detected 3 missing builds files for {jobname}", stdout) + self.assertTrue( + (self.path / "lua-ngcp-kamailio-repos" / "605").exists() + ) + self.assertFalse( + (self.archive_path / "lua-ngcp-kamailio-repos" / "605").exists() + ) + + def test_cleanup_quiet(self): + self.cmd.append("cleanup") + with io.StringIO() as out: + call_command(*self.cmd, stdout=out) + stdout = out.getvalue() + print(stdout) + self.assertEqual(stdout, "") diff --git a/repoapi/test/test_model_queries.py b/repoapi/test/test_model_queries.py index aedf3b4..16b004b 100644 --- a/repoapi/test/test_model_queries.py +++ b/repoapi/test/test_model_queries.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2022 The Sipwise Team - http://sipwise.com +# Copyright (C) 2015-2024 The Sipwise Team - http://sipwise.com # # This program is free software: you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the Free @@ -103,6 +103,12 @@ class JBIQueriesTestCase(BaseTest): JenkinsBuildInfo.objects.purge_release(None, timedelta(weeks=3)) self.assertEqual(JenkinsBuildInfo.objects.count(), prev_count - 1) + def test_job_builds(self): + res = JenkinsBuildInfo.objects.job_builds("fake-get-code") + self.assertEqual(res.count(), 2) + self.assertEqual(res[0], 1) + self.assertEqual(res[1], 2) + class JBIQueriesTrunk(BaseTest): fixtures = ["test_model_queries_trunk.yaml"]