MT#59039 jbi_files command

Provide django command line jbi_files. The cleanup action
will archive build files of missing builds on database.

This will help to cleanup the huge amount of files we keep
right now in JBI_FILES

Change-Id: Ideb5a939807c41c411fa3883c0566221d7221189

repoapi/fixtures/test_model_queries.json

@@ -70,7 +70,7 @@
     },
     {
         "fields": {
-            "buildnumber": 1,
+            "buildnumber": 2,
             "date": "2015-05-04T17:04:57.802Z",
             "gerrit_change": null,
             "gerrit_eventtype": null,

repoapi/management/commands/jbi_files.py

@@ -0,0 +1,72 @@
+# Copyright (C) 2024 The Sipwise Team - http://sipwise.com
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+from os import scandir
+from pathlib import Path
+
+from django.conf import settings
+from django.core.management.base import BaseCommand
+
+from repoapi.models import JenkinsBuildInfo
+from repoapi.utils import cleanup_build
+
+
+class Command(BaseCommand):
+    help = "jbi_files related actions"
+
+    def get_missing_builds(self, jobname, path: Path):
+        qs = JenkinsBuildInfo.objects.job_builds(jobname)
+        known_builds = [str(x) for x in qs]
+        missing_builds = set()
+        with scandir(path) as it:
+            for item in it:
+                if item.is_dir() and item.name not in known_builds:
+                    missing_builds.add(item.name)
+        count = len(missing_builds)
+        if count > 0:
+            self.stdout.write(
+                self.style.NOTICE(
+                    f"detected {count} missing builds files for {jobname}"
+                )
+            )
+        return missing_builds
+
+    def add_arguments(self, parser):
+        parser.add_argument("action", choices=["cleanup"])
+        parser.add_argument("--dry-run", action="store_true")
+
+    def cleanup(self, *args, **options):
+        jobnames = []
+        with scandir(settings.JBI_BASEDIR) as it:
+            for item in it:
+                if item.is_dir():
+                    jobnames.append(item.name)
+        for jobname in jobnames:
+            path = settings.JBI_BASEDIR / jobname
+            dst_path = settings.JBI_ARCHIVE / jobname
+            missing_builds = self.get_missing_builds(jobname, path)
+            if options["dry_run"]:
+                continue
+            for build in missing_builds:
+                cleanup_build(path / build, dst_path)
+            if len(missing_builds) > 0:
+                self.stdout.write(
+                    self.style.SUCCESS(
+                        f"archived {missing_builds} dirs at {path}"
+                    )
+                )
+
+    def handle(self, *args, **options):
+        action = getattr(self, options["action"])
+        action(*args, **options)

repoapi/models/jbi.py

@@ -202,6 +202,15 @@ class JenkinsBuildInfoManager(models.Manager):
                 param_release=release, date__date__lt=_date
             ).delete()
 
+    def job_builds(self, jobname):
+        res = (
+            self.get_queryset()
+            .filter(jobname=jobname)
+            .values_list("buildnumber", flat=True)
+            .order_by("buildnumber")
+        )
+        return res
+
 
 class JenkinsBuildInfo(models.Model):
     tag = models.CharField(max_length=64, null=True)

repoapi/test/test_commands.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2022 The Sipwise Team - http://sipwise.com
+# Copyright (C) 2022-2024 The Sipwise Team - http://sipwise.com
 #
 # This program is free software: you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by the Free
@@ -12,6 +12,8 @@
 #
 # You should have received a copy of the GNU General Public License along
 # with this program.  If not, see <http://www.gnu.org/licenses/>.
+import io
+import shutil
 from tempfile import NamedTemporaryFile
 
 from django.core.management import call_command
@@ -95,3 +97,108 @@ class apikeyTest(BaseTest):
         self.cmd.append("--value")
         self.cmd.append(self.key)
         call_command(*self.cmd)
+
+
+class JBIFilesTest(BaseTest):
+    def setUp(self):
+        super().setUp()
+        self.cmd = [
+            "jbi_files",
+        ]
+
+    def prepare_jbi_files(self, jobname):
+        shutil.copytree(
+            FIXTURES_PATH / "jbi_files", self.path, dirs_exist_ok=True
+        )
+        self.assertTrue((self.path / jobname).exists())
+
+    def test_cleanup(self):
+        jobname = "lua-ngcp-kamailio-repos"
+        self.prepare_jbi_files(jobname)
+        qs = JenkinsBuildInfo.objects.filter(
+            jobname=jobname,
+        )
+        self.assertEqual(qs.count(), 0)
+        self.assertTrue((self.path / jobname / "605").exists())
+        self.cmd.append("cleanup")
+        with io.StringIO() as out:
+            call_command(*self.cmd, stdout=out)
+            stdout = out.getvalue()
+            print(stdout)
+        self.assertIn(f"detected 4 missing builds files for {jobname}", stdout)
+        self.assertFalse(
+            (self.path / "lua-ngcp-kamailio-repos" / "605").exists()
+        )
+        self.assertTrue(
+            (self.archive_path / "lua-ngcp-kamailio-repos" / "605").exists()
+        )
+
+    def test_cleanup_dry_run(self):
+        jobname = "lua-ngcp-kamailio-repos"
+        self.prepare_jbi_files(jobname)
+        qs = JenkinsBuildInfo.objects.filter(
+            jobname=jobname,
+        )
+        self.assertEqual(qs.count(), 0)
+        self.assertTrue((self.path / jobname / "605").exists())
+        self.cmd.append("cleanup")
+        self.cmd.append("--dry-run")
+        with io.StringIO() as out:
+            call_command(*self.cmd, stdout=out)
+            stdout = out.getvalue()
+            print(stdout)
+        self.assertIn(f"detected 4 missing builds files for {jobname}", stdout)
+        self.assertTrue(
+            (self.path / "lua-ngcp-kamailio-repos" / "605").exists()
+        )
+        self.assertFalse(
+            (self.archive_path / "lua-ngcp-kamailio-repos" / "605").exists()
+        )
+
+    def test_cleanup_ok(self):
+        jobname = "lua-ngcp-kamailio-repos"
+        self.prepare_jbi_files(jobname)
+        params = {
+            "gerrit_patchset": "44323",
+            "gerrit_change": "44323",
+            "gerrit_eventtype": "patchset-created",
+            "tag": "de13c0b6-2e70-4c9d-b3a5-3a476149d2d1",
+            "projectname": "lua-ngcp-kamailio",
+            "git_commit_msg": "TT#95650 mocks/pv: ",
+            "job_url": "https://fake/job/lua-ngcp-kamailio-repos/",
+            "buildnumber": 605,
+            "jobname": "lua-ngcp-kamailio-repos",
+            "result": "SUCCESS",
+            "param_tag": "none",
+            "param_branch": "master",
+            "param_release": "none",
+            "param_release_uuid": "",
+            "param_distribution": "buster",
+            "param_ppa": "gerrit_vseva_95650",
+        }
+        JenkinsBuildInfo.objects.create(**params)
+        qs = JenkinsBuildInfo.objects.filter(
+            jobname=jobname,
+        )
+        self.assertEqual(qs.count(), 1)
+        self.assertTrue((self.path / jobname / "605").exists())
+        self.cmd.append("cleanup")
+        with io.StringIO() as out:
+            call_command(*self.cmd, stdout=out)
+            stdout = out.getvalue()
+            print(stdout)
+        self.assertIn(f"detected 3 missing builds files for {jobname}", stdout)
+        self.assertTrue(
+            (self.path / "lua-ngcp-kamailio-repos" / "605").exists()
+        )
+        self.assertFalse(
+            (self.archive_path / "lua-ngcp-kamailio-repos" / "605").exists()
+        )
+
+    def test_cleanup_quiet(self):
+        self.cmd.append("cleanup")
+        with io.StringIO() as out:
+            call_command(*self.cmd, stdout=out)
+            stdout = out.getvalue()
+            print(stdout)
+        self.assertEqual(stdout, "")

repoapi/test/test_model_queries.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2022 The Sipwise Team - http://sipwise.com
+# Copyright (C) 2015-2024 The Sipwise Team - http://sipwise.com
 #
 # This program is free software: you can redistribute it and/or modify it
 # under the terms of the GNU General Public License as published by the Free
@@ -103,6 +103,12 @@ class JBIQueriesTestCase(BaseTest):
         JenkinsBuildInfo.objects.purge_release(None, timedelta(weeks=3))
         self.assertEqual(JenkinsBuildInfo.objects.count(), prev_count - 1)
 
+    def test_job_builds(self):
+        res = JenkinsBuildInfo.objects.job_builds("fake-get-code")
+        self.assertEqual(res.count(), 2)
+        self.assertEqual(res[0], 1)
+        self.assertEqual(res[1], 2)
+
 
 class JBIQueriesTrunk(BaseTest):
     fixtures = ["test_model_queries_trunk.yaml"]