MT#55988 buildinfo: collect build information

* we want some Build Jenkins information
* just for devops

Change-Id: I579e67963c4907b8dea7e1890b394737f8c5e2fd

buildinfo/admin.py

@@ -0,0 +1,35 @@
+# Copyright (C) 2022 The Sipwise Team - http://sipwise.com
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+from django.contrib import admin
+from import_export import resources
+from import_export.admin import ImportExportModelAdmin
+
+from . import models
+
+
+class BuildInfoResource(resources.ModelResource):
+    class Meta:
+        model = models.BuildInfo
+
+
+@admin.register(models.BuildInfo)
+class BuildInfoAdmin(ImportExportModelAdmin):
+    resource_class = BuildInfoResource
+    list_filter = (
+        "param_release",
+        "projectname",
+        "param_distribution",
+        "builton",
+    )

buildinfo/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class BuildinfoConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "buildinfo"

buildinfo/conf.py

@@ -0,0 +1,21 @@
+# Copyright (C) 2022 The Sipwise Team - http://sipwise.com
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+from django.conf import settings  # noqa
+from appconf import AppConf
+
+
+class BuildInfoConf(AppConf):
+    class Meta:
+        prefix = "buildinfo"

buildinfo/migrations/0001_initial.py

@@ -0,0 +1,57 @@
+# Generated by Django 3.2.15 on 2022-12-15 15:34
+from django.db import migrations
+from django.db import models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = []
+
+    operations = [
+        migrations.CreateModel(
+            name="BuildInfo",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("builton", models.CharField(max_length=50)),
+                ("timestamp", models.PositiveBigIntegerField()),
+                ("duration", models.PositiveSmallIntegerField()),
+                ("projectname", models.CharField(max_length=100)),
+                ("buildnumber", models.IntegerField()),
+                ("jobname", models.CharField(max_length=100)),
+                (
+                    "param_tag",
+                    models.CharField(blank=True, max_length=50, null=True),
+                ),
+                (
+                    "param_branch",
+                    models.CharField(blank=True, max_length=50, null=True),
+                ),
+                (
+                    "param_release",
+                    models.CharField(db_index=True, max_length=50, null=True),
+                ),
+                (
+                    "param_release_uuid",
+                    models.CharField(blank=True, max_length=64, null=True),
+                ),
+                (
+                    "param_distribution",
+                    models.CharField(blank=True, max_length=50, null=True),
+                ),
+                (
+                    "param_ppa",
+                    models.CharField(blank=True, max_length=50, null=True),
+                ),
+            ],
+        ),
+    ]

buildinfo/migrations/0002_ldap_grp_perms.py

@@ -0,0 +1,46 @@
+# Generated by Django 3.2.13 on 2022-06-14 16:12
+from django.contrib.auth.management import create_permissions
+from django.db import migrations
+
+
+def add_permissions(apps, schema_editor):
+    """ContentType table is populated after all the migrations applied"""
+    for app_config in apps.get_app_configs():
+        app_config.models_module = True
+        create_permissions(app_config, verbosity=0)
+        app_config.models_module = None
+
+
+def forwards_func(apps, schema_editor):
+    add_permissions(apps, schema_editor)
+    Group = apps.get_model("auth", "Group")
+    Permission = apps.get_model("auth", "Permission")
+    ContentType = apps.get_model("contenttypes", "ContentType")
+    db_alias = schema_editor.connection.alias
+
+    devops_grp = Group.objects.using(db_alias).get(name="devops")
+
+    BuildInfo = apps.get_model("buildinfo", "BuildInfo")
+    ct = ContentType.objects.get_for_model(BuildInfo)
+
+    perm_codenames = []
+    for perm in ("add", "change", "delete", "view"):
+        perm_codenames.append(f"{perm}_buildinfo")
+
+    for codename in perm_codenames:
+        perm = Permission.objects.using(db_alias).get(
+            content_type=ct, codename=codename
+        )
+        devops_grp.permissions.add(perm)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("buildinfo", "0001_initial"),
+        ("repoapi", "0011_ldap_groups"),
+    ]
+
+    operations = [
+        migrations.RunPython(forwards_func),
+    ]

buildinfo/models.py

@@ -0,0 +1,37 @@
+# Copyright (C) 2022 The Sipwise Team - http://sipwise.com
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+from django.db import models
+
+
+class BuildInfo(models.Model):
+    builton = models.CharField(max_length=50, null=False)
+    timestamp = models.PositiveBigIntegerField(null=False)
+    duration = models.PositiveSmallIntegerField(null=False)
+
+    projectname = models.CharField(max_length=100, null=False)
+    buildnumber = models.IntegerField()
+    jobname = models.CharField(max_length=100, null=False)
+    param_tag = models.CharField(max_length=50, null=True, blank=True)
+    param_branch = models.CharField(max_length=50, null=True, blank=True)
+    param_release = models.CharField(max_length=50, null=True, db_index=True)
+    param_release_uuid = models.CharField(max_length=64, null=True, blank=True)
+    param_distribution = models.CharField(max_length=50, null=True, blank=True)
+    param_ppa = models.CharField(max_length=50, null=True, blank=True)
+
+    def __str__(self):
+        return (
+            f"{self.jobname}:{self.buildnumber}:"
+            f"{self.param_branch}:{self.param_release}"
+        )

buildinfo/tasks.py

@@ -0,0 +1,25 @@
+# Copyright (C) 2017-2022 The Sipwise Team - http://sipwise.com
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+import structlog
+from celery import shared_task
+
+from .utils import process_buildinfo
+
+logger = structlog.get_logger(__name__)
+
+
+@shared_task(bind=True)
+def parse_buildinfo(self, jbi_id: int, path: str):
+    process_buildinfo(jbi_id, path)

buildinfo/tests/test_conf.py

@@ -0,0 +1,27 @@
+# Copyright (C) 2022 The Sipwise Team - http://sipwise.com
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+from django.test import SimpleTestCase
+
+
+class TestBuildInfoConf(SimpleTestCase):
+    def test_django_settings(self):
+        from django.conf import settings
+
+        self.assertIsNotNone(settings.RELEASE_CHANGED_JOBS)
+
+    def test_repoapi_settings(self):
+        from buildinfo.conf import settings
+
+        self.assertIsNotNone(settings.RELEASE_CHANGED_JOBS)

buildinfo/tests/test_models.py

@@ -0,0 +1,40 @@
+# Copyright (C) 2022 The Sipwise Team - http://sipwise.com
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+from buildinfo import models
+from repoapi.test.base import BaseTest
+
+
+class TestBuildInfo(BaseTest):
+    def get_defaults(self, info=False):
+        defaults = {
+            "projectname": "fake",
+            "jobname": "upgrade-binaries",
+            "buildnumber": 1,
+            "param_tag": "none",
+            "param_branch": "mr4.5",
+            "param_release": "none",
+            "param_distribution": "wheezy",
+            "param_ppa": "gerrit_MT10339_review2054",
+            "builton": "fake-slave-1",
+            "timestamp": 0,
+            "duration": 0,
+        }
+        return defaults
+
+    def test_model(self):
+        param = self.get_defaults()
+        info = models.BuildInfo.objects.create(**param)
+        self.assertIsNotNone(info.pk)
+        self.assertEqual(str(info), "upgrade-binaries:1:mr4.5:none")

buildinfo/tests/test_utils.py

@@ -0,0 +1,86 @@
+# Copyright (C) 2022 The Sipwise Team - http://sipwise.com
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+from unittest.mock import mock_open
+from unittest.mock import patch
+
+from buildinfo import models
+from buildinfo import utils
+from repoapi.models import JenkinsBuildInfo
+from repoapi.test.base import BaseTest
+
+build_info = """{
+"building": false,
+"description": null,
+"displayName": "#9000",
+"duration": 55479,
+"estimatedDuration": 62270,
+"executor": null,
+"fullDisplayName": "upgrade-binaries #9000",
+"id": "9000",
+"keepLog": false,
+"number": 9000,
+"queueId": 4582301,
+"result": "SUCCESS",
+"timestamp": 1668083617940,
+"url": "https://jenkins-dev.mgm.sipwise.com/job/upgrade-binaries/9000/",
+"builtOn": "jenkins-slave13",
+"changeSet": {
+    "_class": "hudson.scm.EmptyChangeLogSet",
+    "items": [],
+    "kind": null
+},
+"culprits": []}"""
+
+
+class TestBuildInfo(BaseTest):
+    def get_defaults(self, info=False):
+        defaults = {
+            "projectname": "fake",
+            "jobname": "upgrade-binaries",
+            "buildnumber": 1,
+            "param_tag": "none",
+            "param_branch": "mr4.5",
+            "param_release": "none",
+            "param_distribution": "wheezy",
+            "param_ppa": "gerrit_MT10339_review2054",
+        }
+        jbi_defaults = {
+            "tag": "edc90cd9-37f3-4613-9748-ed05a32031c2",
+            "result": "SUCCESS",
+            "job_url": "https://jenkins.mgm.sipwise.com/job/upgrade-binaries/",
+            "git_commit_msg": "7fg4567 TT#0001 whatever",
+        }
+        if info:
+            defaults.update(
+                {
+                    "builton": "fake-slave-1",
+                    "timestamp": 0,
+                    "duration": 0,
+                }
+            )
+            return defaults
+        defaults.update(jbi_defaults)
+        return defaults
+
+    @patch("builtins.open", mock_open(read_data=build_info))
+    @patch("repoapi.utils.dlfile")
+    def test_process_buildinfo(self, dlfile):
+        qs = models.BuildInfo.objects.all()
+        self.assertEqual(qs.count(), 0)
+        param = self.get_defaults()
+        jbi = JenkinsBuildInfo.objects.create(**param)
+        utils.process_buildinfo(jbi.pk, "/tmp/fake.txt")
+        qs = models.BuildInfo.objects.filter(builton="jenkins-slave13")
+        self.assertEqual(qs.count(), 1)

buildinfo/utils.py

@@ -0,0 +1,43 @@
+# Copyright (C) 2017-2022 The Sipwise Team - http://sipwise.com
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+import json
+
+import structlog
+from django.apps import apps
+
+from .models import BuildInfo
+
+logger = structlog.get_logger(__name__)
+
+
+def process_buildinfo(jbi_id: int, path: str):
+    JenkinsBuildInfo = apps.get_model("repoapi", "JenkinsBuildInfo")
+    jbi = JenkinsBuildInfo.objects.get(pk=jbi_id)
+    with open(path, "r") as file:
+        info = json.load(file)
+    BuildInfo.objects.create(
+        builton=info["builtOn"],
+        timestamp=info["timestamp"],
+        duration=info["duration"],
+        projectname=jbi.projectname,
+        jobname=jbi.jobname,
+        buildnumber=jbi.buildnumber,
+        param_tag=jbi.param_tag,
+        param_branch=jbi.param_branch,
+        param_release=jbi.param_release,
+        param_release_uuid=jbi.param_release_uuid,
+        param_distribution=jbi.param_distribution,
+        param_ppa=jbi.param_ppa,
+    )

debian/repoapi.install

@@ -1,5 +1,6 @@
 Makefile usr/share/repoapi
 build usr/share/repoapi
+buildinfo usr/share/repoapi
 gerrit usr/share/repoapi
 hotfix usr/share/repoapi
 manage.py usr/share/repoapi

repoapi/celery.py

@@ -37,6 +37,11 @@ def jbi_parse_hotfix(jbi_id: str, path: str):
     app.send_task("hotfix.tasks.hotfix_released", args=[jbi_id, path])
 
 
+@app.task()
+def jbi_parse_buildinfo(jbi_id: str, path: str):
+    app.send_task("buildinfo.tasks.parse_buildinfo", args=[jbi_id, path])
+
+
 @app.task()
 def process_result(jbi_id: str, path_envVars: str):
     app.send_task(

repoapi/settings/common.py

@@ -44,6 +44,7 @@ INSTALLED_APPS = [
     "panel",
     "release_dashboard",
     "build",
+    "buildinfo",
     "release_changed",
     "repoapi",
 ]

repoapi/tasks.py

@@ -19,6 +19,7 @@ import structlog
 from celery import shared_task
 from django.apps import apps
 
+from .celery import jbi_parse_buildinfo
 from .celery import jbi_parse_hotfix
 from .celery import process_result
 from .conf import settings
@@ -67,6 +68,7 @@ def get_jbi_files(jbi_id, jobname, buildnumber):
     jenkins_get_console(jobname, buildnumber)
     path_envVars = jenkins_get_env(jobname, buildnumber)
     path_build = jenkins_get_build(jobname, buildnumber)
+    jbi_parse_buildinfo.delay(jbi_id, str(path_build))
     if is_download_artifacts(jobname):
         with open(path_build) as data_file:
             data = json.load(data_file)