diff --git a/buildinfo/__init__.py b/buildinfo/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/buildinfo/admin.py b/buildinfo/admin.py new file mode 100644 index 0000000..f35c595 --- /dev/null +++ b/buildinfo/admin.py @@ -0,0 +1,35 @@ +# Copyright (C) 2022 The Sipwise Team - http://sipwise.com +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +from django.contrib import admin +from import_export import resources +from import_export.admin import ImportExportModelAdmin + +from . import models + + +class BuildInfoResource(resources.ModelResource): + class Meta: + model = models.BuildInfo + + +@admin.register(models.BuildInfo) +class BuildInfoAdmin(ImportExportModelAdmin): + resource_class = BuildInfoResource + list_filter = ( + "param_release", + "projectname", + "param_distribution", + "builton", + ) diff --git a/buildinfo/apps.py b/buildinfo/apps.py new file mode 100644 index 0000000..7c724d0 --- /dev/null +++ b/buildinfo/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class BuildinfoConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "buildinfo" diff --git a/buildinfo/conf.py b/buildinfo/conf.py new file mode 100644 index 0000000..72f89d4 --- /dev/null +++ b/buildinfo/conf.py @@ -0,0 +1,21 @@ +# Copyright (C) 2022 The Sipwise Team - http://sipwise.com +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +from django.conf import settings # noqa +from appconf import AppConf + + +class BuildInfoConf(AppConf): + class Meta: + prefix = "buildinfo" diff --git a/buildinfo/migrations/0001_initial.py b/buildinfo/migrations/0001_initial.py new file mode 100644 index 0000000..572c9f8 --- /dev/null +++ b/buildinfo/migrations/0001_initial.py @@ -0,0 +1,57 @@ +# Generated by Django 3.2.15 on 2022-12-15 15:34 +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="BuildInfo", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("builton", models.CharField(max_length=50)), + ("timestamp", models.PositiveBigIntegerField()), + ("duration", models.PositiveSmallIntegerField()), + ("projectname", models.CharField(max_length=100)), + ("buildnumber", models.IntegerField()), + ("jobname", models.CharField(max_length=100)), + ( + "param_tag", + models.CharField(blank=True, max_length=50, null=True), + ), + ( + "param_branch", + models.CharField(blank=True, max_length=50, null=True), + ), + ( + "param_release", + models.CharField(db_index=True, max_length=50, null=True), + ), + ( + "param_release_uuid", + models.CharField(blank=True, max_length=64, null=True), + ), + ( + "param_distribution", + models.CharField(blank=True, max_length=50, null=True), + ), + ( + "param_ppa", + models.CharField(blank=True, max_length=50, null=True), + ), + ], + ), + ] diff --git a/buildinfo/migrations/0002_ldap_grp_perms.py b/buildinfo/migrations/0002_ldap_grp_perms.py new file mode 100644 index 0000000..6dc516c --- /dev/null +++ b/buildinfo/migrations/0002_ldap_grp_perms.py @@ -0,0 +1,46 @@ +# Generated by Django 3.2.13 on 2022-06-14 16:12 +from django.contrib.auth.management import create_permissions +from django.db import migrations + + +def add_permissions(apps, schema_editor): + """ContentType table is populated after all the migrations applied""" + for app_config in apps.get_app_configs(): + app_config.models_module = True + create_permissions(app_config, verbosity=0) + app_config.models_module = None + + +def forwards_func(apps, schema_editor): + add_permissions(apps, schema_editor) + Group = apps.get_model("auth", "Group") + Permission = apps.get_model("auth", "Permission") + ContentType = apps.get_model("contenttypes", "ContentType") + db_alias = schema_editor.connection.alias + + devops_grp = Group.objects.using(db_alias).get(name="devops") + + BuildInfo = apps.get_model("buildinfo", "BuildInfo") + ct = ContentType.objects.get_for_model(BuildInfo) + + perm_codenames = [] + for perm in ("add", "change", "delete", "view"): + perm_codenames.append(f"{perm}_buildinfo") + + for codename in perm_codenames: + perm = Permission.objects.using(db_alias).get( + content_type=ct, codename=codename + ) + devops_grp.permissions.add(perm) + + +class Migration(migrations.Migration): + + dependencies = [ + ("buildinfo", "0001_initial"), + ("repoapi", "0011_ldap_groups"), + ] + + operations = [ + migrations.RunPython(forwards_func), + ] diff --git a/buildinfo/migrations/__init__.py b/buildinfo/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/buildinfo/models.py b/buildinfo/models.py new file mode 100644 index 0000000..d4725e5 --- /dev/null +++ b/buildinfo/models.py @@ -0,0 +1,37 @@ +# Copyright (C) 2022 The Sipwise Team - http://sipwise.com +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +from django.db import models + + +class BuildInfo(models.Model): + builton = models.CharField(max_length=50, null=False) + timestamp = models.PositiveBigIntegerField(null=False) + duration = models.PositiveSmallIntegerField(null=False) + + projectname = models.CharField(max_length=100, null=False) + buildnumber = models.IntegerField() + jobname = models.CharField(max_length=100, null=False) + param_tag = models.CharField(max_length=50, null=True, blank=True) + param_branch = models.CharField(max_length=50, null=True, blank=True) + param_release = models.CharField(max_length=50, null=True, db_index=True) + param_release_uuid = models.CharField(max_length=64, null=True, blank=True) + param_distribution = models.CharField(max_length=50, null=True, blank=True) + param_ppa = models.CharField(max_length=50, null=True, blank=True) + + def __str__(self): + return ( + f"{self.jobname}:{self.buildnumber}:" + f"{self.param_branch}:{self.param_release}" + ) diff --git a/buildinfo/tasks.py b/buildinfo/tasks.py new file mode 100644 index 0000000..d937300 --- /dev/null +++ b/buildinfo/tasks.py @@ -0,0 +1,25 @@ +# Copyright (C) 2017-2022 The Sipwise Team - http://sipwise.com +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +import structlog +from celery import shared_task + +from .utils import process_buildinfo + +logger = structlog.get_logger(__name__) + + +@shared_task(bind=True) +def parse_buildinfo(self, jbi_id: int, path: str): + process_buildinfo(jbi_id, path) diff --git a/buildinfo/tests/__init__.py b/buildinfo/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/buildinfo/tests/test_conf.py b/buildinfo/tests/test_conf.py new file mode 100644 index 0000000..cfbc5d4 --- /dev/null +++ b/buildinfo/tests/test_conf.py @@ -0,0 +1,27 @@ +# Copyright (C) 2022 The Sipwise Team - http://sipwise.com +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +from django.test import SimpleTestCase + + +class TestBuildInfoConf(SimpleTestCase): + def test_django_settings(self): + from django.conf import settings + + self.assertIsNotNone(settings.RELEASE_CHANGED_JOBS) + + def test_repoapi_settings(self): + from buildinfo.conf import settings + + self.assertIsNotNone(settings.RELEASE_CHANGED_JOBS) diff --git a/buildinfo/tests/test_models.py b/buildinfo/tests/test_models.py new file mode 100644 index 0000000..23ed854 --- /dev/null +++ b/buildinfo/tests/test_models.py @@ -0,0 +1,40 @@ +# Copyright (C) 2022 The Sipwise Team - http://sipwise.com +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +from buildinfo import models +from repoapi.test.base import BaseTest + + +class TestBuildInfo(BaseTest): + def get_defaults(self, info=False): + defaults = { + "projectname": "fake", + "jobname": "upgrade-binaries", + "buildnumber": 1, + "param_tag": "none", + "param_branch": "mr4.5", + "param_release": "none", + "param_distribution": "wheezy", + "param_ppa": "gerrit_MT10339_review2054", + "builton": "fake-slave-1", + "timestamp": 0, + "duration": 0, + } + return defaults + + def test_model(self): + param = self.get_defaults() + info = models.BuildInfo.objects.create(**param) + self.assertIsNotNone(info.pk) + self.assertEqual(str(info), "upgrade-binaries:1:mr4.5:none") diff --git a/buildinfo/tests/test_utils.py b/buildinfo/tests/test_utils.py new file mode 100644 index 0000000..74c130d --- /dev/null +++ b/buildinfo/tests/test_utils.py @@ -0,0 +1,86 @@ +# Copyright (C) 2022 The Sipwise Team - http://sipwise.com +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +from unittest.mock import mock_open +from unittest.mock import patch + +from buildinfo import models +from buildinfo import utils +from repoapi.models import JenkinsBuildInfo +from repoapi.test.base import BaseTest + +build_info = """{ +"building": false, +"description": null, +"displayName": "#9000", +"duration": 55479, +"estimatedDuration": 62270, +"executor": null, +"fullDisplayName": "upgrade-binaries #9000", +"id": "9000", +"keepLog": false, +"number": 9000, +"queueId": 4582301, +"result": "SUCCESS", +"timestamp": 1668083617940, +"url": "https://jenkins-dev.mgm.sipwise.com/job/upgrade-binaries/9000/", +"builtOn": "jenkins-slave13", +"changeSet": { + "_class": "hudson.scm.EmptyChangeLogSet", + "items": [], + "kind": null +}, +"culprits": []}""" + + +class TestBuildInfo(BaseTest): + def get_defaults(self, info=False): + defaults = { + "projectname": "fake", + "jobname": "upgrade-binaries", + "buildnumber": 1, + "param_tag": "none", + "param_branch": "mr4.5", + "param_release": "none", + "param_distribution": "wheezy", + "param_ppa": "gerrit_MT10339_review2054", + } + jbi_defaults = { + "tag": "edc90cd9-37f3-4613-9748-ed05a32031c2", + "result": "SUCCESS", + "job_url": "https://jenkins.mgm.sipwise.com/job/upgrade-binaries/", + "git_commit_msg": "7fg4567 TT#0001 whatever", + } + if info: + defaults.update( + { + "builton": "fake-slave-1", + "timestamp": 0, + "duration": 0, + } + ) + return defaults + defaults.update(jbi_defaults) + return defaults + + @patch("builtins.open", mock_open(read_data=build_info)) + @patch("repoapi.utils.dlfile") + def test_process_buildinfo(self, dlfile): + qs = models.BuildInfo.objects.all() + self.assertEqual(qs.count(), 0) + param = self.get_defaults() + jbi = JenkinsBuildInfo.objects.create(**param) + utils.process_buildinfo(jbi.pk, "/tmp/fake.txt") + qs = models.BuildInfo.objects.filter(builton="jenkins-slave13") + self.assertEqual(qs.count(), 1) diff --git a/buildinfo/utils.py b/buildinfo/utils.py new file mode 100644 index 0000000..7a8380c --- /dev/null +++ b/buildinfo/utils.py @@ -0,0 +1,43 @@ +# Copyright (C) 2017-2022 The Sipwise Team - http://sipwise.com +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +import json + +import structlog +from django.apps import apps + +from .models import BuildInfo + +logger = structlog.get_logger(__name__) + + +def process_buildinfo(jbi_id: int, path: str): + JenkinsBuildInfo = apps.get_model("repoapi", "JenkinsBuildInfo") + jbi = JenkinsBuildInfo.objects.get(pk=jbi_id) + with open(path, "r") as file: + info = json.load(file) + BuildInfo.objects.create( + builton=info["builtOn"], + timestamp=info["timestamp"], + duration=info["duration"], + projectname=jbi.projectname, + jobname=jbi.jobname, + buildnumber=jbi.buildnumber, + param_tag=jbi.param_tag, + param_branch=jbi.param_branch, + param_release=jbi.param_release, + param_release_uuid=jbi.param_release_uuid, + param_distribution=jbi.param_distribution, + param_ppa=jbi.param_ppa, + ) diff --git a/debian/repoapi.install b/debian/repoapi.install index 3aba391..7422b17 100644 --- a/debian/repoapi.install +++ b/debian/repoapi.install @@ -1,5 +1,6 @@ Makefile usr/share/repoapi build usr/share/repoapi +buildinfo usr/share/repoapi gerrit usr/share/repoapi hotfix usr/share/repoapi manage.py usr/share/repoapi diff --git a/repoapi/celery.py b/repoapi/celery.py index 4edb39a..7fca54c 100644 --- a/repoapi/celery.py +++ b/repoapi/celery.py @@ -37,6 +37,11 @@ def jbi_parse_hotfix(jbi_id: str, path: str): app.send_task("hotfix.tasks.hotfix_released", args=[jbi_id, path]) +@app.task() +def jbi_parse_buildinfo(jbi_id: str, path: str): + app.send_task("buildinfo.tasks.parse_buildinfo", args=[jbi_id, path]) + + @app.task() def process_result(jbi_id: str, path_envVars: str): app.send_task( diff --git a/repoapi/settings/common.py b/repoapi/settings/common.py index bb1fd4b..0aa8faf 100644 --- a/repoapi/settings/common.py +++ b/repoapi/settings/common.py @@ -44,6 +44,7 @@ INSTALLED_APPS = [ "panel", "release_dashboard", "build", + "buildinfo", "release_changed", "repoapi", ] diff --git a/repoapi/tasks.py b/repoapi/tasks.py index 8b611c0..636d414 100644 --- a/repoapi/tasks.py +++ b/repoapi/tasks.py @@ -19,6 +19,7 @@ import structlog from celery import shared_task from django.apps import apps +from .celery import jbi_parse_buildinfo from .celery import jbi_parse_hotfix from .celery import process_result from .conf import settings @@ -67,6 +68,7 @@ def get_jbi_files(jbi_id, jobname, buildnumber): jenkins_get_console(jobname, buildnumber) path_envVars = jenkins_get_env(jobname, buildnumber) path_build = jenkins_get_build(jobname, buildnumber) + jbi_parse_buildinfo.delay(jbi_id, str(path_build)) if is_download_artifacts(jobname): with open(path_build) as data_file: data = json.load(data_file)