TT#43813 build: split settings per application

Using https://django-appconf.readthedocs.io/en/latest/ helps us to
keep configs per application

Change-Id: I8b69f421bcafe5e93dd8029063496e0e32ef9595

.pre-commit-config.yaml

@@ -6,7 +6,11 @@ repos:
     rev: v1.9.0
     hooks:
     -   id: reorder-python-imports
-        exclude: migrations/
+        exclude: >
+            (?x)^(
+                migrations/.*|
+                .*/conf.py
+            )$
         args: ["--py37-plus"]
 -   repo: https://github.com/psf/black
     rev: stable

build/conf.py

@@ -0,0 +1,31 @@
+# Copyright (C) 2020 The Sipwise Team - http://sipwise.com
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+from django.conf import settings  # noqa
+from appconf import AppConf
+
+
+class BuildConf(AppConf):
+    KEY_AUTH = True
+    REPOS_SCRIPTS_CONFIG_DIR = "/usr/share/sipwise-repos-scripts/config"
+    POOL = 3
+    # sipwise-repos-scripts config files we don't want
+    RELEASES_SKIP = (
+        "internal",
+        "mr0.1",
+    )
+    RELEASE_JOBS = ("release-copy-debs-yml",)
+
+    class Meta:
+        prefix = "build"

build/models/__init__.py

@@ -14,9 +14,9 @@
 # with this program.  If not, see <http://www.gnu.org/licenses/>.
 import logging
 
-from django.conf import settings
 from django.db.models import signals
 
+from ..conf import settings
 from .br import BuildRelease
 from build.tasks import build_release
 from build.tasks import build_resume
@@ -41,7 +41,7 @@ def jbi_manage(sender, **kwargs):
     if jbi.param_release_uuid is None:
         return
     release = jbi.param_release
-    if jbi.jobname in settings.RELEASE_JOBS:
+    if jbi.jobname in settings.BUILD_RELEASE_JOBS:
         if not release.startswith("release-"):
             release = "release-{}".format(jbi.param_release)
     if jbi.param_release == "none":

build/models/br.py

@@ -14,10 +14,10 @@
 # with this program.  If not, see <http://www.gnu.org/licenses/>.
 import logging
 
-from django.conf import settings
 from django.db import models
 from django.forms.models import model_to_dict
 
+from ..conf import settings
 from build.utils import get_simple_release
 from build.utils import ReleaseConfig
 from repoapi.models import JenkinsBuildInfo
@@ -46,7 +46,8 @@ class BuildReleaseManager(models.Manager):
     def release_jobs(self, release_uuid, flat=True):
         qs = self._jbi.get_queryset()
         res = qs.filter(
-            jobname__in=settings.RELEASE_JOBS, param_release_uuid=release_uuid,
+            jobname__in=settings.BUILD_RELEASE_JOBS,
+            param_release_uuid=release_uuid,
         ).distinct()
         if res.exists():
             if flat:
@@ -96,7 +97,7 @@ class BuildRelease(models.Model):
     failed_projects = models.TextField(null=True, editable=False)
     pool_size = models.SmallIntegerField(default=0, editable=False)
     objects = BuildReleaseManager()
-    release_jobs_len = len(",".join(settings.RELEASE_JOBS))
+    release_jobs_len = len(",".join(settings.BUILD_RELEASE_JOBS))
 
     def __str__(self):
         return "%s[%s]" % (self.release, self.uuid)
@@ -207,7 +208,9 @@ class BuildRelease(models.Model):
             if jobname.endswith("-piuparts"):
                 return False
             return self._append_falied(jbi.projectname)
-        if jobname.endswith("-repos") or jobname in settings.RELEASE_JOBS:
+        is_repos = jobname.endswith("-repos")
+        is_rj = jobname in settings.BUILD_RELEASE_JOBS
+        if is_repos or is_rj:
             if jbi.result in ["SUCCESS", "UNSTABLE"]:
                 return self._append_built(jbi.projectname)
         return False
@@ -236,7 +239,7 @@ class BuildRelease(models.Model):
     @property
     def next(self):
         failed_projects = self.failed_projects_list
-        if any(job in failed_projects for job in settings.RELEASE_JOBS):
+        if any(job in failed_projects for job in settings.BUILD_RELEASE_JOBS):
             logger.info("release has failed release_jobs, stop sending jobs")
             return
         res = self._next()

build/tasks.py

@@ -15,8 +15,8 @@
 import logging
 
 from celery import shared_task
-from django.conf import settings
 
+from .conf import settings
 from build.models.br import BuildRelease
 from build.utils import trigger_build
 from build.utils import trigger_copy_deps

build/test/test_conf.py

@@ -0,0 +1,27 @@
+# Copyright (C) 2015 The Sipwise Team - http://sipwise.com
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+from django.test import TestCase
+
+
+class TestBuildConf(TestCase):
+    def test_django_settings(self):
+        from django.conf import settings
+
+        self.assertEqual(settings.BUILD_KEY_AUTH, True)
+
+    def test_build_settings(self):
+        from build.conf import settings
+
+        self.assertEqual(settings.BUILD_KEY_AUTH, True)

build/test/test_utils.py

@@ -75,7 +75,7 @@ class ReleaseConfigTestCase(SimpleTestCase):
         "check-tools",
     ]
 
-    @override_settings(RELEASES_SKIP=["mr0.1"])
+    @override_settings(BUILD_RELEASES_SKIP=["mr0.1"])
     def test_supported_releases(self):
         supported = [
             "trunk",

build/utils.py

@@ -19,11 +19,11 @@ import urllib
 from pathlib import Path
 from uuid import uuid4
 
-from django.conf import settings
 from yaml import load
 from yaml import Loader
 
 from . import exceptions as err
+from .conf import settings
 from repoapi.utils import openurl
 
 logger = logging.getLogger(__name__)
@@ -157,9 +157,11 @@ class ReleaseConfig(object):
 
     @classmethod
     def supported_releases(cls):
-        skip_files = ["{}.yml".format(x) for x in settings.RELEASES_SKIP]
+        skip_files = ["{}.yml".format(x) for x in settings.BUILD_RELEASES_SKIP]
         res = []
-        for root, dirs, files in os.walk(settings.REPOS_SCRIPTS_CONFIG_DIR):
+        for root, dirs, files in os.walk(
+            settings.BUILD_REPOS_SCRIPTS_CONFIG_DIR
+        ):
             for name in files:
                 path_name = Path(name)
                 if path_name.suffix != ".yml":
@@ -183,7 +185,7 @@ class ReleaseConfig(object):
             filename = name
         self.config_file = "{}.yml".format(filename)
         self.config_path = os.path.join(
-            settings.REPOS_SCRIPTS_CONFIG_DIR, self.config_file
+            settings.BUILD_REPOS_SCRIPTS_CONFIG_DIR, self.config_file
         )
         try:
             with open(self.config_path) as f:

panel/views.py

@@ -37,7 +37,7 @@ def release_uuid(request, _uuid):
         "stats_queued": True,
         "build_release": br,
         "projects": projects,
-        "release_jobs_size": len(settings.RELEASE_JOBS),
+        "release_jobs_size": len(settings.BUILD_RELEASE_JOBS),
         "release_jobs": release_jobs,
         "total": len(br.projects_list),
         "queued_projects": queued_projects,

repoapi/models/jbi.py

@@ -57,7 +57,9 @@ class JenkinsBuildInfoManager(models.Manager):
         return res
 
     def releases(self, flat=True):
-        qs = self.get_queryset().exclude(jobname__in=settings.RELEASE_JOBS)
+        qs = self.get_queryset().exclude(
+            jobname__in=settings.BUILD_RELEASE_JOBS
+        )
         res = qs.filter(tag__isnull=False).values("param_release").distinct()
         if res.exists():
             if flat:

repoapi/settings/common.py

@@ -145,14 +145,6 @@ DEBIAN_RELEASES = (
     "wheezy",
     "squeeze",
 )
-# sipwise-repos-scripts config files we don't want
-RELEASES_SKIP = [
-    "internal",
-    "mr0.1",
-]
-RELEASE_JOBS = [
-    "release-copy-debs-yml",
-]
 RELEASE_DASHBOARD_SETTINGS = {
     "debian_supported": DEBIAN_RELEASES,
     "build_deps": (

repoapi/settings/prod.py

@@ -90,7 +90,7 @@ WORKFRONT_NOTE = True
 
 # build app
 BUILD_KEY_AUTH = True
-REPOS_SCRIPTS_CONFIG_DIR = "/usr/share/sipwise-repos-scripts/config"
+BUILD_REPOS_SCRIPTS_CONFIG_DIR = "/usr/share/sipwise-repos-scripts/config"
 
 # celery
 CELERY_BROKER_URL = server_config.get("server", "BROKER_URL")

repoapi/settings/test.py

@@ -93,7 +93,7 @@ DOCKER_IMAGES = {
 
 # build app
 BUILD_KEY_AUTH = True
-REPOS_SCRIPTS_CONFIG_DIR = join(BASE_DIR, "build", "fixtures", "config")
+BUILD_REPOS_SCRIPTS_CONFIG_DIR = join(BASE_DIR, "build", "fixtures", "config")
 
 # celery
 CELERY_BROKER_URL = "memory://localhost/"

requirements/common.txt

@@ -1,5 +1,6 @@
 celery
 Django==1.11.28
+django-appconf
 django-assets
 django-celery-beat<2.0
 django-celery-results

t/Dockerfile

@@ -5,7 +5,7 @@ FROM docker.mgm.sipwise.com/sipwise-buster:latest
 # is updated with the current date. It will force refresh of all
 # of the base images and things like `apt-get update` won't be using
 # old cached versions when the Dockerfile is built.
-ENV REFRESHED_AT 2020-03-20
+ENV REFRESHED_AT 2020-03-21
 
 # test execution; we need the backport of python3-junitxml from our own
 # repository since it's not part of Debian/buster