From 872af3822a8ea412bbcb1b06719d1a1f4d64730b Mon Sep 17 00:00:00 2001
From: Victor Seva <vseva@sipwise.com>
Date: Sat, 21 Mar 2020 19:01:18 +0100
Subject: [PATCH] TT#43813 build: split settings per application
Using https://django-appconf.readthedocs.io/en/latest/ helps us to
keep configs per application
Change-Id: I8b69f421bcafe5e93dd8029063496e0e32ef9595
---
.pre-commit-config.yaml | 6 +++++-
build/conf.py | 31 +++++++++++++++++++++++++++++++
build/models/__init__.py | 4 ++--
build/models/br.py | 13 ++++++++-----
build/tasks.py | 2 +-
build/test/test_conf.py | 27 +++++++++++++++++++++++++++
build/test/test_utils.py | 2 +-
build/utils.py | 10 ++++++----
panel/views.py | 2 +-
repoapi/models/jbi.py | 4 +++-
repoapi/settings/common.py | 8 --------
repoapi/settings/prod.py | 2 +-
repoapi/settings/test.py | 2 +-
requirements/common.txt | 1 +
t/Dockerfile | 2 +-
15 files changed, 89 insertions(+), 27 deletions(-)
create mode 100644 build/conf.py
create mode 100644 build/test/test_conf.py
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 39c3e77..f2cdb19 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -6,7 +6,11 @@ repos:
rev: v1.9.0
hooks:
- id: reorder-python-imports
- exclude: migrations/
+ exclude: >
+ (?x)^(
+ migrations/.*|
+ .*/conf.py
+ )$
args: ["--py37-plus"]
- repo: https://github.com/psf/black
rev: stable
diff --git a/build/conf.py b/build/conf.py
new file mode 100644
index 0000000..a0f4637
--- /dev/null
+++ b/build/conf.py
@@ -0,0 +1,31 @@
+# Copyright (C) 2020 The Sipwise Team - http://sipwise.com
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+from django.conf import settings # noqa
+from appconf import AppConf
+
+
+class BuildConf(AppConf):
+ KEY_AUTH = True
+ REPOS_SCRIPTS_CONFIG_DIR = "/usr/share/sipwise-repos-scripts/config"
+ POOL = 3
+ # sipwise-repos-scripts config files we don't want
+ RELEASES_SKIP = (
+ "internal",
+ "mr0.1",
+ )
+ RELEASE_JOBS = ("release-copy-debs-yml",)
+
+ class Meta:
+ prefix = "build"
diff --git a/build/models/__init__.py b/build/models/__init__.py
index 8c7afd1..b3b449d 100644
--- a/build/models/__init__.py
+++ b/build/models/__init__.py
@@ -14,9 +14,9 @@
# with this program. If not, see <http://www.gnu.org/licenses/>.
import logging
-from django.conf import settings
from django.db.models import signals
+from ..conf import settings
from .br import BuildRelease
from build.tasks import build_release
from build.tasks import build_resume
@@ -41,7 +41,7 @@ def jbi_manage(sender, **kwargs):
if jbi.param_release_uuid is None:
return
release = jbi.param_release
- if jbi.jobname in settings.RELEASE_JOBS:
+ if jbi.jobname in settings.BUILD_RELEASE_JOBS:
if not release.startswith("release-"):
release = "release-{}".format(jbi.param_release)
if jbi.param_release == "none":
diff --git a/build/models/br.py b/build/models/br.py
index 18cd0e0..cfe2f0a 100644
--- a/build/models/br.py
+++ b/build/models/br.py
@@ -14,10 +14,10 @@
# with this program. If not, see <http://www.gnu.org/licenses/>.
import logging
-from django.conf import settings
from django.db import models
from django.forms.models import model_to_dict
+from ..conf import settings
from build.utils import get_simple_release
from build.utils import ReleaseConfig
from repoapi.models import JenkinsBuildInfo
@@ -46,7 +46,8 @@ class BuildReleaseManager(models.Manager):
def release_jobs(self, release_uuid, flat=True):
qs = self._jbi.get_queryset()
res = qs.filter(
- jobname__in=settings.RELEASE_JOBS, param_release_uuid=release_uuid,
+ jobname__in=settings.BUILD_RELEASE_JOBS,
+ param_release_uuid=release_uuid,
).distinct()
if res.exists():
if flat:
@@ -96,7 +97,7 @@ class BuildRelease(models.Model):
failed_projects = models.TextField(null=True, editable=False)
pool_size = models.SmallIntegerField(default=0, editable=False)
objects = BuildReleaseManager()
- release_jobs_len = len(",".join(settings.RELEASE_JOBS))
+ release_jobs_len = len(",".join(settings.BUILD_RELEASE_JOBS))
def __str__(self):
return "%s[%s]" % (self.release, self.uuid)
@@ -207,7 +208,9 @@ class BuildRelease(models.Model):
if jobname.endswith("-piuparts"):
return False
return self._append_falied(jbi.projectname)
- if jobname.endswith("-repos") or jobname in settings.RELEASE_JOBS:
+ is_repos = jobname.endswith("-repos")
+ is_rj = jobname in settings.BUILD_RELEASE_JOBS
+ if is_repos or is_rj:
if jbi.result in ["SUCCESS", "UNSTABLE"]:
return self._append_built(jbi.projectname)
return False
@@ -236,7 +239,7 @@ class BuildRelease(models.Model):
@property
def next(self):
failed_projects = self.failed_projects_list
- if any(job in failed_projects for job in settings.RELEASE_JOBS):
+ if any(job in failed_projects for job in settings.BUILD_RELEASE_JOBS):
logger.info("release has failed release_jobs, stop sending jobs")
return
res = self._next()
diff --git a/build/tasks.py b/build/tasks.py
index aba2b70..74d662b 100644
--- a/build/tasks.py
+++ b/build/tasks.py
@@ -15,8 +15,8 @@
import logging
from celery import shared_task
-from django.conf import settings
+from .conf import settings
from build.models.br import BuildRelease
from build.utils import trigger_build
from build.utils import trigger_copy_deps
diff --git a/build/test/test_conf.py b/build/test/test_conf.py
new file mode 100644
index 0000000..5ed3649
--- /dev/null
+++ b/build/test/test_conf.py
@@ -0,0 +1,27 @@
+# Copyright (C) 2015 The Sipwise Team - http://sipwise.com
+#
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+from django.test import TestCase
+
+
+class TestBuildConf(TestCase):
+ def test_django_settings(self):
+ from django.conf import settings
+
+ self.assertEqual(settings.BUILD_KEY_AUTH, True)
+
+ def test_build_settings(self):
+ from build.conf import settings
+
+ self.assertEqual(settings.BUILD_KEY_AUTH, True)
diff --git a/build/test/test_utils.py b/build/test/test_utils.py
index bb0a131..853e9f9 100644
--- a/build/test/test_utils.py
+++ b/build/test/test_utils.py
@@ -75,7 +75,7 @@ class ReleaseConfigTestCase(SimpleTestCase):
"check-tools",
]
- @override_settings(RELEASES_SKIP=["mr0.1"])
+ @override_settings(BUILD_RELEASES_SKIP=["mr0.1"])
def test_supported_releases(self):
supported = [
"trunk",
diff --git a/build/utils.py b/build/utils.py
index 8894022..29481ac 100644
--- a/build/utils.py
+++ b/build/utils.py
@@ -19,11 +19,11 @@ import urllib
from pathlib import Path
from uuid import uuid4
-from django.conf import settings
from yaml import load
from yaml import Loader
from . import exceptions as err
+from .conf import settings
from repoapi.utils import openurl
logger = logging.getLogger(__name__)
@@ -157,9 +157,11 @@ class ReleaseConfig(object):
@classmethod
def supported_releases(cls):
- skip_files = ["{}.yml".format(x) for x in settings.RELEASES_SKIP]
+ skip_files = ["{}.yml".format(x) for x in settings.BUILD_RELEASES_SKIP]
res = []
- for root, dirs, files in os.walk(settings.REPOS_SCRIPTS_CONFIG_DIR):
+ for root, dirs, files in os.walk(
+ settings.BUILD_REPOS_SCRIPTS_CONFIG_DIR
+ ):
for name in files:
path_name = Path(name)
if path_name.suffix != ".yml":
@@ -183,7 +185,7 @@ class ReleaseConfig(object):
filename = name
self.config_file = "{}.yml".format(filename)
self.config_path = os.path.join(
- settings.REPOS_SCRIPTS_CONFIG_DIR, self.config_file
+ settings.BUILD_REPOS_SCRIPTS_CONFIG_DIR, self.config_file
)
try:
with open(self.config_path) as f:
diff --git a/panel/views.py b/panel/views.py
index 248ccb2..1057d68 100644
--- a/panel/views.py
+++ b/panel/views.py
@@ -37,7 +37,7 @@ def release_uuid(request, _uuid):
"stats_queued": True,
"build_release": br,
"projects": projects,
- "release_jobs_size": len(settings.RELEASE_JOBS),
+ "release_jobs_size": len(settings.BUILD_RELEASE_JOBS),
"release_jobs": release_jobs,
"total": len(br.projects_list),
"queued_projects": queued_projects,
diff --git a/repoapi/models/jbi.py b/repoapi/models/jbi.py
index 48bfb70..5387d53 100644
--- a/repoapi/models/jbi.py
+++ b/repoapi/models/jbi.py
@@ -57,7 +57,9 @@ class JenkinsBuildInfoManager(models.Manager):
return res
def releases(self, flat=True):
- qs = self.get_queryset().exclude(jobname__in=settings.RELEASE_JOBS)
+ qs = self.get_queryset().exclude(
+ jobname__in=settings.BUILD_RELEASE_JOBS
+ )
res = qs.filter(tag__isnull=False).values("param_release").distinct()
if res.exists():
if flat:
diff --git a/repoapi/settings/common.py b/repoapi/settings/common.py
index 066cf9c..2ac8af9 100644
--- a/repoapi/settings/common.py
+++ b/repoapi/settings/common.py
@@ -145,14 +145,6 @@ DEBIAN_RELEASES = (
"wheezy",
"squeeze",
)
-# sipwise-repos-scripts config files we don't want
-RELEASES_SKIP = [
- "internal",
- "mr0.1",
-]
-RELEASE_JOBS = [
- "release-copy-debs-yml",
-]
RELEASE_DASHBOARD_SETTINGS = {
"debian_supported": DEBIAN_RELEASES,
"build_deps": (
diff --git a/repoapi/settings/prod.py b/repoapi/settings/prod.py
index 8375016..c930711 100644
--- a/repoapi/settings/prod.py
+++ b/repoapi/settings/prod.py
@@ -90,7 +90,7 @@ WORKFRONT_NOTE = True
# build app
BUILD_KEY_AUTH = True
-REPOS_SCRIPTS_CONFIG_DIR = "/usr/share/sipwise-repos-scripts/config"
+BUILD_REPOS_SCRIPTS_CONFIG_DIR = "/usr/share/sipwise-repos-scripts/config"
# celery
CELERY_BROKER_URL = server_config.get("server", "BROKER_URL")
diff --git a/repoapi/settings/test.py b/repoapi/settings/test.py
index b8a3af8..3b7a68b 100644
--- a/repoapi/settings/test.py
+++ b/repoapi/settings/test.py
@@ -93,7 +93,7 @@ DOCKER_IMAGES = {
# build app
BUILD_KEY_AUTH = True
-REPOS_SCRIPTS_CONFIG_DIR = join(BASE_DIR, "build", "fixtures", "config")
+BUILD_REPOS_SCRIPTS_CONFIG_DIR = join(BASE_DIR, "build", "fixtures", "config")
# celery
CELERY_BROKER_URL = "memory://localhost/"
diff --git a/requirements/common.txt b/requirements/common.txt
index f0373fc..3b71c09 100644
--- a/requirements/common.txt
+++ b/requirements/common.txt
@@ -1,5 +1,6 @@
celery
Django==1.11.28
+django-appconf
django-assets
django-celery-beat<2.0
django-celery-results
diff --git a/t/Dockerfile b/t/Dockerfile
index 99d46b1..493a142 100644
--- a/t/Dockerfile
+++ b/t/Dockerfile
@@ -5,7 +5,7 @@ FROM docker.mgm.sipwise.com/sipwise-buster:latest
# is updated with the current date. It will force refresh of all
# of the base images and things like `apt-get update` won't be using
# old cached versions when the Dockerfile is built.
-ENV REFRESHED_AT 2020-03-20
+ENV REFRESHED_AT 2020-03-21
# test execution; we need the backport of python3-junitxml from our own
# repository since it's not part of Debian/buster