sipwise
/
repoapi
mirror of https://github.com/sipwise/repoapi.git
1
0
Fork 0
Code Issues Releases Wiki Activity

TT#43813 upgrade celery settings for 4.4

Browse Source 
> https://docs.celeryproject.org/en/latest/django/first-steps-with-django.html

Change-Id: If66605b4675a6896f9bd5c387d300255ea7d368a
changes/14/38714/1
Victor Seva 6 years ago
parent 44b0cdc640
commit 9b591bf435
No known key found for this signature in database
GPG Key ID: B1589889727198E0
9 changed files with 103 additions and 104 deletions
Show Stats Download Patch File Download Diff File

5
hotfix/test/test_hotfix_released.py
Unescape View File

@ -1,15 +1,17 @@
# Copyright (C) 2015 The Sipwise Team - http://sipwise.com # Copyright (C) 2015 The Sipwise Team - http://sipwise.com
#
# This program is free software: you can redistribute it and/or modify it # This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free # under the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) # Software Foundation, either version 3 of the License, or (at your option)
# any later version. # any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT # This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details. # more details.
#
# You should have received a copy of the GNU General Public License along # You should have received a copy of the GNU General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>. # with this program. If not, see <http://www.gnu.org/licenses/>.
from django.test import override_settings
from mock import call from mock import call
from mock import mock_open from mock import mock_open
from mock import patch from mock import patch
@ -34,7 +36,6 @@ debian_changelog = """ngcp-fake (3.8.7.4+0~mr3.8.7.4) unstable; urgency=medium
""" """
@override_settings(CELERY_EAGER_PROPAGATES_EXCEPTIONS=True)
class TestHotfixReleased(BaseTest): class TestHotfixReleased(BaseTest):
def get_defaults(self): def get_defaults(self):
defaults = { defaults = {

14
release_dashboard/test/test_tasks_build.py
Unescape View File

@ -1,27 +1,25 @@
# Copyright (C) 2017 The Sipwise Team - http://sipwise.com # Copyright (C) 2017 The Sipwise Team - http://sipwise.com
#
# This program is free software: you can redistribute it and/or modify it # This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free # under the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) # Software Foundation, either version 3 of the License, or (at your option)
# any later version. # any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT # This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details. # more details.
#
# You should have received a copy of the GNU General Public License along # You should have received a copy of the GNU General Public License along
# with this prograproj. If not, see <http://www.gnu.org/licenses/>. # with this prograproj. If not, see <http://www.gnu.org/licenses/>.
from django.test import TestCase
from mock import patch
from django.test import TestCase, override_settings
from release_dashboard import tasks from release_dashboard import tasks
from mock import patch
@override_settings(CELERY_EAGER_PROPAGATES_EXCEPTIONS=True)
class TasksBuildTestCase(TestCase): class TasksBuildTestCase(TestCase):
@patch("release_dashboard.tasks.gerrit_fetch_info")
@patch('release_dashboard.tasks.gerrit_fetch_info')
def test_gerrit_fetch_all(self, gfi): def test_gerrit_fetch_all(self, gfi):
result = tasks.gerrit_fetch_all.delay() result = tasks.gerrit_fetch_all.delay()
self.assertTrue(result.successful()) self.assertTrue(result.successful())

1
release_dashboard/test/test_tasks_docker.py
Unescape View File

@ -63,7 +63,6 @@ def fake_manifest(url):
return ("{}", uuid.uuid4()) return ("{}", uuid.uuid4())
@override_settings(CELERY_EAGER_PROPAGATES_EXCEPTIONS=True)
@override_settings(DOCKER_REGISTRY_URL="{}") @override_settings(DOCKER_REGISTRY_URL="{}")
@override_settings(DEBUG=False) @override_settings(DEBUG=False)
class TasksDockerTestCase(TestCase): class TasksDockerTestCase(TestCase):

11
repoapi/__init__.py
Unescape View File

@ -1,19 +1,20 @@
# Copyright (C) 2016 The Sipwise Team - http://sipwise.com # Copyright (C) 2016 The Sipwise Team - http://sipwise.com
#
# This program is free software: you can redistribute it and/or modify it # This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free # under the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) # Software Foundation, either version 3 of the License, or (at your option)
# any later version. # any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT # This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details. # more details.
#
# You should have received a copy of the GNU General Public License along # You should have received a copy of the GNU General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>. # with this program. If not, see <http://www.gnu.org/licenses/>.
from __future__ import absolute_import from .celery import app as celery_app
# This will make sure the app is always imported when # This will make sure the app is always imported when
# Django starts so that shared_task will use this app. # Django starts so that shared_task will use this app.
from .celery import app as celery_app # noqa
__all__ = ("celery_app",)

29
repoapi/celery.py
Unescape View File

@ -1,35 +1,36 @@
# Copyright (C) 2016 The Sipwise Team - http://sipwise.com # Copyright (C) 2016 The Sipwise Team - http://sipwise.com
#
# This program is free software: you can redistribute it and/or modify it # This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free # under the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) # Software Foundation, either version 3 of the License, or (at your option)
# any later version. # any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT # This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details. # more details.
#
# You should have received a copy of the GNU General Public License along # You should have received a copy of the GNU General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>. # with this program. If not, see <http://www.gnu.org/licenses/>.
from __future__ import absolute_import
import os import os
from celery import Celery from celery import Celery
# set the default Django settings module for the 'celery' program. # set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'repoapi.settings.prod') os.environ.setdefault("DJANGO_SETTINGS_MODULE", "repoapi.settings.prod")
# pylint: disable=C0413
from django.conf import settings # noqa app = Celery("repoapi")
app = Celery('repoapi') # Using a string here means the worker doesn't have to serialize
# the configuration object to child processes.
# - namespace='CELERY' means all celery-related configuration keys
# should have a `CELERY_` prefix.
app.config_from_object("django.conf:settings", namespace="CELERY")
# Using a string here means the worker will not have to # Load task modules from all registered Django app configs.
# pickle the object when using Windows. app.autodiscover_tasks()
app.config_from_object('django.conf:settings')
app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
@app.task() @app.task()
def jbi_parse_hotfix(jbi_id, path): def jbi_parse_hotfix(jbi_id, path):
app.send_task('hotfix.tasks.hotfix_released', args=[jbi_id, path]) app.send_task("hotfix.tasks.hotfix_released", args=[jbi_id, path])

4
repoapi/settings/dev.py
Unescape View File

@ -27,8 +27,8 @@ LOGGING["loggers"]["repoapi"]["level"] = os.getenv( # noqa
# celery # celery
BROKER_BACKEND = "amqp" BROKER_BACKEND = "amqp"
CELERY_ALWAYS_EAGER = False CELERY_TASK_ALWAYS_EAGER = False
BROKER_URL = "amqp://guest:guest@rabbit" CELERY_BROKER_URL = "amqp://guest:guest@rabbit"
JBI_BASEDIR = os.path.join(BASE_DIR, "jbi_files") # noqa JBI_BASEDIR = os.path.join(BASE_DIR, "jbi_files") # noqa
# Enable access when not accessing from localhost: # Enable access when not accessing from localhost:

4
repoapi/settings/prod.py
Unescape View File

@ -93,8 +93,8 @@ BUILD_KEY_AUTH = True
REPOS_SCRIPTS_CONFIG_DIR = "/usr/share/sipwise-repos-scripts/config" REPOS_SCRIPTS_CONFIG_DIR = "/usr/share/sipwise-repos-scripts/config"
# celery # celery
BROKER_URL = server_config.get("server", "BROKER_URL") CELERY_BROKER_URL = server_config.get("server", "BROKER_URL")
CELERYBEAT_SCHEDULE = { CELERY_BEAT_SCHEDULE = {
# Executes every Sunday morning at 7:30 A.M # Executes every Sunday morning at 7:30 A.M
"purge-trunk": { "purge-trunk": {
"task": "repoapi.tasks.jbi_purge", "task": "repoapi.tasks.jbi_purge",

6
repoapi/settings/test.py
Unescape View File

@ -96,9 +96,9 @@ BUILD_KEY_AUTH = True
REPOS_SCRIPTS_CONFIG_DIR = join(BASE_DIR, "build", "fixtures", "config") REPOS_SCRIPTS_CONFIG_DIR = join(BASE_DIR, "build", "fixtures", "config")
# celery # celery
BROKER_BACKEND = "memory" CELERY_BROKER_URL = "memory://localhost/"
CELERY_ALWAYS_EAGER = True CELERY_TASK_ALWAYS_EAGER = True
CELERY_EAGER_PROPAGATES_EXCEPTIONS = True CELERY_TASK_EAGER_PROPAGATES = True
JBI_BASEDIR = join(RESULTS_DIR, "jbi_files") JBI_BASEDIR = join(RESULTS_DIR, "jbi_files")
JBI_ARTIFACT_JOBS = [ JBI_ARTIFACT_JOBS = [
"fake-release-tools-runner", "fake-release-tools-runner",

133
repoapi/test/test_jbi_info.py
Unescape View File

@ -1,27 +1,30 @@
# Copyright (C) 2015 The Sipwise Team - http://sipwise.com # Copyright (C) 2015 The Sipwise Team - http://sipwise.com
#
# This program is free software: you can redistribute it and/or modify it # This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free # under the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option) # Software Foundation, either version 3 of the License, or (at your option)
# any later version. # any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT # This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details. # more details.
#
# You should have received a copy of the GNU General Public License along # You should have received a copy of the GNU General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>. # with this program. If not, see <http://www.gnu.org/licenses/>.
import os import os
from django.test import override_settings
from django.conf import settings from django.conf import settings
from mock import call
from mock import mock_open
from mock import patch
from repoapi.models import JenkinsBuildInfo from repoapi.models import JenkinsBuildInfo
from repoapi.utils import JBI_CONSOLE_URL, JBI_BUILD_URL, JBI_ARTIFACT_URL
from repoapi.utils import JBI_ENVVARS_URL
from mock import patch, call, mock_open
from repoapi.test.base import BaseTest from repoapi.test.base import BaseTest
from repoapi.utils import JBI_ARTIFACT_URL
from repoapi.utils import JBI_BUILD_URL
from repoapi.utils import JBI_CONSOLE_URL
from repoapi.utils import JBI_ENVVARS_URL
artifacts_json = """{ artifacts_json = """{
"artifacts": [ "artifacts": [
@ -34,116 +37,112 @@ artifacts_json = """{
}""" }"""
@override_settings(CELERY_EAGER_PROPAGATES_EXCEPTIONS=True)
class TestJBICelery(BaseTest): class TestJBICelery(BaseTest):
def get_defaults(self): def get_defaults(self):
defaults = { defaults = {
'tag': "edc90cd9-37f3-4613-9748-ed05a32031c2", "tag": "edc90cd9-37f3-4613-9748-ed05a32031c2",
'projectname': "real-fake", "projectname": "real-fake",
'jobname': "real-fake-gerrit", "jobname": "real-fake-gerrit",
'buildnumber': 1, "buildnumber": 1,
'result': "SUCCESS", "result": "SUCCESS",
'job_url': "job_url": "https://jenkins-dev.mgm.sipwise.com/job"
"https://jenkins-dev.mgm.sipwise.com/job" "/real-fake-gerrit/",
"/real-fake-gerrit/", "param_tag": "none",
'param_tag': "none", "param_branch": "master",
'param_branch': "master", "param_release": "none",
'param_release': "none", "param_distribution": "wheezy",
'param_distribution': "wheezy", "param_ppa": "gerrit_MT10339_review2054",
'param_ppa': "gerrit_MT10339_review2054", "git_commit_msg": "7fg4567 TT#0001 whatever",
'git_commit_msg': "7fg4567 TT#0001 whatever",
} }
return defaults return defaults
@patch('builtins.open', mock_open(read_data=artifacts_json)) @patch("builtins.open", mock_open(read_data=artifacts_json))
@patch('repoapi.utils.dlfile') @patch("repoapi.utils.dlfile")
def test_jbi_path_creation(self, dlfile): def test_jbi_path_creation(self, dlfile):
param = self.get_defaults() param = self.get_defaults()
param['jobname'] = 'fake-me' param["jobname"] = "fake-me"
jbi = JenkinsBuildInfo.objects.create(**param) jbi = JenkinsBuildInfo.objects.create(**param)
base_path = os.path.join(settings.JBI_BASEDIR, base_path = os.path.join(
jbi.jobname, str(jbi.buildnumber)) settings.JBI_BASEDIR, jbi.jobname, str(jbi.buildnumber)
)
self.assertTrue(os.path.isdir(base_path), base_path) self.assertTrue(os.path.isdir(base_path), base_path)
@patch('builtins.open', mock_open(read_data=artifacts_json)) @patch("builtins.open", mock_open(read_data=artifacts_json))
@patch('repoapi.utils.dlfile') @patch("repoapi.utils.dlfile")
def test_jbi_console(self, dlfile): def test_jbi_console(self, dlfile):
param = self.get_defaults() param = self.get_defaults()
jbi = JenkinsBuildInfo.objects.create(**param) jbi = JenkinsBuildInfo.objects.create(**param)
base_path = os.path.join(settings.JBI_BASEDIR, base_path = os.path.join(
jbi.jobname, str(jbi.buildnumber)) settings.JBI_BASEDIR, jbi.jobname, str(jbi.buildnumber)
)
path = os.path.join(base_path, 'console.txt') path = os.path.join(base_path, "console.txt")
url = JBI_CONSOLE_URL.format( url = JBI_CONSOLE_URL.format(
settings.JENKINS_URL, settings.JENKINS_URL, jbi.jobname, jbi.buildnumber
jbi.jobname,
jbi.buildnumber
) )
dlfile.assert_any_call(url, path) dlfile.assert_any_call(url, path)
url = JBI_ARTIFACT_URL.format( url = JBI_ARTIFACT_URL.format(
settings.JENKINS_URL, settings.JENKINS_URL,
jbi.jobname, jbi.jobname,
jbi.buildnumber, jbi.buildnumber,
"builddeps.list" "builddeps.list",
) )
artifact_base_path = os.path.join(base_path, 'artifact') artifact_base_path = os.path.join(base_path, "artifact")
path = os.path.join(artifact_base_path, 'builddeps.list') path = os.path.join(artifact_base_path, "builddeps.list")
self.assertNotIn(call(url, path), dlfile.call_args_list) self.assertNotIn(call(url, path), dlfile.call_args_list)
@patch('builtins.open', mock_open(read_data=artifacts_json)) @patch("builtins.open", mock_open(read_data=artifacts_json))
@patch('repoapi.utils.dlfile') @patch("repoapi.utils.dlfile")
def test_jbi_buildinfo(self, dlfile): def test_jbi_buildinfo(self, dlfile):
param = self.get_defaults() param = self.get_defaults()
jbi = JenkinsBuildInfo.objects.create(**param) jbi = JenkinsBuildInfo.objects.create(**param)
base_path = os.path.join(settings.JBI_BASEDIR, base_path = os.path.join(
jbi.jobname, str(jbi.buildnumber)) settings.JBI_BASEDIR, jbi.jobname, str(jbi.buildnumber)
)
url = JBI_BUILD_URL.format( url = JBI_BUILD_URL.format(
settings.JENKINS_URL, settings.JENKINS_URL, jbi.jobname, jbi.buildnumber
jbi.jobname,
jbi.buildnumber
) )
path = os.path.join(base_path, 'build.json') path = os.path.join(base_path, "build.json")
dlfile.assert_any_call(url, path) dlfile.assert_any_call(url, path)
url = JBI_ARTIFACT_URL.format( url = JBI_ARTIFACT_URL.format(
settings.JENKINS_URL, settings.JENKINS_URL,
jbi.jobname, jbi.jobname,
jbi.buildnumber, jbi.buildnumber,
"builddeps.list" "builddeps.list",
) )
artifact_base_path = os.path.join(base_path, 'artifact') artifact_base_path = os.path.join(base_path, "artifact")
path = os.path.join(artifact_base_path, 'builddeps.list') path = os.path.join(artifact_base_path, "builddeps.list")
self.assertNotIn(call(url, path), dlfile.call_args_list) self.assertNotIn(call(url, path), dlfile.call_args_list)
@patch('builtins.open', mock_open(read_data=artifacts_json)) @patch("builtins.open", mock_open(read_data=artifacts_json))
@patch('repoapi.utils.dlfile') @patch("repoapi.utils.dlfile")
def test_jbi_artifact(self, dlfile): def test_jbi_artifact(self, dlfile):
param = self.get_defaults() param = self.get_defaults()
param['jobname'] = 'fake-release-tools-runner' param["jobname"] = "fake-release-tools-runner"
jbi = JenkinsBuildInfo.objects.create(**param) jbi = JenkinsBuildInfo.objects.create(**param)
base_path = os.path.join(settings.JBI_BASEDIR, base_path = os.path.join(
jbi.jobname, str(jbi.buildnumber)) settings.JBI_BASEDIR, jbi.jobname, str(jbi.buildnumber)
)
url = JBI_ARTIFACT_URL.format( url = JBI_ARTIFACT_URL.format(
settings.JENKINS_URL, settings.JENKINS_URL,
jbi.jobname, jbi.jobname,
jbi.buildnumber, jbi.buildnumber,
"builddeps.list" "builddeps.list",
) )
artifact_base_path = os.path.join(base_path, 'artifact') artifact_base_path = os.path.join(base_path, "artifact")
path = os.path.join(artifact_base_path, 'builddeps.list') path = os.path.join(artifact_base_path, "builddeps.list")
dlfile.assert_any_call(url, path) dlfile.assert_any_call(url, path)
@patch('builtins.open', mock_open(read_data=artifacts_json)) @patch("builtins.open", mock_open(read_data=artifacts_json))
@patch('repoapi.utils.dlfile') @patch("repoapi.utils.dlfile")
def test_jbi_envVars(self, dlfile): def test_jbi_envVars(self, dlfile):
param = self.get_defaults() param = self.get_defaults()
jbi = JenkinsBuildInfo.objects.create(**param) jbi = JenkinsBuildInfo.objects.create(**param)
base_path = os.path.join(settings.JBI_BASEDIR, base_path = os.path.join(
jbi.jobname, str(jbi.buildnumber)) settings.JBI_BASEDIR, jbi.jobname, str(jbi.buildnumber)
)
url = JBI_ENVVARS_URL.format( url = JBI_ENVVARS_URL.format(
settings.JENKINS_URL, settings.JENKINS_URL, jbi.jobname, jbi.buildnumber
jbi.jobname,
jbi.buildnumber
) )
path = os.path.join(base_path, 'envVars.json') path = os.path.join(base_path, "envVars.json")
dlfile.assert_any_call(url, path) dlfile.assert_any_call(url, path)

Write Preview
Loading…
Cancel
Save