TT#43813 upgrade celery settings for 4.4

> https://docs.celeryproject.org/en/latest/django/first-steps-with-django.html

Change-Id: If66605b4675a6896f9bd5c387d300255ea7d368a
changes/14/38714/1
Victor Seva 6 years ago
parent 44b0cdc640
commit 9b591bf435
No known key found for this signature in database
GPG Key ID: B1589889727198E0

@ -1,15 +1,17 @@
# Copyright (C) 2015 The Sipwise Team - http://sipwise.com
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>.
from django.test import override_settings
from mock import call
from mock import mock_open
from mock import patch
@ -34,7 +36,6 @@ debian_changelog = """ngcp-fake (3.8.7.4+0~mr3.8.7.4) unstable; urgency=medium
"""
@override_settings(CELERY_EAGER_PROPAGATES_EXCEPTIONS=True)
class TestHotfixReleased(BaseTest):
def get_defaults(self):
defaults = {

@ -1,27 +1,25 @@
# Copyright (C) 2017 The Sipwise Team - http://sipwise.com
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU General Public License along
# with this prograproj. If not, see <http://www.gnu.org/licenses/>.
from django.test import TestCase
from mock import patch
from django.test import TestCase, override_settings
from release_dashboard import tasks
from mock import patch
@override_settings(CELERY_EAGER_PROPAGATES_EXCEPTIONS=True)
class TasksBuildTestCase(TestCase):
@patch('release_dashboard.tasks.gerrit_fetch_info')
@patch("release_dashboard.tasks.gerrit_fetch_info")
def test_gerrit_fetch_all(self, gfi):
result = tasks.gerrit_fetch_all.delay()
self.assertTrue(result.successful())

@ -63,7 +63,6 @@ def fake_manifest(url):
return ("{}", uuid.uuid4())
@override_settings(CELERY_EAGER_PROPAGATES_EXCEPTIONS=True)
@override_settings(DOCKER_REGISTRY_URL="{}")
@override_settings(DEBUG=False)
class TasksDockerTestCase(TestCase):

@ -1,19 +1,20 @@
# Copyright (C) 2016 The Sipwise Team - http://sipwise.com
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>.
from __future__ import absolute_import
from .celery import app as celery_app
# This will make sure the app is always imported when
# Django starts so that shared_task will use this app.
from .celery import app as celery_app # noqa
__all__ = ("celery_app",)

@ -1,35 +1,36 @@
# Copyright (C) 2016 The Sipwise Team - http://sipwise.com
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>.
from __future__ import absolute_import
import os
from celery import Celery
# set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'repoapi.settings.prod')
# pylint: disable=C0413
from django.conf import settings # noqa
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "repoapi.settings.prod")
app = Celery("repoapi")
app = Celery('repoapi')
# Using a string here means the worker doesn't have to serialize
# the configuration object to child processes.
# - namespace='CELERY' means all celery-related configuration keys
# should have a `CELERY_` prefix.
app.config_from_object("django.conf:settings", namespace="CELERY")
# Using a string here means the worker will not have to
# pickle the object when using Windows.
app.config_from_object('django.conf:settings')
app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
# Load task modules from all registered Django app configs.
app.autodiscover_tasks()
@app.task()
def jbi_parse_hotfix(jbi_id, path):
app.send_task('hotfix.tasks.hotfix_released', args=[jbi_id, path])
app.send_task("hotfix.tasks.hotfix_released", args=[jbi_id, path])

@ -27,8 +27,8 @@ LOGGING["loggers"]["repoapi"]["level"] = os.getenv( # noqa
# celery
BROKER_BACKEND = "amqp"
CELERY_ALWAYS_EAGER = False
BROKER_URL = "amqp://guest:guest@rabbit"
CELERY_TASK_ALWAYS_EAGER = False
CELERY_BROKER_URL = "amqp://guest:guest@rabbit"
JBI_BASEDIR = os.path.join(BASE_DIR, "jbi_files") # noqa
# Enable access when not accessing from localhost:

@ -93,8 +93,8 @@ BUILD_KEY_AUTH = True
REPOS_SCRIPTS_CONFIG_DIR = "/usr/share/sipwise-repos-scripts/config"
# celery
BROKER_URL = server_config.get("server", "BROKER_URL")
CELERYBEAT_SCHEDULE = {
CELERY_BROKER_URL = server_config.get("server", "BROKER_URL")
CELERY_BEAT_SCHEDULE = {
# Executes every Sunday morning at 7:30 A.M
"purge-trunk": {
"task": "repoapi.tasks.jbi_purge",

@ -96,9 +96,9 @@ BUILD_KEY_AUTH = True
REPOS_SCRIPTS_CONFIG_DIR = join(BASE_DIR, "build", "fixtures", "config")
# celery
BROKER_BACKEND = "memory"
CELERY_ALWAYS_EAGER = True
CELERY_EAGER_PROPAGATES_EXCEPTIONS = True
CELERY_BROKER_URL = "memory://localhost/"
CELERY_TASK_ALWAYS_EAGER = True
CELERY_TASK_EAGER_PROPAGATES = True
JBI_BASEDIR = join(RESULTS_DIR, "jbi_files")
JBI_ARTIFACT_JOBS = [
"fake-release-tools-runner",

@ -1,27 +1,30 @@
# Copyright (C) 2015 The Sipwise Team - http://sipwise.com
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>.
import os
from django.test import override_settings
from django.conf import settings
from mock import call
from mock import mock_open
from mock import patch
from repoapi.models import JenkinsBuildInfo
from repoapi.utils import JBI_CONSOLE_URL, JBI_BUILD_URL, JBI_ARTIFACT_URL
from repoapi.utils import JBI_ENVVARS_URL
from mock import patch, call, mock_open
from repoapi.test.base import BaseTest
from repoapi.utils import JBI_ARTIFACT_URL
from repoapi.utils import JBI_BUILD_URL
from repoapi.utils import JBI_CONSOLE_URL
from repoapi.utils import JBI_ENVVARS_URL
artifacts_json = """{
"artifacts": [
@ -34,116 +37,112 @@ artifacts_json = """{
}"""
@override_settings(CELERY_EAGER_PROPAGATES_EXCEPTIONS=True)
class TestJBICelery(BaseTest):
def get_defaults(self):
defaults = {
'tag': "edc90cd9-37f3-4613-9748-ed05a32031c2",
'projectname': "real-fake",
'jobname': "real-fake-gerrit",
'buildnumber': 1,
'result': "SUCCESS",
'job_url':
"https://jenkins-dev.mgm.sipwise.com/job"
"/real-fake-gerrit/",
'param_tag': "none",
'param_branch': "master",
'param_release': "none",
'param_distribution': "wheezy",
'param_ppa': "gerrit_MT10339_review2054",
'git_commit_msg': "7fg4567 TT#0001 whatever",
"tag": "edc90cd9-37f3-4613-9748-ed05a32031c2",
"projectname": "real-fake",
"jobname": "real-fake-gerrit",
"buildnumber": 1,
"result": "SUCCESS",
"job_url": "https://jenkins-dev.mgm.sipwise.com/job"
"/real-fake-gerrit/",
"param_tag": "none",
"param_branch": "master",
"param_release": "none",
"param_distribution": "wheezy",
"param_ppa": "gerrit_MT10339_review2054",
"git_commit_msg": "7fg4567 TT#0001 whatever",
}
return defaults
@patch('builtins.open', mock_open(read_data=artifacts_json))
@patch('repoapi.utils.dlfile')
@patch("builtins.open", mock_open(read_data=artifacts_json))
@patch("repoapi.utils.dlfile")
def test_jbi_path_creation(self, dlfile):
param = self.get_defaults()
param['jobname'] = 'fake-me'
param["jobname"] = "fake-me"
jbi = JenkinsBuildInfo.objects.create(**param)
base_path = os.path.join(settings.JBI_BASEDIR,
jbi.jobname, str(jbi.buildnumber))
base_path = os.path.join(
settings.JBI_BASEDIR, jbi.jobname, str(jbi.buildnumber)
)
self.assertTrue(os.path.isdir(base_path), base_path)
@patch('builtins.open', mock_open(read_data=artifacts_json))
@patch('repoapi.utils.dlfile')
@patch("builtins.open", mock_open(read_data=artifacts_json))
@patch("repoapi.utils.dlfile")
def test_jbi_console(self, dlfile):
param = self.get_defaults()
jbi = JenkinsBuildInfo.objects.create(**param)
base_path = os.path.join(settings.JBI_BASEDIR,
jbi.jobname, str(jbi.buildnumber))
base_path = os.path.join(
settings.JBI_BASEDIR, jbi.jobname, str(jbi.buildnumber)
)
path = os.path.join(base_path, 'console.txt')
path = os.path.join(base_path, "console.txt")
url = JBI_CONSOLE_URL.format(
settings.JENKINS_URL,
jbi.jobname,
jbi.buildnumber
settings.JENKINS_URL, jbi.jobname, jbi.buildnumber
)
dlfile.assert_any_call(url, path)
url = JBI_ARTIFACT_URL.format(
settings.JENKINS_URL,
jbi.jobname,
jbi.buildnumber,
"builddeps.list"
"builddeps.list",
)
artifact_base_path = os.path.join(base_path, 'artifact')
path = os.path.join(artifact_base_path, 'builddeps.list')
artifact_base_path = os.path.join(base_path, "artifact")
path = os.path.join(artifact_base_path, "builddeps.list")
self.assertNotIn(call(url, path), dlfile.call_args_list)
@patch('builtins.open', mock_open(read_data=artifacts_json))
@patch('repoapi.utils.dlfile')
@patch("builtins.open", mock_open(read_data=artifacts_json))
@patch("repoapi.utils.dlfile")
def test_jbi_buildinfo(self, dlfile):
param = self.get_defaults()
jbi = JenkinsBuildInfo.objects.create(**param)
base_path = os.path.join(settings.JBI_BASEDIR,
jbi.jobname, str(jbi.buildnumber))
base_path = os.path.join(
settings.JBI_BASEDIR, jbi.jobname, str(jbi.buildnumber)
)
url = JBI_BUILD_URL.format(
settings.JENKINS_URL,
jbi.jobname,
jbi.buildnumber
settings.JENKINS_URL, jbi.jobname, jbi.buildnumber
)
path = os.path.join(base_path, 'build.json')
path = os.path.join(base_path, "build.json")
dlfile.assert_any_call(url, path)
url = JBI_ARTIFACT_URL.format(
settings.JENKINS_URL,
jbi.jobname,
jbi.buildnumber,
"builddeps.list"
"builddeps.list",
)
artifact_base_path = os.path.join(base_path, 'artifact')
path = os.path.join(artifact_base_path, 'builddeps.list')
artifact_base_path = os.path.join(base_path, "artifact")
path = os.path.join(artifact_base_path, "builddeps.list")
self.assertNotIn(call(url, path), dlfile.call_args_list)
@patch('builtins.open', mock_open(read_data=artifacts_json))
@patch('repoapi.utils.dlfile')
@patch("builtins.open", mock_open(read_data=artifacts_json))
@patch("repoapi.utils.dlfile")
def test_jbi_artifact(self, dlfile):
param = self.get_defaults()
param['jobname'] = 'fake-release-tools-runner'
param["jobname"] = "fake-release-tools-runner"
jbi = JenkinsBuildInfo.objects.create(**param)
base_path = os.path.join(settings.JBI_BASEDIR,
jbi.jobname, str(jbi.buildnumber))
base_path = os.path.join(
settings.JBI_BASEDIR, jbi.jobname, str(jbi.buildnumber)
)
url = JBI_ARTIFACT_URL.format(
settings.JENKINS_URL,
jbi.jobname,
jbi.buildnumber,
"builddeps.list"
"builddeps.list",
)
artifact_base_path = os.path.join(base_path, 'artifact')
path = os.path.join(artifact_base_path, 'builddeps.list')
artifact_base_path = os.path.join(base_path, "artifact")
path = os.path.join(artifact_base_path, "builddeps.list")
dlfile.assert_any_call(url, path)
@patch('builtins.open', mock_open(read_data=artifacts_json))
@patch('repoapi.utils.dlfile')
@patch("builtins.open", mock_open(read_data=artifacts_json))
@patch("repoapi.utils.dlfile")
def test_jbi_envVars(self, dlfile):
param = self.get_defaults()
jbi = JenkinsBuildInfo.objects.create(**param)
base_path = os.path.join(settings.JBI_BASEDIR,
jbi.jobname, str(jbi.buildnumber))
base_path = os.path.join(
settings.JBI_BASEDIR, jbi.jobname, str(jbi.buildnumber)
)
url = JBI_ENVVARS_URL.format(
settings.JENKINS_URL,
jbi.jobname,
jbi.buildnumber
settings.JENKINS_URL, jbi.jobname, jbi.buildnumber
)
path = os.path.join(base_path, 'envVars.json')
path = os.path.join(base_path, "envVars.json")
dlfile.assert_any_call(url, path)

Loading…
Cancel
Save