From 9b591bf43503925642daa11ca610c9c0ba4906d0 Mon Sep 17 00:00:00 2001 From: Victor Seva Date: Sat, 21 Mar 2020 12:58:25 +0100 Subject: [PATCH] TT#43813 upgrade celery settings for 4.4 > https://docs.celeryproject.org/en/latest/django/first-steps-with-django.html Change-Id: If66605b4675a6896f9bd5c387d300255ea7d368a --- hotfix/test/test_hotfix_released.py | 5 +- release_dashboard/test/test_tasks_build.py | 14 +-- release_dashboard/test/test_tasks_docker.py | 1 - repoapi/__init__.py | 11 +- repoapi/celery.py | 29 ++--- repoapi/settings/dev.py | 4 +- repoapi/settings/prod.py | 4 +- repoapi/settings/test.py | 6 +- repoapi/test/test_jbi_info.py | 133 ++++++++++---------- 9 files changed, 103 insertions(+), 104 deletions(-) diff --git a/hotfix/test/test_hotfix_released.py b/hotfix/test/test_hotfix_released.py index e1b5569..b529927 100644 --- a/hotfix/test/test_hotfix_released.py +++ b/hotfix/test/test_hotfix_released.py @@ -1,15 +1,17 @@ # Copyright (C) 2015 The Sipwise Team - http://sipwise.com +# # This program is free software: you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the Free # Software Foundation, either version 3 of the License, or (at your option) # any later version. +# # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # more details. +# # You should have received a copy of the GNU General Public License along # with this program. If not, see . -from django.test import override_settings from mock import call from mock import mock_open from mock import patch @@ -34,7 +36,6 @@ debian_changelog = """ngcp-fake (3.8.7.4+0~mr3.8.7.4) unstable; urgency=medium """ -@override_settings(CELERY_EAGER_PROPAGATES_EXCEPTIONS=True) class TestHotfixReleased(BaseTest): def get_defaults(self): defaults = { diff --git a/release_dashboard/test/test_tasks_build.py b/release_dashboard/test/test_tasks_build.py index e3ef785..5ca063c 100644 --- a/release_dashboard/test/test_tasks_build.py +++ b/release_dashboard/test/test_tasks_build.py @@ -1,27 +1,25 @@ # Copyright (C) 2017 The Sipwise Team - http://sipwise.com - +# # This program is free software: you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the Free # Software Foundation, either version 3 of the License, or (at your option) # any later version. - +# # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # more details. - +# # You should have received a copy of the GNU General Public License along # with this prograproj. If not, see . +from django.test import TestCase +from mock import patch -from django.test import TestCase, override_settings from release_dashboard import tasks -from mock import patch -@override_settings(CELERY_EAGER_PROPAGATES_EXCEPTIONS=True) class TasksBuildTestCase(TestCase): - - @patch('release_dashboard.tasks.gerrit_fetch_info') + @patch("release_dashboard.tasks.gerrit_fetch_info") def test_gerrit_fetch_all(self, gfi): result = tasks.gerrit_fetch_all.delay() self.assertTrue(result.successful()) diff --git a/release_dashboard/test/test_tasks_docker.py b/release_dashboard/test/test_tasks_docker.py index 6c2db5d..e3cf7d8 100644 --- a/release_dashboard/test/test_tasks_docker.py +++ b/release_dashboard/test/test_tasks_docker.py @@ -63,7 +63,6 @@ def fake_manifest(url): return ("{}", uuid.uuid4()) -@override_settings(CELERY_EAGER_PROPAGATES_EXCEPTIONS=True) @override_settings(DOCKER_REGISTRY_URL="{}") @override_settings(DEBUG=False) class TasksDockerTestCase(TestCase): diff --git a/repoapi/__init__.py b/repoapi/__init__.py index 77dee68..c216710 100644 --- a/repoapi/__init__.py +++ b/repoapi/__init__.py @@ -1,19 +1,20 @@ # Copyright (C) 2016 The Sipwise Team - http://sipwise.com - +# # This program is free software: you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the Free # Software Foundation, either version 3 of the License, or (at your option) # any later version. - +# # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # more details. - +# # You should have received a copy of the GNU General Public License along # with this program. If not, see . -from __future__ import absolute_import +from .celery import app as celery_app # This will make sure the app is always imported when # Django starts so that shared_task will use this app. -from .celery import app as celery_app # noqa + +__all__ = ("celery_app",) diff --git a/repoapi/celery.py b/repoapi/celery.py index 6d16374..96185db 100644 --- a/repoapi/celery.py +++ b/repoapi/celery.py @@ -1,35 +1,36 @@ # Copyright (C) 2016 The Sipwise Team - http://sipwise.com - +# # This program is free software: you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the Free # Software Foundation, either version 3 of the License, or (at your option) # any later version. - +# # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # more details. - +# # You should have received a copy of the GNU General Public License along # with this program. If not, see . -from __future__ import absolute_import - import os + from celery import Celery # set the default Django settings module for the 'celery' program. -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'repoapi.settings.prod') -# pylint: disable=C0413 -from django.conf import settings # noqa +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "repoapi.settings.prod") + +app = Celery("repoapi") -app = Celery('repoapi') +# Using a string here means the worker doesn't have to serialize +# the configuration object to child processes. +# - namespace='CELERY' means all celery-related configuration keys +# should have a `CELERY_` prefix. +app.config_from_object("django.conf:settings", namespace="CELERY") -# Using a string here means the worker will not have to -# pickle the object when using Windows. -app.config_from_object('django.conf:settings') -app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) +# Load task modules from all registered Django app configs. +app.autodiscover_tasks() @app.task() def jbi_parse_hotfix(jbi_id, path): - app.send_task('hotfix.tasks.hotfix_released', args=[jbi_id, path]) + app.send_task("hotfix.tasks.hotfix_released", args=[jbi_id, path]) diff --git a/repoapi/settings/dev.py b/repoapi/settings/dev.py index 6b614da..0c72d9e 100644 --- a/repoapi/settings/dev.py +++ b/repoapi/settings/dev.py @@ -27,8 +27,8 @@ LOGGING["loggers"]["repoapi"]["level"] = os.getenv( # noqa # celery BROKER_BACKEND = "amqp" -CELERY_ALWAYS_EAGER = False -BROKER_URL = "amqp://guest:guest@rabbit" +CELERY_TASK_ALWAYS_EAGER = False +CELERY_BROKER_URL = "amqp://guest:guest@rabbit" JBI_BASEDIR = os.path.join(BASE_DIR, "jbi_files") # noqa # Enable access when not accessing from localhost: diff --git a/repoapi/settings/prod.py b/repoapi/settings/prod.py index 861d0d1..8375016 100644 --- a/repoapi/settings/prod.py +++ b/repoapi/settings/prod.py @@ -93,8 +93,8 @@ BUILD_KEY_AUTH = True REPOS_SCRIPTS_CONFIG_DIR = "/usr/share/sipwise-repos-scripts/config" # celery -BROKER_URL = server_config.get("server", "BROKER_URL") -CELERYBEAT_SCHEDULE = { +CELERY_BROKER_URL = server_config.get("server", "BROKER_URL") +CELERY_BEAT_SCHEDULE = { # Executes every Sunday morning at 7:30 A.M "purge-trunk": { "task": "repoapi.tasks.jbi_purge", diff --git a/repoapi/settings/test.py b/repoapi/settings/test.py index acae1d0..b8a3af8 100644 --- a/repoapi/settings/test.py +++ b/repoapi/settings/test.py @@ -96,9 +96,9 @@ BUILD_KEY_AUTH = True REPOS_SCRIPTS_CONFIG_DIR = join(BASE_DIR, "build", "fixtures", "config") # celery -BROKER_BACKEND = "memory" -CELERY_ALWAYS_EAGER = True -CELERY_EAGER_PROPAGATES_EXCEPTIONS = True +CELERY_BROKER_URL = "memory://localhost/" +CELERY_TASK_ALWAYS_EAGER = True +CELERY_TASK_EAGER_PROPAGATES = True JBI_BASEDIR = join(RESULTS_DIR, "jbi_files") JBI_ARTIFACT_JOBS = [ "fake-release-tools-runner", diff --git a/repoapi/test/test_jbi_info.py b/repoapi/test/test_jbi_info.py index c54c3c3..ce5c6ef 100644 --- a/repoapi/test/test_jbi_info.py +++ b/repoapi/test/test_jbi_info.py @@ -1,27 +1,30 @@ # Copyright (C) 2015 The Sipwise Team - http://sipwise.com - +# # This program is free software: you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the Free # Software Foundation, either version 3 of the License, or (at your option) # any later version. - +# # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # more details. - +# # You should have received a copy of the GNU General Public License along # with this program. If not, see . - import os -from django.test import override_settings from django.conf import settings +from mock import call +from mock import mock_open +from mock import patch + from repoapi.models import JenkinsBuildInfo -from repoapi.utils import JBI_CONSOLE_URL, JBI_BUILD_URL, JBI_ARTIFACT_URL -from repoapi.utils import JBI_ENVVARS_URL -from mock import patch, call, mock_open from repoapi.test.base import BaseTest +from repoapi.utils import JBI_ARTIFACT_URL +from repoapi.utils import JBI_BUILD_URL +from repoapi.utils import JBI_CONSOLE_URL +from repoapi.utils import JBI_ENVVARS_URL artifacts_json = """{ "artifacts": [ @@ -34,116 +37,112 @@ artifacts_json = """{ }""" -@override_settings(CELERY_EAGER_PROPAGATES_EXCEPTIONS=True) class TestJBICelery(BaseTest): - def get_defaults(self): defaults = { - 'tag': "edc90cd9-37f3-4613-9748-ed05a32031c2", - 'projectname': "real-fake", - 'jobname': "real-fake-gerrit", - 'buildnumber': 1, - 'result': "SUCCESS", - 'job_url': - "https://jenkins-dev.mgm.sipwise.com/job" - "/real-fake-gerrit/", - 'param_tag': "none", - 'param_branch': "master", - 'param_release': "none", - 'param_distribution': "wheezy", - 'param_ppa': "gerrit_MT10339_review2054", - 'git_commit_msg': "7fg4567 TT#0001 whatever", + "tag": "edc90cd9-37f3-4613-9748-ed05a32031c2", + "projectname": "real-fake", + "jobname": "real-fake-gerrit", + "buildnumber": 1, + "result": "SUCCESS", + "job_url": "https://jenkins-dev.mgm.sipwise.com/job" + "/real-fake-gerrit/", + "param_tag": "none", + "param_branch": "master", + "param_release": "none", + "param_distribution": "wheezy", + "param_ppa": "gerrit_MT10339_review2054", + "git_commit_msg": "7fg4567 TT#0001 whatever", } return defaults - @patch('builtins.open', mock_open(read_data=artifacts_json)) - @patch('repoapi.utils.dlfile') + @patch("builtins.open", mock_open(read_data=artifacts_json)) + @patch("repoapi.utils.dlfile") def test_jbi_path_creation(self, dlfile): param = self.get_defaults() - param['jobname'] = 'fake-me' + param["jobname"] = "fake-me" jbi = JenkinsBuildInfo.objects.create(**param) - base_path = os.path.join(settings.JBI_BASEDIR, - jbi.jobname, str(jbi.buildnumber)) + base_path = os.path.join( + settings.JBI_BASEDIR, jbi.jobname, str(jbi.buildnumber) + ) self.assertTrue(os.path.isdir(base_path), base_path) - @patch('builtins.open', mock_open(read_data=artifacts_json)) - @patch('repoapi.utils.dlfile') + @patch("builtins.open", mock_open(read_data=artifacts_json)) + @patch("repoapi.utils.dlfile") def test_jbi_console(self, dlfile): param = self.get_defaults() jbi = JenkinsBuildInfo.objects.create(**param) - base_path = os.path.join(settings.JBI_BASEDIR, - jbi.jobname, str(jbi.buildnumber)) + base_path = os.path.join( + settings.JBI_BASEDIR, jbi.jobname, str(jbi.buildnumber) + ) - path = os.path.join(base_path, 'console.txt') + path = os.path.join(base_path, "console.txt") url = JBI_CONSOLE_URL.format( - settings.JENKINS_URL, - jbi.jobname, - jbi.buildnumber + settings.JENKINS_URL, jbi.jobname, jbi.buildnumber ) dlfile.assert_any_call(url, path) url = JBI_ARTIFACT_URL.format( settings.JENKINS_URL, jbi.jobname, jbi.buildnumber, - "builddeps.list" + "builddeps.list", ) - artifact_base_path = os.path.join(base_path, 'artifact') - path = os.path.join(artifact_base_path, 'builddeps.list') + artifact_base_path = os.path.join(base_path, "artifact") + path = os.path.join(artifact_base_path, "builddeps.list") self.assertNotIn(call(url, path), dlfile.call_args_list) - @patch('builtins.open', mock_open(read_data=artifacts_json)) - @patch('repoapi.utils.dlfile') + @patch("builtins.open", mock_open(read_data=artifacts_json)) + @patch("repoapi.utils.dlfile") def test_jbi_buildinfo(self, dlfile): param = self.get_defaults() jbi = JenkinsBuildInfo.objects.create(**param) - base_path = os.path.join(settings.JBI_BASEDIR, - jbi.jobname, str(jbi.buildnumber)) + base_path = os.path.join( + settings.JBI_BASEDIR, jbi.jobname, str(jbi.buildnumber) + ) url = JBI_BUILD_URL.format( - settings.JENKINS_URL, - jbi.jobname, - jbi.buildnumber + settings.JENKINS_URL, jbi.jobname, jbi.buildnumber ) - path = os.path.join(base_path, 'build.json') + path = os.path.join(base_path, "build.json") dlfile.assert_any_call(url, path) url = JBI_ARTIFACT_URL.format( settings.JENKINS_URL, jbi.jobname, jbi.buildnumber, - "builddeps.list" + "builddeps.list", ) - artifact_base_path = os.path.join(base_path, 'artifact') - path = os.path.join(artifact_base_path, 'builddeps.list') + artifact_base_path = os.path.join(base_path, "artifact") + path = os.path.join(artifact_base_path, "builddeps.list") self.assertNotIn(call(url, path), dlfile.call_args_list) - @patch('builtins.open', mock_open(read_data=artifacts_json)) - @patch('repoapi.utils.dlfile') + @patch("builtins.open", mock_open(read_data=artifacts_json)) + @patch("repoapi.utils.dlfile") def test_jbi_artifact(self, dlfile): param = self.get_defaults() - param['jobname'] = 'fake-release-tools-runner' + param["jobname"] = "fake-release-tools-runner" jbi = JenkinsBuildInfo.objects.create(**param) - base_path = os.path.join(settings.JBI_BASEDIR, - jbi.jobname, str(jbi.buildnumber)) + base_path = os.path.join( + settings.JBI_BASEDIR, jbi.jobname, str(jbi.buildnumber) + ) url = JBI_ARTIFACT_URL.format( settings.JENKINS_URL, jbi.jobname, jbi.buildnumber, - "builddeps.list" + "builddeps.list", ) - artifact_base_path = os.path.join(base_path, 'artifact') - path = os.path.join(artifact_base_path, 'builddeps.list') + artifact_base_path = os.path.join(base_path, "artifact") + path = os.path.join(artifact_base_path, "builddeps.list") dlfile.assert_any_call(url, path) - @patch('builtins.open', mock_open(read_data=artifacts_json)) - @patch('repoapi.utils.dlfile') + @patch("builtins.open", mock_open(read_data=artifacts_json)) + @patch("repoapi.utils.dlfile") def test_jbi_envVars(self, dlfile): param = self.get_defaults() jbi = JenkinsBuildInfo.objects.create(**param) - base_path = os.path.join(settings.JBI_BASEDIR, - jbi.jobname, str(jbi.buildnumber)) + base_path = os.path.join( + settings.JBI_BASEDIR, jbi.jobname, str(jbi.buildnumber) + ) url = JBI_ENVVARS_URL.format( - settings.JENKINS_URL, - jbi.jobname, - jbi.buildnumber + settings.JENKINS_URL, jbi.jobname, jbi.buildnumber ) - path = os.path.join(base_path, 'envVars.json') + path = os.path.join(base_path, "envVars.json") dlfile.assert_any_call(url, path)