diff --git a/Makefile b/Makefile index 15414f1..873479f 100644 --- a/Makefile +++ b/Makefile @@ -13,8 +13,10 @@ venv_prod: requirements/prod.txt ################################### test: - ./manage.py jenkins --enable-coverage --noinput --output-dir $(RESULTS) \ - --settings="repoapi.settings.dev" + RESULTS=$(RESULTS) ./manage.py jenkins --enable-coverage --noinput --output-dir $(RESULTS) \ + --settings="repoapi.settings.test" + +################################### deploy: venv_prod source $(VAR_DIR)/venv_prod/bin/activate && \ @@ -32,6 +34,17 @@ run_dev: IP=$(shell ip a show dev eth0 scope global | grep inet | awk '{print $$2}' | cut -d/ -f1); \ ./manage.py runserver_plus $$IP:8000 --settings="repoapi.settings.dev" +worker_dev: + ./manage.py celery worker --loglevel=info --settings="repoapi.settings.dev" + +makemigrations_dev: + ./manage.py makemigrations --settings="repoapi.settings.dev" + +migrate_dev: + ./manage.py migrate --settings="repoapi.settings.dev" + +shell_dev: + ./manage.py shell --settings="repoapi.settings.dev" ################################### # get rid of test files diff --git a/repoapi.ini b/repoapi.ini index 47f2850..27ea563 100644 --- a/repoapi.ini +++ b/repoapi.ini @@ -13,3 +13,6 @@ home = /var/lib/repoapi/venv_prod env = DJANGO_SETTINGS_MODULE=repoapi.settings.prod # spawn 20 uWSGI worker processes workers = 20 +# celery +workerpid = /var/lib/repoapi/celery-worker.pid +smart-attach-daemon = %(workerpid) %(home)/bin/python %(chdir)/manage.py celery worker --pidfile=%(workerpid) -l info diff --git a/repoapi/__init__.py b/repoapi/__init__.py index e69de29..77dee68 100644 --- a/repoapi/__init__.py +++ b/repoapi/__init__.py @@ -0,0 +1,19 @@ +# Copyright (C) 2016 The Sipwise Team - http://sipwise.com + +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. + +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. + +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +from __future__ import absolute_import + +# This will make sure the app is always imported when +# Django starts so that shared_task will use this app. +from .celery import app as celery_app # noqa diff --git a/repoapi/celery.py b/repoapi/celery.py new file mode 100644 index 0000000..5bdc677 --- /dev/null +++ b/repoapi/celery.py @@ -0,0 +1,30 @@ +# Copyright (C) 2016 The Sipwise Team - http://sipwise.com + +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. + +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. + +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +from __future__ import absolute_import + +import os +from celery import Celery + +# set the default Django settings module for the 'celery' program. +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'repoapi.settings.prod') +# pylint: disable=C0413 +from django.conf import settings # noqa + +app = Celery('repoapi') + +# Using a string here means the worker will not have to +# pickle the object when using Windows. +app.config_from_object('django.conf:settings') +app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) diff --git a/repoapi/models.py b/repoapi/models.py index 44be59a..c46c1a6 100644 --- a/repoapi/models.py +++ b/repoapi/models.py @@ -19,6 +19,7 @@ from django.db import models from django.db.models import signals from django.conf import settings from repoapi import utils +from .tasks import get_jbi_files logger = logging.getLogger(__name__) workfront_re = re.compile(r"TT#(\d+)") @@ -101,6 +102,12 @@ class JenkinsBuildInfo(models.Model): return "%s:%d[%s]" % (self.jobname, self.buildnumber, self.tag) +def jbi_manage(sender, **kwargs): + if kwargs["created"]: + instance = kwargs["instance"] + get_jbi_files.delay(instance.jobname, instance.buildnumber) + +signals.post_save.connect(jbi_manage, sender=JenkinsBuildInfo) class GerritRepoInfo(models.Model): param_ppa = models.CharField(max_length=50, null=False) diff --git a/repoapi/settings/common.py b/repoapi/settings/common.py index ce8aa31..28143c5 100644 --- a/repoapi/settings/common.py +++ b/repoapi/settings/common.py @@ -37,6 +37,7 @@ INSTALLED_APPS = [ 'rest_framework_swagger', 'django_extensions', 'django_assets', + 'djcelery', ] MIDDLEWARE_CLASSES = ( @@ -135,3 +136,8 @@ LOGGING = { } JENKINS_TOKEN = "sipwise_jenkins_ci" + +CELERY_TASK_SERIALIZER = 'json' +CELERY_RESULT_SERIALIZER = 'json' +CELERY_ACCEPT_CONTENT = ['json'] +CELERY_RESULT_BACKEND = 'djcelery.backends.database:DatabaseBackend' diff --git a/repoapi/settings/dev.py b/repoapi/settings/dev.py index 93acea5..452cc35 100644 --- a/repoapi/settings/dev.py +++ b/repoapi/settings/dev.py @@ -15,49 +15,12 @@ # Build paths inside the project like this: os.path.join(BASE_DIR, ...) import os -# pylint: disable=W0401,W0614 -from .common import * -BASE_DIR = os.path.dirname( - os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +# pylint: disable=W0401,W0614,C0413 +from .test import * -# Quick-start development settings - unsuitable for production -# See https://docs.djangoproject.com/en/1.8/howto/deployment/checklist/ - -# SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = ')+0h68-(g30hg1awc6!y65cwws6j^qd5=&pc2@h430=9x@bf%2' - -# SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True - -ALLOWED_HOSTS = [] - -TESTING_APPS = [ - 'django_jenkins', -] -INSTALLED_APPS.extend(TESTING_APPS) -INSTALLED_APPS.extend(PROJECT_APPS) - -# Database -# https://docs.djangoproject.com/en/1.8/ref/settings/#databases - -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), - } -} - -# django-jenkins -JENKINS_TASKS = ( - 'django_jenkins.tasks.run_pylint', - 'django_jenkins.tasks.run_flake8', -) -PYLINT_RCFILE = 'pylint.cfg' - -LOGGING['loggers']['repoapi']['level'] = os.getenv('DJANGO_LOG_LEVEL', 'DEBUG') - -JENKINS_URL = "http://localhost" -GERRIT_URL = "https://gerrit.local/{}" -GITWEB_URL = "https://git.local/gitweb/?p={}.git;a=commit;h={}" -WORKFRONT_CREDENTIALS = os.path.join(BASE_DIR, '.workfront.ini') +# celery +BROKER_BACKEND = 'amqp' +CELERY_ALWAYS_EAGER = False +BROKER_URL = 'amqp://guest:guest@rabbit' +JBI_BASEDIR = os.path.join(BASE_DIR, 'jbi_files') diff --git a/repoapi/settings/prod.py b/repoapi/settings/prod.py index 921dda7..961ed66 100644 --- a/repoapi/settings/prod.py +++ b/repoapi/settings/prod.py @@ -56,3 +56,6 @@ GERRIT_URL = "https://gerrit.mgm.sipwise.com/{}" GITWEB_URL = "https://git.mgm.sipwise.com/gitweb/?p={}.git;a=commit;h={}" WORKFRONT_CREDENTIALS = os.path.join(BASE_DIR, '/etc/jenkins_jobs/workfront.ini') +# celery +BROKER_URL = 'amqp://guest:guest@localhost' +JBI_BASEDIR = os.path.join(VAR_DIR, 'jbi_files') diff --git a/repoapi/settings/test.py b/repoapi/settings/test.py new file mode 100644 index 0000000..66f7473 --- /dev/null +++ b/repoapi/settings/test.py @@ -0,0 +1,70 @@ +# Copyright (C) 2015 The Sipwise Team - http://sipwise.com + +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. + +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. + +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +import os +# pylint: disable=W0401,W0614 +from .common import * + +BASE_DIR = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +os.environ.setdefault('RESULTS', '/tmp') +RESULTS_DIR = os.environ['RESULTS'] + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/1.8/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = ')+0h68-(g30hg1awc6!y65cwws6j^qd5=&pc2@h430=9x@bf%2' + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +ALLOWED_HOSTS = [] + +TESTING_APPS = [ + 'django_jenkins', +] +INSTALLED_APPS.extend(TESTING_APPS) +INSTALLED_APPS.extend(PROJECT_APPS) + +# Database +# https://docs.djangoproject.com/en/1.8/ref/settings/#databases + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), + } +} + +# django-jenkins +JENKINS_TASKS = ( + 'django_jenkins.tasks.run_pylint', + 'django_jenkins.tasks.run_flake8', +) +PYLINT_RCFILE = 'pylint.cfg' + +DJANGO_LOG_LEVEL = 'DEBUG' + +JENKINS_URL = "http://localhost" +GERRIT_URL = "https://gerrit.local/{}" +GITWEB_URL = "https://git.local/gitweb/?p={}.git;a=commit;h={}" +WORKFRONT_CREDENTIALS = os.path.join(BASE_DIR, '.workfront.ini') + +# celery +BROKER_BACKEND = 'memory' +CELERY_ALWAYS_EAGER = True +JBI_BASEDIR = os.path.join(RESULTS_DIR, 'jbi_files') diff --git a/repoapi/tasks.py b/repoapi/tasks.py new file mode 100644 index 0000000..55d4014 --- /dev/null +++ b/repoapi/tasks.py @@ -0,0 +1,24 @@ +# Copyright (C) 2016 The Sipwise Team - http://sipwise.com + +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. + +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. + +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +from __future__ import absolute_import + +from celery import shared_task +from .utils import jenkins_get_console, jenkins_get_job + + +@shared_task +def get_jbi_files(jobname, buildnumber): + jenkins_get_console(jobname, buildnumber) + jenkins_get_job(jobname, buildnumber) diff --git a/repoapi/test/test_jbi_info.py b/repoapi/test/test_jbi_info.py new file mode 100644 index 0000000..c67ab45 --- /dev/null +++ b/repoapi/test/test_jbi_info.py @@ -0,0 +1,74 @@ +# Copyright (C) 2015 The Sipwise Team - http://sipwise.com + +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. + +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. + +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . + +import os +import shutil + +from django.test import TestCase +from django.conf import settings +from repoapi.models import JenkinsBuildInfo +from repoapi.utils import JBI_CONSOLE_URL, JBI_JOB_URL +from mock import patch, call + +class TestJBICelery(TestCase): + + def get_defaults(self): + defaults = { + 'tag': "edc90cd9-37f3-4613-9748-ed05a32031c2", + 'projectname': "real-fake", + 'jobname': "real-fake-gerrit", + 'buildnumber': 1, + 'result': "SUCCESS", + 'job_url': "https://jenkins.mgm.sipwise.com/job/real-fake-gerrit/", + 'param_tag': "none", + 'param_branch': "master", + 'param_release': "none", + 'param_distribution': "wheezy", + 'param_ppa': "gerrit_MT10339_review2054", + 'git_commit_msg': "7fg4567 TT#0001 whatever", + } + return defaults + + def setUp(self): + if not os.path.exists(settings.JBI_BASEDIR): + os.makedirs(settings.JBI_BASEDIR) + + def tearDown(self): + if os.path.exists(settings.JBI_BASEDIR): + shutil.rmtree(settings.JBI_BASEDIR) + + @patch('repoapi.utils.dlfile') + def test_jbi_path_creation(self, dlfile): + param = self.get_defaults() + jbi = JenkinsBuildInfo.objects.create(**param) + base_path = os.path.join(settings.JBI_BASEDIR, + jbi.jobname, str(jbi.buildnumber)) + self.assertTrue(os.path.exists(settings.JBI_BASEDIR), settings.JBI_BASEDIR) + self.assertTrue(os.path.exists(base_path)) + path = os.path.join(base_path, 'console.txt') + url = JBI_CONSOLE_URL.format( + settings.JENKINS_URL, + jbi.jobname, + jbi.buildnumber + ) + calls = [call(url, path),] + url = JBI_JOB_URL.format( + settings.JENKINS_URL, + jbi.jobname, + jbi.buildnumber + ) + path = os.path.join(base_path, 'job.json') + calls.append(call(url, path)) + dlfile.assert_has_calls(calls) diff --git a/repoapi/utils.py b/repoapi/utils.py index 08b8612..2b6aa8a 100644 --- a/repoapi/utils.py +++ b/repoapi/utils.py @@ -12,14 +12,20 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . +from __future__ import absolute_import -import urllib2 +from distutils.dir_util import mkpath import logging +import os import subprocess +import urllib2 from django.conf import settings logger = logging.getLogger(__name__) +JBI_CONSOLE_URL = "{}/job/{}/{}/consoleText" +JBI_JOB_URL = "{}/job/{}/{}/api/json" + def executeAndReturnOutput(command, env=None): proc = subprocess.Popen(command, stdout=subprocess.PIPE, @@ -30,6 +36,16 @@ def executeAndReturnOutput(command, env=None): return proc.returncode, stdoutdata, stderrdata +def dlfile(url, path): + if settings.DEBUG: + logger.info("I would call %s", url) + else: + remote_file = urllib2.urlopen(url) + logger.debug("url:[%s]", url) + with open(path, "wb") as local_file: + local_file.write(remote_file.read()) + + def openurl(url): req = urllib2.Request(url) logger.debug("url:[%s]", url) @@ -52,6 +68,35 @@ def jenkins_remove_ppa(repo): openurl(url) +def _jenkins_get(url, base_path, filename): + mkpath(base_path) + path = os.path.join(base_path, filename) + logger.info("url:[%s] path[%s]", url, path) + dlfile(url, path) + + +def jenkins_get_console(jobname, buildnumber): + url = JBI_CONSOLE_URL.format( + settings.JENKINS_URL, + jobname, + buildnumber + ) + base_path = os.path.join(settings.JBI_BASEDIR, + jobname, str(buildnumber)) + _jenkins_get(url, base_path, 'console.txt') + + +def jenkins_get_job(jobname, buildnumber): + url = JBI_JOB_URL.format( + settings.JENKINS_URL, + jobname, + buildnumber + ) + base_path = os.path.join(settings.JBI_BASEDIR, + jobname, str(buildnumber)) + _jenkins_get(url, base_path, 'job.json') + + def workfront_note_send(_id, message): command = [ "/usr/bin/workfront-post-note", diff --git a/requirements/common.txt b/requirements/common.txt index 6ccb5ee..e8bb5e6 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -8,3 +8,4 @@ markdown django-filter six webassets +django-celery diff --git a/t/Dockerfile b/t/Dockerfile index 32503e7..2da2fda 100644 --- a/t/Dockerfile +++ b/t/Dockerfile @@ -5,10 +5,10 @@ FROM docker.mgm.sipwise.com/sipwise-jessie:latest # is updated with the current date. It will force refresh of all # of the base images and things like `apt-get update` won't be using # old cached versions when the Dockerfile is built. -ENV REFRESHED_AT 2016-07-18 +ENV REFRESHED_AT 2016-07-19 RUN apt-get update -RUN apt-get install --assume-yes python2.7 python2.7-dev python-distribute python-pip git +RUN apt-get install --assume-yes python2.7 python2.7-dev python-distribute python-pip git screen # Get pip to download and install requirements: COPY dev.txt test.txt common.txt /tmp/ @@ -33,5 +33,11 @@ WORKDIR /code/ # ./t/testrunner # # Run django inside docker: -# make run_dev +# % pip install -r t/dev.txt && make run_dev +# +# We need a working rabbit server, so in another terminal: +# % docker run --rm --hostname repoapi-rabbit --name repoapi-rabbit rabbitmq:3 +# +# use screen to get a working worker in the background: +# % make worker_dev ################################################################################ diff --git a/t/common.txt b/t/common.txt index 6ccb5ee..e8bb5e6 100644 --- a/t/common.txt +++ b/t/common.txt @@ -8,3 +8,4 @@ markdown django-filter six webassets +django-celery