diff --git a/Makefile b/Makefile
index 15414f1..873479f 100644
--- a/Makefile
+++ b/Makefile
@@ -13,8 +13,10 @@ venv_prod: requirements/prod.txt
###################################
test:
- ./manage.py jenkins --enable-coverage --noinput --output-dir $(RESULTS) \
- --settings="repoapi.settings.dev"
+ RESULTS=$(RESULTS) ./manage.py jenkins --enable-coverage --noinput --output-dir $(RESULTS) \
+ --settings="repoapi.settings.test"
+
+###################################
deploy: venv_prod
source $(VAR_DIR)/venv_prod/bin/activate && \
@@ -32,6 +34,17 @@ run_dev:
IP=$(shell ip a show dev eth0 scope global | grep inet | awk '{print $$2}' | cut -d/ -f1); \
./manage.py runserver_plus $$IP:8000 --settings="repoapi.settings.dev"
+worker_dev:
+ ./manage.py celery worker --loglevel=info --settings="repoapi.settings.dev"
+
+makemigrations_dev:
+ ./manage.py makemigrations --settings="repoapi.settings.dev"
+
+migrate_dev:
+ ./manage.py migrate --settings="repoapi.settings.dev"
+
+shell_dev:
+ ./manage.py shell --settings="repoapi.settings.dev"
###################################
# get rid of test files
diff --git a/repoapi.ini b/repoapi.ini
index 47f2850..27ea563 100644
--- a/repoapi.ini
+++ b/repoapi.ini
@@ -13,3 +13,6 @@ home = /var/lib/repoapi/venv_prod
env = DJANGO_SETTINGS_MODULE=repoapi.settings.prod
# spawn 20 uWSGI worker processes
workers = 20
+# celery
+workerpid = /var/lib/repoapi/celery-worker.pid
+smart-attach-daemon = %(workerpid) %(home)/bin/python %(chdir)/manage.py celery worker --pidfile=%(workerpid) -l info
diff --git a/repoapi/__init__.py b/repoapi/__init__.py
index e69de29..77dee68 100644
--- a/repoapi/__init__.py
+++ b/repoapi/__init__.py
@@ -0,0 +1,19 @@
+# Copyright (C) 2016 The Sipwise Team - http://sipwise.com
+
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+# more details.
+
+# You should have received a copy of the GNU General Public License along
+# with this program. If not, see .
+from __future__ import absolute_import
+
+# This will make sure the app is always imported when
+# Django starts so that shared_task will use this app.
+from .celery import app as celery_app # noqa
diff --git a/repoapi/celery.py b/repoapi/celery.py
new file mode 100644
index 0000000..5bdc677
--- /dev/null
+++ b/repoapi/celery.py
@@ -0,0 +1,30 @@
+# Copyright (C) 2016 The Sipwise Team - http://sipwise.com
+
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+# more details.
+
+# You should have received a copy of the GNU General Public License along
+# with this program. If not, see .
+from __future__ import absolute_import
+
+import os
+from celery import Celery
+
+# set the default Django settings module for the 'celery' program.
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'repoapi.settings.prod')
+# pylint: disable=C0413
+from django.conf import settings # noqa
+
+app = Celery('repoapi')
+
+# Using a string here means the worker will not have to
+# pickle the object when using Windows.
+app.config_from_object('django.conf:settings')
+app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
diff --git a/repoapi/models.py b/repoapi/models.py
index 44be59a..c46c1a6 100644
--- a/repoapi/models.py
+++ b/repoapi/models.py
@@ -19,6 +19,7 @@ from django.db import models
from django.db.models import signals
from django.conf import settings
from repoapi import utils
+from .tasks import get_jbi_files
logger = logging.getLogger(__name__)
workfront_re = re.compile(r"TT#(\d+)")
@@ -101,6 +102,12 @@ class JenkinsBuildInfo(models.Model):
return "%s:%d[%s]" % (self.jobname,
self.buildnumber, self.tag)
+def jbi_manage(sender, **kwargs):
+ if kwargs["created"]:
+ instance = kwargs["instance"]
+ get_jbi_files.delay(instance.jobname, instance.buildnumber)
+
+signals.post_save.connect(jbi_manage, sender=JenkinsBuildInfo)
class GerritRepoInfo(models.Model):
param_ppa = models.CharField(max_length=50, null=False)
diff --git a/repoapi/settings/common.py b/repoapi/settings/common.py
index ce8aa31..28143c5 100644
--- a/repoapi/settings/common.py
+++ b/repoapi/settings/common.py
@@ -37,6 +37,7 @@ INSTALLED_APPS = [
'rest_framework_swagger',
'django_extensions',
'django_assets',
+ 'djcelery',
]
MIDDLEWARE_CLASSES = (
@@ -135,3 +136,8 @@ LOGGING = {
}
JENKINS_TOKEN = "sipwise_jenkins_ci"
+
+CELERY_TASK_SERIALIZER = 'json'
+CELERY_RESULT_SERIALIZER = 'json'
+CELERY_ACCEPT_CONTENT = ['json']
+CELERY_RESULT_BACKEND = 'djcelery.backends.database:DatabaseBackend'
diff --git a/repoapi/settings/dev.py b/repoapi/settings/dev.py
index 93acea5..452cc35 100644
--- a/repoapi/settings/dev.py
+++ b/repoapi/settings/dev.py
@@ -15,49 +15,12 @@
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
import os
-# pylint: disable=W0401,W0614
-from .common import *
-BASE_DIR = os.path.dirname(
- os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+# pylint: disable=W0401,W0614,C0413
+from .test import *
-# Quick-start development settings - unsuitable for production
-# See https://docs.djangoproject.com/en/1.8/howto/deployment/checklist/
-
-# SECURITY WARNING: keep the secret key used in production secret!
-SECRET_KEY = ')+0h68-(g30hg1awc6!y65cwws6j^qd5=&pc2@h430=9x@bf%2'
-
-# SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = True
-
-ALLOWED_HOSTS = []
-
-TESTING_APPS = [
- 'django_jenkins',
-]
-INSTALLED_APPS.extend(TESTING_APPS)
-INSTALLED_APPS.extend(PROJECT_APPS)
-
-# Database
-# https://docs.djangoproject.com/en/1.8/ref/settings/#databases
-
-DATABASES = {
- 'default': {
- 'ENGINE': 'django.db.backends.sqlite3',
- 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
- }
-}
-
-# django-jenkins
-JENKINS_TASKS = (
- 'django_jenkins.tasks.run_pylint',
- 'django_jenkins.tasks.run_flake8',
-)
-PYLINT_RCFILE = 'pylint.cfg'
-
-LOGGING['loggers']['repoapi']['level'] = os.getenv('DJANGO_LOG_LEVEL', 'DEBUG')
-
-JENKINS_URL = "http://localhost"
-GERRIT_URL = "https://gerrit.local/{}"
-GITWEB_URL = "https://git.local/gitweb/?p={}.git;a=commit;h={}"
-WORKFRONT_CREDENTIALS = os.path.join(BASE_DIR, '.workfront.ini')
+# celery
+BROKER_BACKEND = 'amqp'
+CELERY_ALWAYS_EAGER = False
+BROKER_URL = 'amqp://guest:guest@rabbit'
+JBI_BASEDIR = os.path.join(BASE_DIR, 'jbi_files')
diff --git a/repoapi/settings/prod.py b/repoapi/settings/prod.py
index 921dda7..961ed66 100644
--- a/repoapi/settings/prod.py
+++ b/repoapi/settings/prod.py
@@ -56,3 +56,6 @@ GERRIT_URL = "https://gerrit.mgm.sipwise.com/{}"
GITWEB_URL = "https://git.mgm.sipwise.com/gitweb/?p={}.git;a=commit;h={}"
WORKFRONT_CREDENTIALS = os.path.join(BASE_DIR,
'/etc/jenkins_jobs/workfront.ini')
+# celery
+BROKER_URL = 'amqp://guest:guest@localhost'
+JBI_BASEDIR = os.path.join(VAR_DIR, 'jbi_files')
diff --git a/repoapi/settings/test.py b/repoapi/settings/test.py
new file mode 100644
index 0000000..66f7473
--- /dev/null
+++ b/repoapi/settings/test.py
@@ -0,0 +1,70 @@
+# Copyright (C) 2015 The Sipwise Team - http://sipwise.com
+
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+# more details.
+
+# You should have received a copy of the GNU General Public License along
+# with this program. If not, see .
+
+# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
+import os
+# pylint: disable=W0401,W0614
+from .common import *
+
+BASE_DIR = os.path.dirname(
+ os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+os.environ.setdefault('RESULTS', '/tmp')
+RESULTS_DIR = os.environ['RESULTS']
+
+# Quick-start development settings - unsuitable for production
+# See https://docs.djangoproject.com/en/1.8/howto/deployment/checklist/
+
+# SECURITY WARNING: keep the secret key used in production secret!
+SECRET_KEY = ')+0h68-(g30hg1awc6!y65cwws6j^qd5=&pc2@h430=9x@bf%2'
+
+# SECURITY WARNING: don't run with debug turned on in production!
+DEBUG = True
+
+ALLOWED_HOSTS = []
+
+TESTING_APPS = [
+ 'django_jenkins',
+]
+INSTALLED_APPS.extend(TESTING_APPS)
+INSTALLED_APPS.extend(PROJECT_APPS)
+
+# Database
+# https://docs.djangoproject.com/en/1.8/ref/settings/#databases
+
+DATABASES = {
+ 'default': {
+ 'ENGINE': 'django.db.backends.sqlite3',
+ 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
+ }
+}
+
+# django-jenkins
+JENKINS_TASKS = (
+ 'django_jenkins.tasks.run_pylint',
+ 'django_jenkins.tasks.run_flake8',
+)
+PYLINT_RCFILE = 'pylint.cfg'
+
+DJANGO_LOG_LEVEL = 'DEBUG'
+
+JENKINS_URL = "http://localhost"
+GERRIT_URL = "https://gerrit.local/{}"
+GITWEB_URL = "https://git.local/gitweb/?p={}.git;a=commit;h={}"
+WORKFRONT_CREDENTIALS = os.path.join(BASE_DIR, '.workfront.ini')
+
+# celery
+BROKER_BACKEND = 'memory'
+CELERY_ALWAYS_EAGER = True
+JBI_BASEDIR = os.path.join(RESULTS_DIR, 'jbi_files')
diff --git a/repoapi/tasks.py b/repoapi/tasks.py
new file mode 100644
index 0000000..55d4014
--- /dev/null
+++ b/repoapi/tasks.py
@@ -0,0 +1,24 @@
+# Copyright (C) 2016 The Sipwise Team - http://sipwise.com
+
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+# more details.
+
+# You should have received a copy of the GNU General Public License along
+# with this program. If not, see .
+from __future__ import absolute_import
+
+from celery import shared_task
+from .utils import jenkins_get_console, jenkins_get_job
+
+
+@shared_task
+def get_jbi_files(jobname, buildnumber):
+ jenkins_get_console(jobname, buildnumber)
+ jenkins_get_job(jobname, buildnumber)
diff --git a/repoapi/test/test_jbi_info.py b/repoapi/test/test_jbi_info.py
new file mode 100644
index 0000000..c67ab45
--- /dev/null
+++ b/repoapi/test/test_jbi_info.py
@@ -0,0 +1,74 @@
+# Copyright (C) 2015 The Sipwise Team - http://sipwise.com
+
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+# more details.
+
+# You should have received a copy of the GNU General Public License along
+# with this program. If not, see .
+
+import os
+import shutil
+
+from django.test import TestCase
+from django.conf import settings
+from repoapi.models import JenkinsBuildInfo
+from repoapi.utils import JBI_CONSOLE_URL, JBI_JOB_URL
+from mock import patch, call
+
+class TestJBICelery(TestCase):
+
+ def get_defaults(self):
+ defaults = {
+ 'tag': "edc90cd9-37f3-4613-9748-ed05a32031c2",
+ 'projectname': "real-fake",
+ 'jobname': "real-fake-gerrit",
+ 'buildnumber': 1,
+ 'result': "SUCCESS",
+ 'job_url': "https://jenkins.mgm.sipwise.com/job/real-fake-gerrit/",
+ 'param_tag': "none",
+ 'param_branch': "master",
+ 'param_release': "none",
+ 'param_distribution': "wheezy",
+ 'param_ppa': "gerrit_MT10339_review2054",
+ 'git_commit_msg': "7fg4567 TT#0001 whatever",
+ }
+ return defaults
+
+ def setUp(self):
+ if not os.path.exists(settings.JBI_BASEDIR):
+ os.makedirs(settings.JBI_BASEDIR)
+
+ def tearDown(self):
+ if os.path.exists(settings.JBI_BASEDIR):
+ shutil.rmtree(settings.JBI_BASEDIR)
+
+ @patch('repoapi.utils.dlfile')
+ def test_jbi_path_creation(self, dlfile):
+ param = self.get_defaults()
+ jbi = JenkinsBuildInfo.objects.create(**param)
+ base_path = os.path.join(settings.JBI_BASEDIR,
+ jbi.jobname, str(jbi.buildnumber))
+ self.assertTrue(os.path.exists(settings.JBI_BASEDIR), settings.JBI_BASEDIR)
+ self.assertTrue(os.path.exists(base_path))
+ path = os.path.join(base_path, 'console.txt')
+ url = JBI_CONSOLE_URL.format(
+ settings.JENKINS_URL,
+ jbi.jobname,
+ jbi.buildnumber
+ )
+ calls = [call(url, path),]
+ url = JBI_JOB_URL.format(
+ settings.JENKINS_URL,
+ jbi.jobname,
+ jbi.buildnumber
+ )
+ path = os.path.join(base_path, 'job.json')
+ calls.append(call(url, path))
+ dlfile.assert_has_calls(calls)
diff --git a/repoapi/utils.py b/repoapi/utils.py
index 08b8612..2b6aa8a 100644
--- a/repoapi/utils.py
+++ b/repoapi/utils.py
@@ -12,14 +12,20 @@
# You should have received a copy of the GNU General Public License along
# with this program. If not, see .
+from __future__ import absolute_import
-import urllib2
+from distutils.dir_util import mkpath
import logging
+import os
import subprocess
+import urllib2
from django.conf import settings
logger = logging.getLogger(__name__)
+JBI_CONSOLE_URL = "{}/job/{}/{}/consoleText"
+JBI_JOB_URL = "{}/job/{}/{}/api/json"
+
def executeAndReturnOutput(command, env=None):
proc = subprocess.Popen(command, stdout=subprocess.PIPE,
@@ -30,6 +36,16 @@ def executeAndReturnOutput(command, env=None):
return proc.returncode, stdoutdata, stderrdata
+def dlfile(url, path):
+ if settings.DEBUG:
+ logger.info("I would call %s", url)
+ else:
+ remote_file = urllib2.urlopen(url)
+ logger.debug("url:[%s]", url)
+ with open(path, "wb") as local_file:
+ local_file.write(remote_file.read())
+
+
def openurl(url):
req = urllib2.Request(url)
logger.debug("url:[%s]", url)
@@ -52,6 +68,35 @@ def jenkins_remove_ppa(repo):
openurl(url)
+def _jenkins_get(url, base_path, filename):
+ mkpath(base_path)
+ path = os.path.join(base_path, filename)
+ logger.info("url:[%s] path[%s]", url, path)
+ dlfile(url, path)
+
+
+def jenkins_get_console(jobname, buildnumber):
+ url = JBI_CONSOLE_URL.format(
+ settings.JENKINS_URL,
+ jobname,
+ buildnumber
+ )
+ base_path = os.path.join(settings.JBI_BASEDIR,
+ jobname, str(buildnumber))
+ _jenkins_get(url, base_path, 'console.txt')
+
+
+def jenkins_get_job(jobname, buildnumber):
+ url = JBI_JOB_URL.format(
+ settings.JENKINS_URL,
+ jobname,
+ buildnumber
+ )
+ base_path = os.path.join(settings.JBI_BASEDIR,
+ jobname, str(buildnumber))
+ _jenkins_get(url, base_path, 'job.json')
+
+
def workfront_note_send(_id, message):
command = [
"/usr/bin/workfront-post-note",
diff --git a/requirements/common.txt b/requirements/common.txt
index 6ccb5ee..e8bb5e6 100644
--- a/requirements/common.txt
+++ b/requirements/common.txt
@@ -8,3 +8,4 @@ markdown
django-filter
six
webassets
+django-celery
diff --git a/t/Dockerfile b/t/Dockerfile
index 32503e7..2da2fda 100644
--- a/t/Dockerfile
+++ b/t/Dockerfile
@@ -5,10 +5,10 @@ FROM docker.mgm.sipwise.com/sipwise-jessie:latest
# is updated with the current date. It will force refresh of all
# of the base images and things like `apt-get update` won't be using
# old cached versions when the Dockerfile is built.
-ENV REFRESHED_AT 2016-07-18
+ENV REFRESHED_AT 2016-07-19
RUN apt-get update
-RUN apt-get install --assume-yes python2.7 python2.7-dev python-distribute python-pip git
+RUN apt-get install --assume-yes python2.7 python2.7-dev python-distribute python-pip git screen
# Get pip to download and install requirements:
COPY dev.txt test.txt common.txt /tmp/
@@ -33,5 +33,11 @@ WORKDIR /code/
# ./t/testrunner
#
# Run django inside docker:
-# make run_dev
+# % pip install -r t/dev.txt && make run_dev
+#
+# We need a working rabbit server, so in another terminal:
+# % docker run --rm --hostname repoapi-rabbit --name repoapi-rabbit rabbitmq:3
+#
+# use screen to get a working worker in the background:
+# % make worker_dev
################################################################################
diff --git a/t/common.txt b/t/common.txt
index 6ccb5ee..e8bb5e6 100644
--- a/t/common.txt
+++ b/t/common.txt
@@ -8,3 +8,4 @@ markdown
django-filter
six
webassets
+django-celery