TT#1735 use celery to deal with tasks

* Dockerfile: add screen and refresh due new requirements
* create settings/test.py just for tests
  - celery in memory ( no need of server )
  - files in RESULTS
* repoapi/tasks.py for celery tasks
  - download console.txt and job.json from jenkins
    when a JenkinsBuildInfo is created
* repoapi.ini:
  - fire worker from uwsgi
  http://uwsgi-docs.readthedocs.io/en/latest/AttachingDaemons.html

Change-Id: Ib23c45194878a6fdbbe547058013e39516ea2d17

Makefile

@@ -13,8 +13,10 @@ venv_prod: requirements/prod.txt
 ###################################
 
 test:
-	./manage.py jenkins --enable-coverage --noinput --output-dir $(RESULTS) \
+	RESULTS=$(RESULTS) ./manage.py jenkins --enable-coverage --noinput --output-dir $(RESULTS) \
-		--settings="repoapi.settings.dev"
+		--settings="repoapi.settings.test"
+
+###################################
 
 deploy: venv_prod
 	source $(VAR_DIR)/venv_prod/bin/activate && \
@@ -32,6 +34,17 @@ run_dev:
 	IP=$(shell ip a show dev eth0 scope global | grep inet | awk '{print $$2}' | cut -d/ -f1); \
 	./manage.py runserver_plus $$IP:8000 --settings="repoapi.settings.dev"
 
+worker_dev:
+	./manage.py celery worker --loglevel=info --settings="repoapi.settings.dev"
+
+makemigrations_dev:
+	./manage.py makemigrations --settings="repoapi.settings.dev"
+
+migrate_dev:
+	./manage.py migrate --settings="repoapi.settings.dev"
+
+shell_dev:
+	./manage.py shell --settings="repoapi.settings.dev"
 ###################################
 
 # get rid of test files

repoapi.ini

@@ -13,3 +13,6 @@ home            = /var/lib/repoapi/venv_prod
 env = DJANGO_SETTINGS_MODULE=repoapi.settings.prod
 # spawn 20 uWSGI worker processes
 workers = 20
+# celery
+workerpid = /var/lib/repoapi/celery-worker.pid
+smart-attach-daemon = %(workerpid) %(home)/bin/python %(chdir)/manage.py celery worker --pidfile=%(workerpid) -l info

repoapi/__init__.py

@@ -0,0 +1,19 @@
+# Copyright (C) 2016 The Sipwise Team - http://sipwise.com
+
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+from __future__ import absolute_import
+
+# This will make sure the app is always imported when
+# Django starts so that shared_task will use this app.
+from .celery import app as celery_app  # noqa

repoapi/celery.py

@@ -0,0 +1,30 @@
+# Copyright (C) 2016 The Sipwise Team - http://sipwise.com
+
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+from __future__ import absolute_import
+
+import os
+from celery import Celery
+
+# set the default Django settings module for the 'celery' program.
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'repoapi.settings.prod')
+# pylint: disable=C0413
+from django.conf import settings  # noqa
+
+app = Celery('repoapi')
+
+# Using a string here means the worker will not have to
+# pickle the object when using Windows.
+app.config_from_object('django.conf:settings')
+app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)

repoapi/models.py

@@ -19,6 +19,7 @@ from django.db import models
 from django.db.models import signals
 from django.conf import settings
 from repoapi import utils
+from .tasks import get_jbi_files
 
 logger = logging.getLogger(__name__)
 workfront_re = re.compile(r"TT#(\d+)")
@@ -101,6 +102,12 @@ class JenkinsBuildInfo(models.Model):
         return "%s:%d[%s]" % (self.jobname,
                               self.buildnumber, self.tag)
 
+def jbi_manage(sender, **kwargs):
+    if kwargs["created"]:
+        instance = kwargs["instance"]
+        get_jbi_files.delay(instance.jobname, instance.buildnumber)
+
+signals.post_save.connect(jbi_manage, sender=JenkinsBuildInfo)
 
 class GerritRepoInfo(models.Model):
     param_ppa = models.CharField(max_length=50, null=False)

repoapi/settings/common.py

@@ -37,6 +37,7 @@ INSTALLED_APPS = [
     'rest_framework_swagger',
     'django_extensions',
     'django_assets',
+    'djcelery',
 ]
 
 MIDDLEWARE_CLASSES = (
@@ -135,3 +136,8 @@ LOGGING = {
 }
 
 JENKINS_TOKEN = "sipwise_jenkins_ci"
+
+CELERY_TASK_SERIALIZER = 'json'
+CELERY_RESULT_SERIALIZER = 'json'
+CELERY_ACCEPT_CONTENT = ['json']
+CELERY_RESULT_BACKEND = 'djcelery.backends.database:DatabaseBackend'

repoapi/settings/dev.py

@@ -15,49 +15,12 @@
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
 import os
-# pylint: disable=W0401,W0614
-from .common import *
 
-BASE_DIR = os.path.dirname(
+# pylint: disable=W0401,W0614,C0413
-    os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from .test import *
 
-# Quick-start development settings - unsuitable for production
+# celery
-# See https://docs.djangoproject.com/en/1.8/howto/deployment/checklist/
+BROKER_BACKEND = 'amqp'
-
+CELERY_ALWAYS_EAGER = False
-# SECURITY WARNING: keep the secret key used in production secret!
+BROKER_URL = 'amqp://guest:guest@rabbit'
-SECRET_KEY = ')+0h68-(g30hg1awc6!y65cwws6j^qd5=&pc2@h430=9x@bf%2'
+JBI_BASEDIR = os.path.join(BASE_DIR, 'jbi_files')
-
-# SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = True
-
-ALLOWED_HOSTS = []
-
-TESTING_APPS = [
-    'django_jenkins',
-]
-INSTALLED_APPS.extend(TESTING_APPS)
-INSTALLED_APPS.extend(PROJECT_APPS)
-
-# Database
-# https://docs.djangoproject.com/en/1.8/ref/settings/#databases
-
-DATABASES = {
-    'default': {
-        'ENGINE': 'django.db.backends.sqlite3',
-        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
-    }
-}
-
-# django-jenkins
-JENKINS_TASKS = (
-    'django_jenkins.tasks.run_pylint',
-    'django_jenkins.tasks.run_flake8',
-)
-PYLINT_RCFILE = 'pylint.cfg'
-
-LOGGING['loggers']['repoapi']['level'] = os.getenv('DJANGO_LOG_LEVEL', 'DEBUG')
-
-JENKINS_URL = "http://localhost"
-GERRIT_URL = "https://gerrit.local/{}"
-GITWEB_URL = "https://git.local/gitweb/?p={}.git;a=commit;h={}"
-WORKFRONT_CREDENTIALS = os.path.join(BASE_DIR, '.workfront.ini')

repoapi/settings/prod.py

@@ -56,3 +56,6 @@ GERRIT_URL = "https://gerrit.mgm.sipwise.com/{}"
 GITWEB_URL = "https://git.mgm.sipwise.com/gitweb/?p={}.git;a=commit;h={}"
 WORKFRONT_CREDENTIALS = os.path.join(BASE_DIR,
                                      '/etc/jenkins_jobs/workfront.ini')
+# celery
+BROKER_URL = 'amqp://guest:guest@localhost'
+JBI_BASEDIR = os.path.join(VAR_DIR, 'jbi_files')

repoapi/settings/test.py

@@ -0,0 +1,70 @@
+# Copyright (C) 2015 The Sipwise Team - http://sipwise.com
+
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
+import os
+# pylint: disable=W0401,W0614
+from .common import *
+
+BASE_DIR = os.path.dirname(
+    os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+os.environ.setdefault('RESULTS', '/tmp')
+RESULTS_DIR = os.environ['RESULTS']
+
+# Quick-start development settings - unsuitable for production
+# See https://docs.djangoproject.com/en/1.8/howto/deployment/checklist/
+
+# SECURITY WARNING: keep the secret key used in production secret!
+SECRET_KEY = ')+0h68-(g30hg1awc6!y65cwws6j^qd5=&pc2@h430=9x@bf%2'
+
+# SECURITY WARNING: don't run with debug turned on in production!
+DEBUG = True
+
+ALLOWED_HOSTS = []
+
+TESTING_APPS = [
+    'django_jenkins',
+]
+INSTALLED_APPS.extend(TESTING_APPS)
+INSTALLED_APPS.extend(PROJECT_APPS)
+
+# Database
+# https://docs.djangoproject.com/en/1.8/ref/settings/#databases
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.sqlite3',
+        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
+    }
+}
+
+# django-jenkins
+JENKINS_TASKS = (
+    'django_jenkins.tasks.run_pylint',
+    'django_jenkins.tasks.run_flake8',
+)
+PYLINT_RCFILE = 'pylint.cfg'
+
+DJANGO_LOG_LEVEL = 'DEBUG'
+
+JENKINS_URL = "http://localhost"
+GERRIT_URL = "https://gerrit.local/{}"
+GITWEB_URL = "https://git.local/gitweb/?p={}.git;a=commit;h={}"
+WORKFRONT_CREDENTIALS = os.path.join(BASE_DIR, '.workfront.ini')
+
+# celery
+BROKER_BACKEND = 'memory'
+CELERY_ALWAYS_EAGER = True
+JBI_BASEDIR = os.path.join(RESULTS_DIR, 'jbi_files')

repoapi/tasks.py

@@ -0,0 +1,24 @@
+# Copyright (C) 2016 The Sipwise Team - http://sipwise.com
+
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+from __future__ import absolute_import
+
+from celery import shared_task
+from .utils import jenkins_get_console, jenkins_get_job
+
+
+@shared_task
+def get_jbi_files(jobname, buildnumber):
+    jenkins_get_console(jobname, buildnumber)
+    jenkins_get_job(jobname, buildnumber)

repoapi/test/test_jbi_info.py

@@ -0,0 +1,74 @@
+# Copyright (C) 2015 The Sipwise Team - http://sipwise.com
+
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import shutil
+
+from django.test import TestCase
+from django.conf import settings
+from repoapi.models import JenkinsBuildInfo
+from repoapi.utils import JBI_CONSOLE_URL, JBI_JOB_URL
+from mock import patch, call
+
+class TestJBICelery(TestCase):
+
+    def get_defaults(self):
+        defaults = {
+            'tag': "edc90cd9-37f3-4613-9748-ed05a32031c2",
+            'projectname': "real-fake",
+            'jobname': "real-fake-gerrit",
+            'buildnumber': 1,
+            'result': "SUCCESS",
+            'job_url': "https://jenkins.mgm.sipwise.com/job/real-fake-gerrit/",
+            'param_tag': "none",
+            'param_branch': "master",
+            'param_release': "none",
+            'param_distribution': "wheezy",
+            'param_ppa': "gerrit_MT10339_review2054",
+            'git_commit_msg': "7fg4567 TT#0001 whatever",
+        }
+        return defaults
+
+    def setUp(self):
+        if not os.path.exists(settings.JBI_BASEDIR):
+            os.makedirs(settings.JBI_BASEDIR)
+
+    def tearDown(self):
+        if os.path.exists(settings.JBI_BASEDIR):
+            shutil.rmtree(settings.JBI_BASEDIR)
+
+    @patch('repoapi.utils.dlfile')
+    def test_jbi_path_creation(self, dlfile):
+        param = self.get_defaults()
+        jbi = JenkinsBuildInfo.objects.create(**param)
+        base_path = os.path.join(settings.JBI_BASEDIR,
+                                 jbi.jobname, str(jbi.buildnumber))
+        self.assertTrue(os.path.exists(settings.JBI_BASEDIR), settings.JBI_BASEDIR)
+        self.assertTrue(os.path.exists(base_path))
+        path = os.path.join(base_path, 'console.txt')
+        url = JBI_CONSOLE_URL.format(
+            settings.JENKINS_URL,
+            jbi.jobname,
+            jbi.buildnumber
+        )
+        calls = [call(url, path),]
+        url = JBI_JOB_URL.format(
+            settings.JENKINS_URL,
+            jbi.jobname,
+            jbi.buildnumber
+        )
+        path = os.path.join(base_path, 'job.json')
+        calls.append(call(url, path))
+        dlfile.assert_has_calls(calls)

repoapi/utils.py

@@ -12,14 +12,20 @@
 
 # You should have received a copy of the GNU General Public License along
 # with this program.  If not, see <http://www.gnu.org/licenses/>.
+from __future__ import absolute_import
 
-import urllib2
+from distutils.dir_util import mkpath
 import logging
+import os
 import subprocess
+import urllib2
 from django.conf import settings
 
 logger = logging.getLogger(__name__)
 
+JBI_CONSOLE_URL = "{}/job/{}/{}/consoleText"
+JBI_JOB_URL = "{}/job/{}/{}/api/json"
+
 
 def executeAndReturnOutput(command, env=None):
     proc = subprocess.Popen(command, stdout=subprocess.PIPE,
@@ -30,6 +36,16 @@ def executeAndReturnOutput(command, env=None):
     return proc.returncode, stdoutdata, stderrdata
 
 
+def dlfile(url, path):
+    if settings.DEBUG:
+        logger.info("I would call %s", url)
+    else:
+        remote_file = urllib2.urlopen(url)
+        logger.debug("url:[%s]", url)
+        with open(path, "wb") as local_file:
+            local_file.write(remote_file.read())
+
+
 def openurl(url):
     req = urllib2.Request(url)
     logger.debug("url:[%s]", url)
@@ -52,6 +68,35 @@ def jenkins_remove_ppa(repo):
         openurl(url)
 
 
+def _jenkins_get(url, base_path, filename):
+    mkpath(base_path)
+    path = os.path.join(base_path, filename)
+    logger.info("url:[%s] path[%s]", url, path)
+    dlfile(url, path)
+
+
+def jenkins_get_console(jobname, buildnumber):
+    url = JBI_CONSOLE_URL.format(
+        settings.JENKINS_URL,
+        jobname,
+        buildnumber
+    )
+    base_path = os.path.join(settings.JBI_BASEDIR,
+                             jobname, str(buildnumber))
+    _jenkins_get(url, base_path, 'console.txt')
+
+
+def jenkins_get_job(jobname, buildnumber):
+    url = JBI_JOB_URL.format(
+        settings.JENKINS_URL,
+        jobname,
+        buildnumber
+    )
+    base_path = os.path.join(settings.JBI_BASEDIR,
+                             jobname, str(buildnumber))
+    _jenkins_get(url, base_path, 'job.json')
+
+
 def workfront_note_send(_id, message):
     command = [
         "/usr/bin/workfront-post-note",

requirements/common.txt

@@ -8,3 +8,4 @@ markdown
 django-filter
 six
 webassets
+django-celery

t/Dockerfile

@@ -5,10 +5,10 @@ FROM docker.mgm.sipwise.com/sipwise-jessie:latest
 # is updated with the current date. It will force refresh of all
 # of the base images and things like `apt-get update` won't be using
 # old cached versions when the Dockerfile is built.
-ENV REFRESHED_AT 2016-07-18
+ENV REFRESHED_AT 2016-07-19
 
 RUN apt-get update
-RUN apt-get install --assume-yes python2.7 python2.7-dev python-distribute python-pip git
+RUN apt-get install --assume-yes python2.7 python2.7-dev python-distribute python-pip git screen
 
 # Get pip to download and install requirements:
 COPY dev.txt test.txt common.txt /tmp/
@@ -33,5 +33,11 @@ WORKDIR /code/
 #   ./t/testrunner
 #
 # Run django inside docker:
-#   make run_dev
+# % pip install -r t/dev.txt && make run_dev
+#
+# We need a working rabbit server, so in another terminal:
+# % docker run --rm --hostname repoapi-rabbit --name repoapi-rabbit rabbitmq:3
+#
+# use screen to get a working worker in the background:
+# % make worker_dev
 ################################################################################

t/common.txt

@@ -8,3 +8,4 @@ markdown
 django-filter
 six
 webassets
+django-celery