TT#15305 build release REST API

* using https://github.com/manosim/django-rest-framework-api-key
  for Permissions. Will generate an Auth KEY for jenkins to be able
  to trigger build from there using build REST API
* store the info needed to trigger builds for a set of projects for
  a release
  - build is a list of projects to be built with branch or tag as orig
    and release as destination for a debian distribution
* added release_uuid to JenkinsJobInfo model
  - release_uuid is the way to group project builds for a release
  - tag works for group of jobs for a project

Change-Id: Iebeaa54308c3e569a167f55d98c184b52248af8a

build/admin.py

@@ -0,0 +1,22 @@
+# Copyright (C) 2017 The Sipwise Team - http://sipwise.com
+
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from django.contrib import admin
+from . import models
+
+
+@admin.register(models.BuildRelease)
+class JenkinsBuildInfoAdmin(admin.ModelAdmin):
+    list_filter = ('release',)

build/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class ReleaseConfig(AppConfig):
+    name = 'build'

build/fixtures/test_models.json

@@ -0,0 +1,15 @@
+[
+{
+  "model": "build.buildrelease",
+  "pk": 1,
+  "fields": {
+    "uuid": "dbe569f7-eab6-4532-a6d1-d31fb559649b",
+    "start_date": "2017-07-14T07:55:11.714Z",
+    "tag": "",
+    "branch": "mr5.5",
+    "release": "release-mr5.5",
+    "distribution": "auto",
+    "projects": "kamailio,lua-ngcp-kamailio,ngcp-panel"
+  }
+}
+]

build/migrations/0001_initial.py

@@ -0,0 +1,32 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9 on 2017-06-09 19:28
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='BuildRelease',
+            fields=[
+                ('id', models.AutoField(auto_created=True,
+                                        primary_key=True,
+                                        serialize=False,
+                                        verbose_name='ID')),
+                ('uuid', models.CharField(max_length=64, unique=True)),
+                ('start_date', models.DateTimeField(auto_now_add=True)),
+                ('tag', models.CharField(max_length=50, null=True)),
+                ('branch', models.CharField(max_length=50)),
+                ('release', models.CharField(db_index=True, max_length=50)),
+                ('distribution', models.CharField(max_length=50)),
+                ('projects', models.TextField(null=False)),
+            ],
+        ),
+    ]

build/models/__init__.py

@@ -0,0 +1,27 @@
+# Copyright (C) 2017 The Sipwise Team - http://sipwise.com
+
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from django.db.models import signals
+from .br import BuildRelease
+from build.tasks import build_release
+
+
+def br_manage(sender, **kwargs):
+    if kwargs["created"]:
+        instance = kwargs["instance"]
+        build_release.delay(instance.uuid)
+
+post_save = signals.post_save.connect
+post_save(br_manage, sender=BuildRelease)

build/models/br.py

@@ -0,0 +1,36 @@
+# Copyright (C) 2017 The Sipwise Team - http://sipwise.com
+
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+import logging
+from django.db import models
+
+logger = logging.getLogger(__name__)
+
+
+class BuildRelease(models.Model):
+    uuid = models.CharField(max_length=64, unique=True, null=False)
+    start_date = models.DateTimeField(auto_now_add=True)
+    tag = models.CharField(max_length=50, null=True, blank=True)
+    branch = models.CharField(max_length=50, null=False)
+    release = models.CharField(max_length=50, null=False,
+                               db_index=True)
+    distribution = models.CharField(max_length=50, null=False)
+    projects = models.TextField(null=False)
+
+    def __str__(self):
+        return "%s[%s]" % (self.release, self.uuid)
+
+    @property
+    def projects_list(self):
+        return [x.strip() for x in self.projects.split(',')]

build/serializers.py

@@ -0,0 +1,31 @@
+# Copyright (C) 2017 The Sipwise Team - http://sipwise.com
+
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from rest_framework import serializers
+from . import models
+
+
+class BuildReleaseSerializer(serializers.ModelSerializer):
+
+    class Meta:
+        model = models.BuildRelease
+        fields = '__all__'
+
+    def validate_projects(self, value):
+        projects = [x.strip() for x in value.split(',')]
+        if len(projects) <= 0:
+            raise serializers.ValidationError(
+                "projects is not a list of coma separate elements")
+        return ','.join(projects)

build/tasks.py

@@ -0,0 +1,36 @@
+# Copyright (C) 2017 The Sipwise Team - http://sipwise.com
+
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+from __future__ import absolute_import
+
+import logging
+from celery import shared_task
+from build.utils import trigger_build
+from build.models.br import BuildRelease
+
+logger = logging.getLogger(__name__)
+
+
+@shared_task(ignore_result=True)
+def build_release(uuid):
+    instance = BuildRelease.objects.get(uuid=uuid)
+    if instance.tag:
+        branch_or_tag = "tag/%s" % instance.tag
+    else:
+        branch_or_tag = "branch/%s" % instance.branch
+    for project in instance.projects_list:
+        url = trigger_build(project, instance.uuid, instance.release,
+                            trigger_branch_or_tag=branch_or_tag,
+                            trigger_distribution=instance.distribution)
+        logger.debug("%s triggered" % url)

build/test/test_models.py

@@ -0,0 +1,28 @@
+# Copyright (C) 2017 The Sipwise Team - http://sipwise.com
+
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+
+# You should have received a copy of the GNU General Public License along
+# with this prograproj.  If not, see <http://www.gnu.org/licenses/>.
+
+from django.test import TestCase, override_settings
+from build.models import BuildRelease
+
+
+@override_settings(DEBUG=True)
+class BuildReleaseTestCase(TestCase):
+    fixtures = ['test_models', ]
+
+    def test_projects_list(self):
+        build = BuildRelease.objects.get(
+            uuid="dbe569f7-eab6-4532-a6d1-d31fb559649b")
+        self.assertItemsEqual(build.projects_list,
+                              ['kamailio', 'lua-ngcp-kamailio', 'ngcp-panel'])

build/test/test_rest.py

@@ -0,0 +1,88 @@
+# Copyright (C) 2017 The Sipwise Team - http://sipwise.com
+
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from django.core.urlresolvers import reverse
+from rest_framework import status
+from rest_framework.test import APITestCase
+from django.test import override_settings
+from repoapi.test.base import BaseTest
+from rest_framework_api_key.models import APIKey
+from rest_framework_api_key.helpers import generate_key
+
+
+class APIAuthenticatedTestCase(BaseTest, APITestCase):
+
+    APP_NAME = 'Project Tests'
+
+    def setUp(self):
+        super(APIAuthenticatedTestCase, self).setUp()
+        self.app_key = APIKey.objects.create(
+            name=self.APP_NAME, key=generate_key())
+        self.client.credentials(HTTP_API_KEY=self.app_key.key)
+
+
+@override_settings(DEBUG=True)
+class TestRest(APIAuthenticatedTestCase):
+
+    def setUp(self):
+        super(TestRest, self).setUp()
+        self.url = reverse('build:list')
+
+    def test_trunk_empty_projects(self):
+        data = {
+            'uuid': 'fake_uuid',
+            'tag': None,
+            'branch': 'master',
+            'release': 'release-trunk-stretch',
+            'distribution': 'stretch',
+            'projects': None
+        }
+        response = self.client.post(self.url, data, format='json')
+        self.assertNotEqual(response.status_code, status.HTTP_201_CREATED)
+
+    def test_trunk_wrong_projects(self):
+        data = {
+            'uuid': 'fake_uuid',
+            'tag': None,
+            'branch': 'master',
+            'release': 'release-trunk-stretch',
+            'distribution': 'stretch',
+            'projects': ''
+        }
+        response = self.client.post(self.url, data, format='json')
+        self.assertNotEqual(response.status_code, status.HTTP_201_CREATED)
+
+    def test_trunk(self):
+        data = {
+            'uuid': 'fake_uuid',
+            'tag': None,
+            'branch': 'master',
+            'release': 'release-trunk-stretch',
+            'distribution': 'stretch',
+            'projects': '  kamailio , sems'
+        }
+        data_all = {
+            'uuid': 'fake_uuid',
+            'tag': None,
+            'branch': 'master',
+            'release': 'release-trunk-stretch',
+            'distribution': 'stretch',
+            'projects': 'kamailio,sems'
+        }
+        response = self.client.post(self.url, data, format='json')
+        self.assertEqual(response.status_code, status.HTTP_201_CREATED)
+        data_all['start_date'] = response.data['start_date']
+        data_all['id'] = response.data['id']
+        self.assertItemsEqual(response.data, data_all)

build/urls.py

@@ -0,0 +1,24 @@
+# Copyright (C) 2017 The Sipwise Team - http://sipwise.com
+
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from django.conf.urls import url
+from . import views
+
+urlpatterns = [
+    url(r'^$', views.BuildReleaseList.as_view(),
+        name='list'),
+    url(r'^(?P<pk>[0-9]+)/?$', views.BuildReleaseDetail.as_view(),
+        name='detail'),
+]

build/utils.py

@@ -0,0 +1,60 @@
+# Copyright (C) 2017 The Sipwise Team - http://sipwise.com
+
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import logging
+import urllib
+import uuid
+from django.conf import settings
+from repoapi.utils import openurl
+
+logger = logging.getLogger(__name__)
+
+project_url = ("{base}/job/{job}/buildWithParameters?"
+               "token={token}&cause={cause}&branch={branch}&"
+               "tag={tag}&release={release}&"
+               "distribution={distribution}&uuid={uuid}&"
+               "release_uuid={release_uuid}")
+
+
+def trigger_build(project, release_uuid, trigger_release=None,
+                  trigger_branch_or_tag=None,
+                  trigger_distribution=None):
+    params = {
+        'base': settings.JENKINS_URL,
+        'job': project,
+        'token': urllib.quote(settings.JENKINS_TOKEN),
+        'cause': urllib.quote(trigger_release),
+        'branch': 'none',
+        'tag': 'none',
+        'release': urllib.quote(trigger_release),
+        'distribution': urllib.quote(trigger_distribution),
+        'uuid': uuid.uuid4(),
+        'release_uuid': release_uuid,
+    }
+    if trigger_branch_or_tag.startswith("tag/"):
+        tag = trigger_branch_or_tag.split("tag/")[1]
+        params['tag'] = urllib.quote(tag)
+    elif trigger_branch_or_tag.startswith("branch/"):
+        branch = trigger_branch_or_tag.split("branch/")[1]
+        params['branch'] = urllib.quote(branch)
+    else:
+        params['branch'] = urllib.quote(trigger_branch_or_tag)
+
+    url = project_url.format(**params)
+    if settings.DEBUG:
+        logger.debug("Debug mode, would trigger: %s", url)
+    else:
+        openurl(url)
+    return "{base}/job/{job}/".format(**params)

build/views.py

@@ -0,0 +1,41 @@
+# Copyright (C) 2017 The Sipwise Team - http://sipwise.com
+
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from django.conf import settings
+import django_filters
+from rest_framework import generics
+from rest_framework_api_key.permissions import HasAPIAccess
+from . import models, serializers
+
+
+class BuildReleaseFilter(django_filters.FilterSet):
+
+    class Meta:
+        model = models.BuildRelease
+        fields = ['release', 'start_date']
+        order_by = ['-start_date', ]
+
+
+class BuildReleaseList(generics.ListCreateAPIView):
+    if settings.BUILD_KEY_AUTH:
+        permission_classes = (HasAPIAccess,)
+    queryset = models.BuildRelease.objects.all()
+    serializer_class = serializers.BuildReleaseSerializer
+    filter_class = BuildReleaseFilter
+
+
+class BuildReleaseDetail(generics.RetrieveAPIView):
+    queryset = models.BuildRelease.objects.all()
+    serializer_class = serializers.BuildReleaseSerializer

debian/install

@@ -1,4 +1,5 @@
 Makefile usr/share/repoapi
+build usr/share/repoapi
 hotfix usr/share/repoapi
 manage.py usr/share/repoapi
 panel usr/share/repoapi

repoapi/fixtures/test_model_queries.json

@@ -11,6 +11,7 @@
             "param_distribution": "wheezy",
             "param_ppa": null,
             "param_release": "mr3.1-fake",
+            "param_release_uuid": null,
             "param_tag": null,
             "projectname": "fake",
             "jobname": "fake-source",
@@ -33,6 +34,7 @@
             "param_distribution": "wheezy",
             "param_ppa": null,
             "param_release": "mr3.1-fake",
+            "param_release_uuid": null,
             "param_tag": null,
             "projectname": "fake",
             "jobname": "fake-get-code",
@@ -55,6 +57,7 @@
             "param_distribution": "wheezy",
             "param_ppa": null,
             "param_release": "mr3.1-fake",
+            "param_release_uuid": null,
             "param_tag": null,
             "projectname": "fake",
             "jobname": "fake-source-tests",
@@ -77,6 +80,7 @@
             "param_distribution": "wheezy",
             "param_ppa": null,
             "param_release": "mr3.1-fake",
+            "param_release_uuid": null,
             "param_tag": null,
             "projectname": "fake",
             "jobname": "fake-get-code",
@@ -99,6 +103,7 @@
             "param_distribution": "wheezy",
             "param_ppa": null,
             "param_release": "mr3.1-fake",
+            "param_release_uuid": null,
             "param_tag": null,
             "projectname": "release-update-deb-symlinks",
             "jobname": "release-update-deb-symlinks",

repoapi/migrations/0008_auto_20170609_1954.py

@@ -0,0 +1,28 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9 on 2017-06-09 19:54
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('repoapi', '0007_auto_20160718_1136'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='jenkinsbuildinfo',
+            name='param_release_uuid',
+            field=models.CharField(max_length=64, null=True),
+        ),
+        migrations.AlterIndexTogether(
+            name='jenkinsbuildinfo',
+            index_together=set([
+                ('param_release_uuid', 'tag'),
+                ('param_release', 'projectname', 'tag'),
+                ('param_release', 'projectname')
+            ]),
+        ),
+    ]

repoapi/models/jbi.py

@@ -152,6 +152,7 @@ class JenkinsBuildInfo(models.Model):
     param_branch = models.CharField(max_length=50, null=True)
     param_release = models.CharField(max_length=50, null=True,
                                      db_index=True)
+    param_release_uuid = models.CharField(max_length=64, null=True)
     param_distribution = models.CharField(max_length=50, null=True)
     param_ppa = models.CharField(max_length=50, null=True)
 
@@ -164,6 +165,7 @@ class JenkinsBuildInfo(models.Model):
         index_together = [
             ["param_release", "projectname"],
             ["param_release", "projectname", "tag"],
+            ["param_release_uuid", "tag"],
         ]
 
     def is_job_url_allowed(self):
@@ -174,8 +176,8 @@ class JenkinsBuildInfo(models.Model):
         return False
 
     def __str__(self):
-        return "%s:%d[%s]" % (self.jobname,
-                              self.buildnumber, self.tag)
+        return "[%s]%s:%d[%s]" % (self.param_release_uuid, self.jobname,
+                                  self.buildnumber, self.tag)
 
 
 def jbi_manage(sender, **kwargs):

repoapi/settings/common.py

@@ -26,6 +26,7 @@ PROJECT_APPS = [
     'hotfix',
     'panel',
     'release_dashboard',
+    'build',
 ]
 
 INSTALLED_APPS = [
@@ -36,6 +37,7 @@ INSTALLED_APPS = [
     'django.contrib.messages',
     'django.contrib.staticfiles',
     'rest_framework',
+    'rest_framework_api_key',
     'rest_framework_swagger',
     'django_extensions',
     'django_assets',

repoapi/settings/dev.py

@@ -22,6 +22,9 @@ from .test import *
 LOGGING['loggers']['release_dashboard']['level'] = \
     os.getenv('DJANGO_LOG_LEVEL', 'DEBUG')
 
+# build app
+BUILD_KEY_AUTH = False
+
 # celery
 BROKER_BACKEND = 'amqp'
 CELERY_ALWAYS_EAGER = False

repoapi/settings/prod.py

@@ -72,6 +72,9 @@ WORKFRONT_CREDENTIALS = os.path.join(BASE_DIR,
                                      '/etc/jenkins_jobs/workfront.ini')
 WORKFRONT_NOTE = True
 
+# build app
+BUILD_KEY_AUTH = True
+
 # celery
 BROKER_URL = server_config.get('server', 'BROKER_URL')
 JBI_BASEDIR = os.path.join(VAR_DIR, 'jbi_files')

repoapi/settings/test.py

@@ -94,6 +94,9 @@ DOCKER_IMAGES = {
     ],
 }
 
+# build app
+BUILD_KEY_AUTH = True
+
 # celery
 BROKER_BACKEND = 'memory'
 CELERY_ALWAYS_EAGER = True

repoapi/test/test_rest.py

@@ -47,6 +47,7 @@ class TestRest(BaseTest, APITestCase):
             "param_tag": None,
             "param_branch": None,
             "param_release": None,
+            "param_release_uuid": None,
             "param_distribution": None,
             "param_ppa": None,
             "repo_name": None,

repoapi/urls.py

@@ -55,6 +55,8 @@ api_patterns = [
     url(r'^docker/tag/(?P<pk>[0-9]+)/$',
         docker.DockerTagDetail.as_view(),
         name='dockertag-detail'),
+    url(r'^build/',
+        include('build.urls', namespace='build')),
 ]
 
 api_patterns = format_suffix_patterns(api_patterns)

repoapi/views.py

@@ -39,6 +39,8 @@ def api_root(request, _format=None):
                                     request=request, format=_format),
         'release': reverse('release-list',
                            request=request, format=_format),
+        'build': reverse('build:list',
+                         request=request, format=_format),
     })
 
 

requirements/common.txt

@@ -3,6 +3,7 @@ django-extensions
 yuicompressor
 django-assets
 djangorestframework==3.4
+drfapikey
 django-rest-swagger
 markdown
 django-filter

t/Dockerfile

@@ -5,7 +5,7 @@ FROM docker.mgm.sipwise.com/sipwise-jessie:latest
 # is updated with the current date. It will force refresh of all
 # of the base images and things like `apt-get update` won't be using
 # old cached versions when the Dockerfile is built.
-ENV REFRESHED_AT 2017-06-21
+ENV REFRESHED_AT 2017-06-28
 
 RUN apt-get update
 RUN apt-get install --assume-yes python2.7 python2.7-dev \