From 16a3bcc4e08247f077cedf8655e22e6feabdf118 Mon Sep 17 00:00:00 2001 From: Victor Seva Date: Fri, 4 Dec 2015 10:06:07 +0100 Subject: [PATCH] MT#3762 release_dashboard base * fix duplicate rule in Makefile * Implement just hotfix for now * /var/lib/repoapi/gerrit.ini needed [gerrit REST HTTP password] Change-Id: I80286e9d97846cf037566eaca74079362fa6e54a --- Makefile | 2 +- debian/install | 1 + debian/rules | 3 + panel/templates/panel/base.html | 4 +- release_dashboard/__init__.py | 0 release_dashboard/admin.py | 22 +++ release_dashboard/migrations/0001_initial.py | 31 ++++ release_dashboard/migrations/__init__.py | 0 release_dashboard/models.py | 88 ++++++++++ .../static/release_dashboard/js/build.js | 27 +++ .../static/release_dashboard/js/csrf.js | 31 ++++ .../static/release_dashboard/js/hotfix.js | 61 +++++++ .../release_dashboard/js/refresh_info.js | 66 ++++++++ release_dashboard/tasks.py | 38 +++++ .../templates/release_dashboard/base.html | 4 + .../templates/release_dashboard/build.html | 79 +++++++++ .../release_dashboard/build_deps.html | 79 +++++++++ .../templates/release_dashboard/hotfix.html | 58 +++++++ .../templates/release_dashboard/index.html | 36 ++++ .../templates/release_dashboard/refresh.html | 53 ++++++ release_dashboard/test/__init__.py | 0 release_dashboard/test/test_models.py | 155 ++++++++++++++++++ release_dashboard/urls.py | 28 ++++ release_dashboard/utils.py | 87 ++++++++++ release_dashboard/views.py | 142 ++++++++++++++++ repoapi/settings/common.py | 98 +++++++++++ repoapi/settings/dev.py | 3 + repoapi/settings/prod.py | 7 + repoapi/settings/test.py | 2 + repoapi/urls.py | 2 + 30 files changed, 1205 insertions(+), 2 deletions(-) create mode 100644 release_dashboard/__init__.py create mode 100644 release_dashboard/admin.py create mode 100644 release_dashboard/migrations/0001_initial.py create mode 100644 release_dashboard/migrations/__init__.py create mode 100644 release_dashboard/models.py create mode 100644 release_dashboard/static/release_dashboard/js/build.js create mode 100644 release_dashboard/static/release_dashboard/js/csrf.js create mode 100644 release_dashboard/static/release_dashboard/js/hotfix.js create mode 100644 release_dashboard/static/release_dashboard/js/refresh_info.js create mode 100644 release_dashboard/tasks.py create mode 100644 release_dashboard/templates/release_dashboard/base.html create mode 100644 release_dashboard/templates/release_dashboard/build.html create mode 100644 release_dashboard/templates/release_dashboard/build_deps.html create mode 100644 release_dashboard/templates/release_dashboard/hotfix.html create mode 100644 release_dashboard/templates/release_dashboard/index.html create mode 100644 release_dashboard/templates/release_dashboard/refresh.html create mode 100644 release_dashboard/test/__init__.py create mode 100644 release_dashboard/test/test_models.py create mode 100644 release_dashboard/urls.py create mode 100644 release_dashboard/utils.py create mode 100644 release_dashboard/views.py diff --git a/Makefile b/Makefile index 443aaa3..6115004 100644 --- a/Makefile +++ b/Makefile @@ -28,7 +28,7 @@ migrate: venv_prod ./manage.py migrate --settings="repoapi.settings.prod" chown www-data:www-data $(VAR_DIR)/db.sqlite3 -shell_dev: venv_prod +shell: venv_prod source $(VAR_DIR)/venv_prod/bin/activate && \ ./manage.py shell --settings="repoapi.settings.prod" diff --git a/debian/install b/debian/install index 799b70b..2a21d7e 100644 --- a/debian/install +++ b/debian/install @@ -2,6 +2,7 @@ hotfix usr/share/repoapi Makefile usr/share/repoapi manage.py usr/share/repoapi panel usr/share/repoapi +release_dashboard usr/share/repoapi repoapi usr/share/repoapi repoapi.ini etc/uwsgi/apps-available requirements usr/share/repoapi diff --git a/debian/rules b/debian/rules index aab0089..7175d65 100755 --- a/debian/rules +++ b/debian/rules @@ -16,6 +16,9 @@ override_dh_auto_test: override_dh_auto_install: echo "fakesecretkey" > .secret_key + echo "[gerrit]" > gerrit.ini + echo "HTTP_USER=fake" >> gerrit.ini + echo "HTTP_PASSWD=fakeHTTPpass" >> gerrit.ini VAR_DIR=$(shell pwd) make deploy make clean diff --git a/panel/templates/panel/base.html b/panel/templates/panel/base.html index 597276a..5437472 100644 --- a/panel/templates/panel/base.html +++ b/panel/templates/panel/base.html @@ -31,7 +31,9 @@ diff --git a/release_dashboard/__init__.py b/release_dashboard/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/release_dashboard/admin.py b/release_dashboard/admin.py new file mode 100644 index 0000000..b930db9 --- /dev/null +++ b/release_dashboard/admin.py @@ -0,0 +1,22 @@ +# Copyright (C) 2016 The Sipwise Team - http://sipwise.com + +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. + +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. + +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . + +from django.contrib import admin +from . import models + + +@admin.register(models.Project) +class ProjectAdmin(admin.ModelAdmin): + pass diff --git a/release_dashboard/migrations/0001_initial.py b/release_dashboard/migrations/0001_initial.py new file mode 100644 index 0000000..7c3c145 --- /dev/null +++ b/release_dashboard/migrations/0001_initial.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9 on 2016-09-23 11:48 +from __future__ import unicode_literals + +from django.db import migrations, models +from django_extensions.db.fields import ModificationDateTimeField +from django_extensions.db.fields.json import JSONField + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='Project', + fields=[ + ('id', models.AutoField( + auto_created=True, primary_key=True, + serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=50, unique=True)), + ('json_tags', JSONField(null=True)), + ('json_branches', JSONField(null=True)), + ('modified', ModificationDateTimeField( + auto_now=True, null=True)), + ], + ), + ] diff --git a/release_dashboard/migrations/__init__.py b/release_dashboard/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/release_dashboard/models.py b/release_dashboard/models.py new file mode 100644 index 0000000..2fdeda6 --- /dev/null +++ b/release_dashboard/models.py @@ -0,0 +1,88 @@ +# Copyright (C) 2016 The Sipwise Team - http://sipwise.com + +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. + +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. + +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +import logging +import re +import json +from django.db import models +from django_extensions.db.fields.json import JSONField +from django_extensions.db.fields import ModificationDateTimeField + +logger = logging.getLogger(__name__) + + +class Project(models.Model): + name = models.CharField(max_length=50, unique=True, null=False) + json_tags = JSONField(null=True) + json_branches = JSONField(null=True) + modified = ModificationDateTimeField(null=True) + + @classmethod + def _filter_values(cls, values, val_ok_filter, regex=None): + res = set() + + for value in values: + logger.debug("ref[%s]", value["ref"]) + match = re.search(val_ok_filter, value["ref"]) + if match: + val_ok = match.group(1) + if regex is not None: + if re.search(regex, val_ok): + res.add(val_ok) + logger.debug("val_ok[%s] regex", val_ok) + else: + logger.debug("val_ok[%s]", val_ok) + res.add(val_ok) + return sorted(res, reverse=True) + + @classmethod + def _get_filtered_json(cls, text): + """gerrit responds with malformed json + https://gerrit-review.googlesource.com/Documentation/rest-api.html#output + """ + logging.debug("json[:5]: %s", text[:5]) + return json.loads(text[5:]) + + def __str__(self): + return self.name + + @property + def tags(self): + return Project._filter_values(self.json_tags, '^refs/tags/(.+)$') + + @tags.setter + def tags(self, value): + self.json_tags = Project._get_filtered_json(value) + + @property + def branches(self): + return Project._filter_values(self.json_branches, '^refs/heads/(.+)$') + + @branches.setter + def branches(self, value): + self.json_branches = Project._get_filtered_json(value) + + def filter_tags(self, regex): + return Project._filter_values(self.json_tags, + '^refs/tags/(.+)$', regex) + + def filter_branches(self, regex): + return Project._filter_values(self.json_branches, + '^refs/heads/(.+)$', regex) + + def branches_mrXX(self): + return self.filter_branches(r'^mr[0-9]+\.[0-9]+$') + + def branches_mrXXX(self): + return self.filter_branches(r'^mr[0-9]+\.[0-9]+\.[0-9]+$') diff --git a/release_dashboard/static/release_dashboard/js/build.js b/release_dashboard/static/release_dashboard/js/build.js new file mode 100644 index 0000000..2ebc996 --- /dev/null +++ b/release_dashboard/static/release_dashboard/js/build.js @@ -0,0 +1,27 @@ +/** + * + */ +$('select#common_select').change(function() { + var selected_version = $('select#common_select option:selected').val(); + var ignored = $('.version option[value="ignore"]' ); + var version = ""; + if(selected_version.match(/^branch/)) { + version = selected_version.replace( + /^branch\/(mr[0-9]+\.[0-9]+(\.[0-9]+)?)$/g, "$1"); + } + else { + version = selected_version.replace( + /^tag\/(mr[0-9]+\.[0-9]+\.[0-9]+)(\.[0-9]+)?$/g, "$1"); + } + // set ignored for all + ignored.prop('selected', true); + ignored.each(function(){ $(this).change(); }); + $('tr.repo option[value="ignore"]').closest('tr').children('td,th').css('background-color','#F78181'); + var selected = $('.version option[value="'+ selected_version + '"]' ) + selected.prop('selected', true); + selected.each(function(){ $(this).change(); }); + $('tr.repo option[value="'+ selected_version + '"]').closest('tr').children('td,th').css('background-color','white'); + var text = "Selected " + selected.length + " of " + ignored.length; + $('#select_text_info').text(text); + $('input#version_release').val("release-" + version); +}); diff --git a/release_dashboard/static/release_dashboard/js/csrf.js b/release_dashboard/static/release_dashboard/js/csrf.js new file mode 100644 index 0000000..4be544f --- /dev/null +++ b/release_dashboard/static/release_dashboard/js/csrf.js @@ -0,0 +1,31 @@ +/** + * + * + */ +function csrfSafeMethod(method) { + // these HTTP methods do not require CSRF protection + return (/^(GET|HEAD|OPTIONS|TRACE)$/.test(method)); +} + +function csrftokenFunc(xhr, settings) { + if (!csrfSafeMethod(settings.type) && !this.crossDomain) { + xhr.setRequestHeader("X-CSRFToken", csrftoken); + } +} + +function getCookie(name) { + var cookieValue = null; + if (document.cookie && document.cookie !== '') { + var cookies = document.cookie.split(';'); + for (var i = 0; i < cookies.length; i++) { + var cookie = jQuery.trim(cookies[i]); + // Does this cookie string begin with the name we want? + if (cookie.substring(0, name.length + 1) === (name + '=')) { + cookieValue = decodeURIComponent(cookie.substring(name.length + 1)); + break; + } + } + } + return cookieValue; +} +var csrftoken = getCookie('csrftoken'); diff --git a/release_dashboard/static/release_dashboard/js/hotfix.js b/release_dashboard/static/release_dashboard/js/hotfix.js new file mode 100644 index 0000000..2d88c5e --- /dev/null +++ b/release_dashboard/static/release_dashboard/js/hotfix.js @@ -0,0 +1,61 @@ +/** + * + * + */ +$('button.hotfix').click(function(e){ + // don't send the form + e.preventDefault(); + var button = $(this); + var id = button.attr('id').replace('hotfix_',''); + var branch = $('select#version_' + id + ' option:selected').val().replace('branch/', ''); + var repo = id; + var span = $('span#hotfix_error_' + id); + var push = $('select#push_' + id + ' option:selected').val(); + + $.ajax({ + url: branch + '/' + repo + '/', + type: 'POST', + data: JSON.stringify({push: push }), + contentType: "application/json; charset=utf-8", + dataType: "json", + success: successFunc, + error: errorFunc, + beforeSend: csrftokenFunc + }); + + button.attr("disabled", "disabled"); + span.html('processing'); + span.show(); + + function successFunc(data, status) { + span.html(''); + span.append('Done'); + button.removeAttr("disabled"); + } + + function errorFunc(jqXHR, status, error) { + span.html(error); + button.removeAttr("disabled"); + } +}); + +$('td.version > select').change(function() { + var id = $(this).attr('id').replace('version_',''); + var version = $(this).val(); + var button = $('button#hotfix_' + id); + var span = $('span#hotfix_error_' + id); + + if (version.match(/^branch\/mr[0-9]+\.[0-9]+\.[0-9]+$/)) { + button.html("Release hotfix"); + button.removeAttr("disabled"); + } + else { + button.html("Select branch to hotfix"); + button.attr("disabled", "disabled"); + } + span.html(''); +}); + +$( document ).ready(function() { + $('td.version > select option[value^="branch/mr"]').each(function(){ $(this).change(); }); +}); diff --git a/release_dashboard/static/release_dashboard/js/refresh_info.js b/release_dashboard/static/release_dashboard/js/refresh_info.js new file mode 100644 index 0000000..7a63ffb --- /dev/null +++ b/release_dashboard/static/release_dashboard/js/refresh_info.js @@ -0,0 +1,66 @@ +/** + * refresh info + */ +$('button#refresh_all').click(function(e){ + // don't send the form + e.preventDefault(); + var button = $(this); + var span = $('span#refresh_all_error'); + + $.ajax({ + url: '/release_panel/refresh/', + type: 'POST', + success: successFunc, + error: errorFunc, + beforeSend: csrftokenFunc + }); + + function successFunc(data, status) { + span.html(''); + span.append('Done
'); + span.append('This will take a while. Refresh the page in a few'); + } + + function errorFunc(jqXHR, status, error) { + span.html(error); + button.removeAttr("disabled"); + } + + //deactivate button + button.attr("disabled", "disabled"); + span.html('processing'); + span.show(); +}); + +$('button.refresh').click(function(e){ + // don't send the form + e.preventDefault(); + var button = $(this); + var project = button.attr('id').replace('refresh_',''); + var span = $('span#refresh_error_' + project ); + + function successFunc(data, status) { + span.html(''); + span.append('Done'); + button.removeAttr("disabled"); + } + + function errorFunc(jqXHR, status, error) { + span.html(error); + button.removeAttr("disabled"); + } + + $.ajax({ + url: '/release_panel/refresh/' + project + '/', + type: 'POST', + success: successFunc, + error: errorFunc, + beforeSend: csrftokenFunc + }); + + //deactivate button + button.attr("disabled", "disabled"); + span.html('processing'); + span.show(); + +}); diff --git a/release_dashboard/tasks.py b/release_dashboard/tasks.py new file mode 100644 index 0000000..72eff48 --- /dev/null +++ b/release_dashboard/tasks.py @@ -0,0 +1,38 @@ +# Copyright (C) 2016 The Sipwise Team - http://sipwise.com + +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. + +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. + +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +from __future__ import absolute_import + +import logging +from celery import shared_task +from release_dashboard.models import Project +from django.conf import settings +from .utils import get_gerrit_tags, get_gerrit_branches + +logger = logging.getLogger(__name__) +rd_settings = settings.RELEASE_DASHBOARD_SETTINGS + + +@shared_task(ignore_result=True) +def gerrit_fetch_info(projectname): + project, _ = Project.objects.get_or_create(name=projectname) + project.tags = get_gerrit_tags(projectname) + project.branches = get_gerrit_branches(projectname) + project.save() + + +@shared_task(ignore_result=True) +def gerrit_fetch_all(): + for project in rd_settings['projects']: + gerrit_fetch_info.delay(project) diff --git a/release_dashboard/templates/release_dashboard/base.html b/release_dashboard/templates/release_dashboard/base.html new file mode 100644 index 0000000..8148861 --- /dev/null +++ b/release_dashboard/templates/release_dashboard/base.html @@ -0,0 +1,4 @@ +{% extends "panel/base.html" %} +{% block applist %} +
  • Panel
    • +{% endblock %} \ No newline at end of file diff --git a/release_dashboard/templates/release_dashboard/build.html b/release_dashboard/templates/release_dashboard/build.html new file mode 100644 index 0000000..7c3b979 --- /dev/null +++ b/release_dashboard/templates/release_dashboard/build.html @@ -0,0 +1,79 @@ +{% extends "release_dashboard/base.html" %} +{% load staticfiles %} +{% block title %}Build release{% endblock %} +{% block navlist %} +
  • Release Dashboard
    • +
  • Build release
    • +{% endblock %} +{% block content %} +
    +
    +
    +

    Actions

    +
    +
    +
    {% csrf_token %} +
    + + + +
    +
    + + +
    +
    + + +
    +
    +
    +
    +
    + + + + + + + + + {% for p in projects %} + + + + + {% endfor %} + +
    ProjectVersion
    + +
    +
    +
    +{% endblock %} +{% block extrajs %} + +{% endblock %} diff --git a/release_dashboard/templates/release_dashboard/build_deps.html b/release_dashboard/templates/release_dashboard/build_deps.html new file mode 100644 index 0000000..a720376 --- /dev/null +++ b/release_dashboard/templates/release_dashboard/build_deps.html @@ -0,0 +1,79 @@ +{% extends "release_dashboard/base.html" %} +{% load staticfiles %} +{% block title %}Build dependences{% endblock %} +{% block navlist %} +
  • Release Dashboard
    • +
  • Build dependences
    • +{% endblock %} +{% block content %} +
    +
    +
    +

    Actions

    +
    +
    +
    {% csrf_token %} +
    + + + +
    +
    + + +
    +
    + + +
    +
    +
    +
    +
    + + + + + + + + + {% for p in projects %} + + + + + {% endfor %} + +
    ProjectVersion
    + +
    +
    +
    +{% endblock %} +{% block extrajs %} + +{% endblock %} \ No newline at end of file diff --git a/release_dashboard/templates/release_dashboard/hotfix.html b/release_dashboard/templates/release_dashboard/hotfix.html new file mode 100644 index 0000000..e033b06 --- /dev/null +++ b/release_dashboard/templates/release_dashboard/hotfix.html @@ -0,0 +1,58 @@ +{% extends "release_dashboard/base.html" %} +{% load staticfiles %} +{% block title %}Hotfixes{% endblock %} +{% block navlist %} +
  • Release Dashboard
    • +
  • Hotfixes
    • +{% endblock %} +{% block content %} +
    + + + + + + + + + + + + {% csrf_token %} + {% for p in projects %} + + + + + + + + {% endfor %} + +
    ProjectVersionPush
    + + + + + + + +
    +
    + +{% endblock %} +{% block extrajs %} + + +{% endblock %} diff --git a/release_dashboard/templates/release_dashboard/index.html b/release_dashboard/templates/release_dashboard/index.html new file mode 100644 index 0000000..93ff615 --- /dev/null +++ b/release_dashboard/templates/release_dashboard/index.html @@ -0,0 +1,36 @@ +{% extends "release_dashboard/base.html" %} +{% block title %}Release Dashboard{% endblock %} +{% block navlist %} +
  • Release Dashboard
    • +{% endblock %} +{% block content %} +
    +
    +
    +

    Actions

    +
    +
    + +
    +
    +
    +{% endblock %} diff --git a/release_dashboard/templates/release_dashboard/refresh.html b/release_dashboard/templates/release_dashboard/refresh.html new file mode 100644 index 0000000..70158c9 --- /dev/null +++ b/release_dashboard/templates/release_dashboard/refresh.html @@ -0,0 +1,53 @@ +{% extends "release_dashboard/base.html" %} +{% load staticfiles %} +{% block title %}Hotfixes{% endblock %} +{% block navlist %} +
  • Release Dashboard
    • +
  • Refresh Info
    • +{% endblock %} +{% block content %} +
    +
    +
    +

    Actions

    +
    +
    +
    {% csrf_token %} +
    + + +
    +
    +
    +
    +
    + + + + + + + + + + {% csrf_token %} + {% for p in projects %} + + + + + + {% endfor %} + +
    Project
    + + + +
    +
    +
    +{% endblock %} +{% block extrajs %} + + +{% endblock %} diff --git a/release_dashboard/test/__init__.py b/release_dashboard/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/release_dashboard/test/test_models.py b/release_dashboard/test/test_models.py new file mode 100644 index 0000000..a07843b --- /dev/null +++ b/release_dashboard/test/test_models.py @@ -0,0 +1,155 @@ +# Copyright (C) 2016 The Sipwise Team - http://sipwise.com + +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. + +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. + +# You should have received a copy of the GNU General Public License along +# with this prograproj. If not, see . +import copy +from django.test import TestCase +from release_dashboard.models import Project + +GERRIT_REST_TAGS = """ +)]}' +[ + { + "ref": "refs/tags/mr2.0.0" + }, + { + "ref": "refs/tags/mr1.0.0" + } +] +""" +FILTERED_TAGS = [ + { + "ref": "refs/tags/mr2.0.0" + }, + { + "ref": "refs/tags/mr1.0.0" + } +] +GERRIT_REST_BRANCHES = """ +)]}' +[ + { + "ref": "refs/heads/master" + }, + { + "ref": "refs/heads/vseva/1789" + } +] +""" +FILTERED_BRANCHES = [ + { + "ref": "refs/heads/master" + }, + { + "ref": "refs/heads/vseva/1789" + } +] + + +class ProjectTestCase(TestCase): + + def test_create(self): + proj = Project.objects.create(name="fake") + self.assertEquals(proj.name, "fake") + + def test_tags(self): + proj = Project.objects.create(name="fake", json_tags=FILTERED_TAGS) + self.assertEquals(proj.name, "fake") + self.assertIsInstance(proj.tags, list) + self.assertItemsEqual(proj.tags, ["mr2.0.0", "mr1.0.0", ]) + + def test_tags_null(self): + proj = Project.objects.create(name="fake") + self.assertEquals(proj.name, "fake") + self.assertIsInstance(proj.tags, list) + self.assertItemsEqual(proj.tags, []) + + def test_branches(self): + proj = Project.objects.create(name="fake", + json_branches=FILTERED_BRANCHES) + self.assertEquals(proj.name, "fake") + self.assertIsInstance(proj.branches, list) + self.assertItemsEqual(proj.branches, ["vseva/1789", "master"]) + + def test_branches_null(self): + proj = Project.objects.create(name="fake") + self.assertEquals(proj.name, "fake") + self.assertIsInstance(proj.branches, list) + self.assertItemsEqual(proj.branches, []) + + def test_filtered_json(self): + res = Project._get_filtered_json(GERRIT_REST_TAGS) + self.assertEquals(res, FILTERED_TAGS) + + def test_filter_values(self): + values = copy.deepcopy(FILTERED_TAGS) + res = Project._filter_values(FILTERED_TAGS, '^refs/tags/(.+)$') + self.assertEquals(res, ['mr2.0.0', 'mr1.0.0']) + values.append({"ref": "no/no"}) + res = Project._filter_values(values, '^refs/tags/(.+)$') + self.assertEquals(res, ['mr2.0.0', 'mr1.0.0']) + + def test_filter_values_regex(self): + values = copy.deepcopy(FILTERED_TAGS) + res = Project._filter_values(FILTERED_TAGS, '^refs/tags/(.+)$', + r'^mr[0-9]+\.[0-9]+\.[0-9]+$') + self.assertEquals(res, ['mr2.0.0', 'mr1.0.0']) + values.append({"ref": "refs/tags/3.7.8"}) + res = Project._filter_values(values, '^refs/tags/(.+)$', + r'^mr[0-9]+\.[0-9]+\.[0-9]+$') + self.assertEquals(res, ['mr2.0.0', 'mr1.0.0']) + res = Project._filter_values(values, '^refs/tags/(.+)$', + r'^[0-9]+\.[0-9]+\.[0-9]+$') + self.assertEquals(res, ['3.7.8']) + + def test_tags_set(self): + proj = Project.objects.create(name="fake") + self.assertEquals(proj.name, "fake") + self.assertIsInstance(proj.tags, list) + self.assertItemsEqual(proj.tags, []) + proj.tags = GERRIT_REST_TAGS + self.assertItemsEqual(proj.tags, ["mr2.0.0", "mr1.0.0"]) + + def test_branches_set(self): + proj = Project.objects.create(name="fake") + self.assertEquals(proj.name, "fake") + self.assertIsInstance(proj.branches, list) + self.assertItemsEqual(proj.branches, []) + proj.branches = GERRIT_REST_BRANCHES + self.assertItemsEqual(proj.branches, ["master", "vseva/1789"]) + + def test_branches_mrXX(self): + tmp = [ + {"ref": "refs/heads/master"}, + {"ref": "refs/heads/mr0.1"}, + {"ref": "refs/heads/mr0.1.1"}, + {"ref": "refs/heads/vseva/nono"}, + ] + proj = Project.objects.create(name="fake", + json_branches=tmp) + self.assertEquals(proj.name, "fake") + self.assertItemsEqual(proj.branches_mrXX(), + ["mr0.1", ]) + + def test_branches_mrXXX(self): + tmp = [ + {"ref": "refs/heads/master"}, + {"ref": "refs/heads/mr0.1"}, + {"ref": "refs/heads/mr0.1.1"}, + {"ref": "refs/heads/vseva/nono"}, + ] + proj = Project.objects.create(name="fake", + json_branches=tmp) + self.assertEquals(proj.name, "fake") + self.assertItemsEqual(proj.branches_mrXXX(), + ["mr0.1.1", ]) diff --git a/release_dashboard/urls.py b/release_dashboard/urls.py new file mode 100644 index 0000000..0684d47 --- /dev/null +++ b/release_dashboard/urls.py @@ -0,0 +1,28 @@ +# Copyright (C) 2015 The Sipwise Team - http://sipwise.com + +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. + +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. + +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . + +from django.conf.urls import url +from . import views + +urlpatterns = [ + url(r'^$', views.index, name='index'), + url(r'^build_deps/$', views.build_deps, name='build_deps'), + url(r'^build/$', views.build_release, name='build_release'), + url(r'^hotfix/$', views.hotfix, name='hotfix'), + url(r'^hotfix/(?P[^/]+)/(?P[^/]+)/$', + views.hotfix_build), + url(r'^refresh/$', views.refresh_all, name='refresh_all'), + url(r'^refresh/(?P[^/]+)/$', views.refresh, name='refresh'), +] diff --git a/release_dashboard/utils.py b/release_dashboard/utils.py new file mode 100644 index 0000000..ae74517 --- /dev/null +++ b/release_dashboard/utils.py @@ -0,0 +1,87 @@ +# Copyright (C) 2015 The Sipwise Team - http://sipwise.com + +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. + +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. + +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . + +import logging +import uuid +import urllib +import requests +from requests.auth import HTTPDigestAuth +from django.conf import settings +from repoapi.utils import openurl +from release_dashboard.models import Project + +logger = logging.getLogger(__name__) + + +def get_response(url): + auth = HTTPDigestAuth( + settings.GERRIT_REST_HTTP_USER, + settings.GERRIT_REST_HTTP_PASSWD) + response = requests.get(url, auth=auth) + return response + + +def trigger_hotfix(project, branch, push="yes"): + flow_uuid = uuid.uuid4() + params = { + "base": settings.JENKINS_URL, + 'token': urllib.quote(settings.JENKINS_TOKEN), + 'action': urllib.quote("--hotfix"), + 'branch': urllib.quote(branch), + 'project': urllib.quote(project), + 'push': urllib.quote(push), + 'uuid': flow_uuid, + } + url = ("{base}/job/release-tools-runner/buildWithParameters?" + "token={token}&action={action}&branch={branch}&" + "PROJECTNAME={project}&repository={project}&" + "push={push}&uuid={uuid}".format(**params)) + + if settings.DEBUG: + logger.debug("Debug mode, would trigger: %s", url) + # raise Exception("debug error") + else: + openurl(url) + return "%s/job/release-tools-runner/" % settings.JENKINS_URL + + +def get_gerrit_info(url): + if settings.DEBUG: + logger.debug("Debug mode, would trigger: %s", url) + return r")]}'\n[]" + else: + response = get_response(url) + response.raise_for_status() + return response.text + + +def get_tags(projectname, regex=None): + project, _ = Project.objects.get_or_create(name=projectname) + return project.filter_tags(regex) + + +def get_branches(projectname, regex=None): + project, _ = Project.objects.get_or_create(name=projectname) + return project.filter_branches(regex) + + +def get_gerrit_tags(project, regex=None): + url = settings.GERRIT_URL.format("a/projects/%s/tags/" % project) + return get_gerrit_info(url) + + +def get_gerrit_branches(project, regex=None): + url = settings.GERRIT_URL.format("a/projects/%s/branches/" % project) + return get_gerrit_info(url) diff --git a/release_dashboard/views.py b/release_dashboard/views.py new file mode 100644 index 0000000..c0cfc0d --- /dev/null +++ b/release_dashboard/views.py @@ -0,0 +1,142 @@ +# Copyright (C) 2015 The Sipwise Team - http://sipwise.com + +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. + +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. + +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . + +import logging +import re +import json +from django.shortcuts import render +from django.http import HttpResponseNotFound, JsonResponse +from django.conf import settings +from django.views.decorators.http import require_http_methods +from release_dashboard.models import Project +from .utils import get_tags, get_branches, trigger_hotfix +from .tasks import gerrit_fetch_info, gerrit_fetch_all + +rd_settings = settings.RELEASE_DASHBOARD_SETTINGS +logger = logging.getLogger(__name__) +regex_hotfix = re.compile(r'^mr[0-9]+\.[0-9]+\.[0-9]+$') +regex_mr = re.compile(r'^mr.+$') + + +def index(request): + context = {} + return render(request, 'release_dashboard/index.html', context) + + +def _projects_versions(projects, regex=None, tag_only=False, ): + res = [] + for project in projects: + info = { + 'name': project, + 'tags': get_tags(project, regex), + } + if not tag_only: + info['branches'] = get_branches(project, regex) + res.append(info) + logger.debug(res) + return res + + +def _common_versions(context): + common_versions = {'tags': set(), 'branches': set()} + + for project in context['projects']: + common_versions['tags'] |= set(project['tags']) + common_versions['branches'] |= set(project['branches']) + context['common_versions'] = { + 'tags': sorted(common_versions['tags']), + 'branches': sorted(common_versions['branches']), + } + + +@require_http_methods(["POST", ]) +def hotfix_build(request, branch, project): + if project not in rd_settings['projects']: + error = "repo:%s not valid" % project + logger.error(error) + return HttpResponseNotFound(error) + + if not regex_hotfix.match(branch): + error = "branch:%s not valid. Not mrX.X.X format" % branch + logger.error(error) + return HttpResponseNotFound(error) + proj = Project.objects.get(name=project) + + if branch not in proj.branches_mrXXX(): + error = "branch:%s not valid" % branch + logger.error(error) + return HttpResponseNotFound(error) + + json_data = json.loads(request.body) + if json_data['push'] == 'no': + logger.warn("dryrun for %s:%s", project, branch) + url = trigger_hotfix(project, branch, json_data['push']) + return JsonResponse({'url': url}) + + +def build_deps(request): + context = { + 'projects': _projects_versions( + rd_settings['build_deps'], + regex_mr + ), + 'debian': rd_settings['debian_supported'], + } + _common_versions(context) + return render(request, 'release_dashboard/build_deps.html', context) + + +def hotfix(request): + context = { + 'projects': _projects_versions( + rd_settings['projects'], + regex_hotfix + ) + } + return render(request, 'release_dashboard/hotfix.html', context) + + +def build_release(request): + context = { + 'projects': _projects_versions( + rd_settings['projects'], + regex_mr + ), + 'debian': rd_settings['debian_supported'], + } + _common_versions(context) + return render(request, 'release_dashboard/build.html', context) + + +def refresh_all(request): + if request.method == "POST": + res = gerrit_fetch_all.delay() + return JsonResponse({'url': '/flower/task/%s' % res.id}) + else: + projects = [] + for project in rd_settings['projects']: + info = { + 'name': project, + 'tags': None + } + projects.append(info) + return render(request, 'release_dashboard/refresh.html', + {'projects': projects}) + + +@require_http_methods(["POST", ]) +def refresh(request, project): + res = gerrit_fetch_info.delay(project) + return JsonResponse({'url': '/flower/task/%s' % res.id}) diff --git a/repoapi/settings/common.py b/repoapi/settings/common.py index d732ea5..6ccc160 100644 --- a/repoapi/settings/common.py +++ b/repoapi/settings/common.py @@ -25,6 +25,7 @@ PROJECT_APPS = [ 'repoapi', 'hotfix', 'panel', + 'release_dashboard', ] INSTALLED_APPS = [ @@ -134,6 +135,10 @@ LOGGING = { 'handlers': ['console'], 'level': os.getenv('DJANGO_LOG_LEVEL', 'INFO'), }, + 'release_dashboard': { + 'handlers': ['console'], + 'level': os.getenv('DJANGO_LOG_LEVEL', 'INFO'), + }, }, } @@ -145,3 +150,96 @@ CELERY_ACCEPT_CONTENT = ['json'] CELERY_RESULT_BACKEND = 'djcelery.backends.database:DatabaseBackend' HOTFIX_ARTIFACT = 'debian_changelog.txt' + +RELEASE_DASHBOARD_SETTINGS = { + 'debian_supported': ('jessie', 'wheezy', 'squeeze'), + 'build_deps': ( + "data-hal", "libswrate", "sipwise-base", "mediaproxy-ng", + "ngcp-schema", "rtpengine", "libtcap", "libinewrate" + ), + 'projects': ( + "acc-cdi", + "asterisk", + "asterisk-sounds", + "asterisk-voicemail", + "backup-tools", + "bulk-processor", + "bootenv", + "captagent", + "cdr-exporter", + "cfg-schema", + "check-tools", + "cleanup-tools", + "cloudpbx-devices", + "cloudpbx-sources", + "collectd-mod-redis", + "comx", + "comx-application", + "comx-fileshare-service", + "comx-sip", + "comx-xmpp", + "data-hal", + "db-schema", + "diva-drivers", + "documentation", + "faxserver", + "glusterfs-config", + "heartbeat", + "hylafaxplus", + "iaxmodem", + "installer", + "kamailio", + "kamailio-config-tests", + "keyring", + "kibana", + "klish", + "libinewrate", + "libswrate", + "libtcap", + "license-client", + "lnpd", + "lua-ngcp-kamailio", + "mediaproxy-ng", + "mediaproxy-redis", + "mediator", + "megacli", + "metapackages", + "monitoring-tools", + "netscript", + "ngcp-api-tools", + "ngcp-klish-config", + "ngcp-panel", + "ngcp-prompts", + "ngcp-rtcengine", + "ngcp-schema", + "ngcp-status", + "ngcp-support", + "ngcpcfg", + "ngcpcfg-api", + "ngcpcfg-ha", + "ngrep-sip", + "ossbss", + "prosody", + "pushd", + "rate-o-mat", + "reminder", + "rtpengine", + "rtpengine-redis", + "sems", + "sems-app", + "sems-ha", + "sems-pbx", + "sems-prompts", + "sipsak", + "sipwise-base", + "snmp-agent", + "system-tests", + "system-tools", + "templates", + "upgrade", + "vmnotify", + "voisniff-ng", + "www_admin", + "www_csc" + ), +} diff --git a/repoapi/settings/dev.py b/repoapi/settings/dev.py index 452cc35..c66ef5c 100644 --- a/repoapi/settings/dev.py +++ b/repoapi/settings/dev.py @@ -19,6 +19,9 @@ import os # pylint: disable=W0401,W0614,C0413 from .test import * +LOGGING['loggers']['release_dashboard']['level'] = \ + os.getenv('DJANGO_LOG_LEVEL', 'DEBUG') + # celery BROKER_BACKEND = 'amqp' CELERY_ALWAYS_EAGER = False diff --git a/repoapi/settings/prod.py b/repoapi/settings/prod.py index a8a3682..41fccd4 100644 --- a/repoapi/settings/prod.py +++ b/repoapi/settings/prod.py @@ -15,6 +15,7 @@ # Build paths inside the project like this: os.path.join(BASE_DIR, ...) import os +from ConfigParser import RawConfigParser # pylint: disable=W0401,W0614 from .common import * @@ -53,6 +54,12 @@ LOGGING['loggers']['repoapi']['level'] = os.getenv('DJANGO_LOG_LEVEL', 'INFO') JENKINS_URL = "https://jenkins.mgm.sipwise.com" GERRIT_URL = "https://gerrit.mgm.sipwise.com/{}" + +gerrit_config = RawConfigParser() +gerrit_config.read(os.path.join(VAR_DIR, 'gerrit.ini')) +GERRIT_REST_HTTP_USER = gerrit_config.get('gerrit', 'HTTP_USER') +GERRIT_REST_HTTP_PASSWD = gerrit_config.get('gerrit', 'HTTP_PASSWD') + GITWEB_URL = "https://git.mgm.sipwise.com/gitweb/?p={}.git;a=commit;h={}" WORKFRONT_CREDENTIALS = os.path.join(BASE_DIR, '/etc/jenkins_jobs/workfront.ini') diff --git a/repoapi/settings/test.py b/repoapi/settings/test.py index 1bf6681..0f60342 100644 --- a/repoapi/settings/test.py +++ b/repoapi/settings/test.py @@ -61,6 +61,8 @@ DJANGO_LOG_LEVEL = 'DEBUG' JENKINS_URL = "http://localhost" GERRIT_URL = "https://gerrit.local/{}" +GERRIT_REST_HTTP_USER = 'jenkins' +GERRIT_REST_HTTP_PASSWD = 'verysecrethttppasswd' GITWEB_URL = "https://git.local/gitweb/?p={}.git;a=commit;h={}" WORKFRONT_CREDENTIALS = os.path.join(BASE_DIR, '.workfront.ini') diff --git a/repoapi/urls.py b/repoapi/urls.py index 937308f..4ca841c 100644 --- a/repoapi/urls.py +++ b/repoapi/urls.py @@ -57,4 +57,6 @@ urlpatterns = [ url(r'^docs/', views.schema_view), url(r'^panel/', include('panel.urls', namespace='panel')), + url(r'^release_panel/', + include('release_dashboard.urls', namespace='release_dashboard')), ]