From d85f9e0af79366df2ad0af985e46f5ed45b01241 Mon Sep 17 00:00:00 2001 From: Victor Seva Date: Mon, 25 Jul 2016 17:20:53 +0200 Subject: [PATCH] TT#1951 hotfix notes for workfront * release-tools-runner sends a note "hotfix triggered" * TODO: how to be sure that version reaches public release ?? Change-Id: Ib220f36d642172d8c89f4df35543193be16d37ce --- debian/control | 1 + hotfix/__init__.py | 0 hotfix/migrations/0001_initial.py | 34 +++++++++++ hotfix/migrations/__init__.py | 0 hotfix/models.py | 48 +++++++++++++++ hotfix/tasks.py | 32 ++++++++++ hotfix/test/__init__.py | 0 hotfix/test/test_hotfix_released.py | 94 +++++++++++++++++++++++++++++ hotfix/utils.py | 44 ++++++++++++++ repoapi/celery.py | 5 ++ repoapi/models/jbi.py | 4 +- repoapi/settings/common.py | 3 + repoapi/tasks.py | 12 ++-- t/Dockerfile | 3 +- 14 files changed, 274 insertions(+), 6 deletions(-) create mode 100644 hotfix/__init__.py create mode 100644 hotfix/migrations/0001_initial.py create mode 100644 hotfix/migrations/__init__.py create mode 100644 hotfix/models.py create mode 100644 hotfix/tasks.py create mode 100644 hotfix/test/__init__.py create mode 100644 hotfix/test/test_hotfix_released.py create mode 100644 hotfix/utils.py diff --git a/debian/control b/debian/control index 7b0eded..6e131ad 100644 --- a/debian/control +++ b/debian/control @@ -13,6 +13,7 @@ Section: python Architecture: all Depends: make, python, + python-debian, virtualenv, sqlite3, uwsgi-plugin-python, diff --git a/hotfix/__init__.py b/hotfix/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/hotfix/migrations/0001_initial.py b/hotfix/migrations/0001_initial.py new file mode 100644 index 0000000..1c1255e --- /dev/null +++ b/hotfix/migrations/0001_initial.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9 on 2016-07-26 12:58 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='WorkfrontNoteInfo', + fields=[ + ('id', models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name='ID')), + ('workfront_id', models.CharField(max_length=50)), + ('projectname', models.CharField(max_length=50)), + ('version', models.CharField(max_length=50)), + ], + ), + migrations.AlterUniqueTogether( + name='workfrontnoteinfo', + unique_together=set( + [('workfront_id', 'projectname', 'version')]), + ), + ] diff --git a/hotfix/migrations/__init__.py b/hotfix/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/hotfix/models.py b/hotfix/models.py new file mode 100644 index 0000000..220e2b6 --- /dev/null +++ b/hotfix/models.py @@ -0,0 +1,48 @@ +# Copyright (C) 2015 The Sipwise Team - http://sipwise.com + +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. + +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. + +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +import logging +import re + +from django.db import models +from django.db.models.signals import post_save +from django.conf import settings + +logger = logging.getLogger(__name__) +workfront_re = re.compile(r"TT#(\d+)") + + +class WorkfrontNoteInfo(models.Model): + workfront_id = models.CharField(max_length=50, null=False) + projectname = models.CharField(max_length=50, null=False) + version = models.CharField(max_length=50, null=False) + + class Meta: + unique_together = [ + "workfront_id", + "projectname", + "version" + ] + + @staticmethod + def getIds(change): + """ + parses text searching for Workfront TT# ocurrences + returns a list of IDs + """ + if change: + res = workfront_re.findall(change) + return set(res) + else: + return set() diff --git a/hotfix/tasks.py b/hotfix/tasks.py new file mode 100644 index 0000000..566c4a9 --- /dev/null +++ b/hotfix/tasks.py @@ -0,0 +1,32 @@ +# Copyright (C) 2016 The Sipwise Team - http://sipwise.com + +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. + +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. + +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +from __future__ import absolute_import + +import logging +from celery import shared_task +from django.conf import settings +from .utils import parse_changelog, create_note +from repoapi.models import JenkinsBuildInfo + +logger = logging.getLogger(__name__) + + +@shared_task(ignore_result=True) +def hotfix_released(jbi_id, path): + jbi = JenkinsBuildInfo.objects.get(pk=jbi_id) + logger.info('hotfix_released[%s] %s', jbi, path) + wids, changelog = parse_changelog(path) + for wid in wids: + create_note(wid, jbi.projectname, changelog.full_version) diff --git a/hotfix/test/__init__.py b/hotfix/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/hotfix/test/test_hotfix_released.py b/hotfix/test/test_hotfix_released.py new file mode 100644 index 0000000..4a7bc99 --- /dev/null +++ b/hotfix/test/test_hotfix_released.py @@ -0,0 +1,94 @@ +# Copyright (C) 2015 The Sipwise Team - http://sipwise.com + +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. + +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. + +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . + +import os +import shutil + +from django.test import TestCase, override_settings +from django.conf import settings +from mock import patch, call, mock_open +from hotfix import tasks, utils, models +from repoapi.models import JenkinsBuildInfo + +debian_changelog = """ngcp-fake (3.8.7.4+0~mr3.8.7.4) unstable; urgency=medium + + [ Kirill Solomko ] + * [ee3c706] MT#21499 add mysql replication options + + [ Victor Seva ] + * [aabb345] TT#345 fake comment + * [aabb123] MT#8989 TT#123 fake comment + + [ Sipwise Jenkins Builder ] + + -- whatever Fri, 22 Jul 2016 17:29:27 +0200 +""" + + +@override_settings(CELERY_EAGER_PROPAGATES_EXCEPTIONS=True) +class TestHotfixReleased(TestCase): + + def get_defaults(self): + defaults = { + 'tag': "edc90cd9-37f3-4613-9748-ed05a32031c2", + 'projectname': "fake", + 'jobname': "release-tools-runner", + 'buildnumber': 1, + 'result': "SUCCESS", + 'job_url': "https://jenkins.mgm.sipwise.com/job/real-fake-gerrit/", + 'param_tag': "none", + 'param_branch': "mr4.5", + 'param_release': "none", + 'param_distribution': "wheezy", + 'param_ppa': "gerrit_MT10339_review2054", + 'git_commit_msg': "7fg4567 TT#0001 whatever", + } + return defaults + + @patch('__builtin__.open', mock_open(read_data=debian_changelog)) + def test_parse_changelog(self): + ids, changelog = utils.parse_changelog("/tmp/fake.txt") + self.assertItemsEqual(ids, ["345", "123"]) + self.assertEquals(changelog.full_version, "3.8.7.4+0~mr3.8.7.4") + self.assertEquals(changelog.package, "ngcp-fake") + + @patch('__builtin__.open', mock_open(read_data=debian_changelog)) + @patch('repoapi.utils.dlfile') + @patch('repoapi.utils.workfront_note_send') + def test_hotfixreleased(self, wns, dlfile): + param = self.get_defaults() + jbi = JenkinsBuildInfo.objects.create(**param) + tasks.hotfix_released.delay(jbi.pk, "/tmp/fake.txt") + projectname = "fake" + version = "3.8.7.4+0~mr3.8.7.4" + gri = models.WorkfrontNoteInfo.objects.filter( + projectname=projectname, + version=version) + self.assertEquals(gri.count(), 2) + gri = models.WorkfrontNoteInfo.objects.filter( + workfront_id="345", + projectname=projectname, + version=version) + self.assertEquals(gri.count(), 1) + msg = "hotfix %s %s triggered" % (projectname, version) + calls = [call("345", msg), ] + gri = models.WorkfrontNoteInfo.objects.filter( + workfront_id="123", + projectname=projectname, + version=version) + self.assertEquals(gri.count(), 1) + msg = "hotfix %s %s triggered" % (projectname, version) + calls.append(call("123", msg)) + wns.assert_has_calls(calls) diff --git a/hotfix/utils.py b/hotfix/utils.py new file mode 100644 index 0000000..9fa5262 --- /dev/null +++ b/hotfix/utils.py @@ -0,0 +1,44 @@ +# Copyright (C) 2016 The Sipwise Team - http://sipwise.com + +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. + +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. + +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +from __future__ import absolute_import +import logging +from debian.changelog import Changelog +from .models import WorkfrontNoteInfo +from repoapi import utils + +logger = logging.getLogger(__name__) + + +def parse_changelog(path): + changelog = Changelog() + with open(path, 'r') as file_changelog: + changelog.parse_changelog(file_changelog.read()) + set_ids = set() + for block in changelog: + for change in block.changes(): + set_ids = set_ids.union(WorkfrontNoteInfo.getIds(change)) + return (set_ids, changelog) + + +def create_note(wid, projectname, version): + wni = WorkfrontNoteInfo.objects + + note, created = wni.get_or_create( + workfront_id=wid, + projectname=projectname, + version=version) + if created: + msg = "hotfix %s %s triggered" % (projectname, version) + utils.workfront_note_send(wid, msg) diff --git a/repoapi/celery.py b/repoapi/celery.py index 5bdc677..f5303a4 100644 --- a/repoapi/celery.py +++ b/repoapi/celery.py @@ -28,3 +28,8 @@ app = Celery('repoapi') # pickle the object when using Windows. app.config_from_object('django.conf:settings') app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) + + +@app.task() +def jbi_parse_hotfix(jbi_id, path): + app.send_task('hotfix_released', jbi_id, path) diff --git a/repoapi/models/jbi.py b/repoapi/models/jbi.py index c8f0195..7facef9 100644 --- a/repoapi/models/jbi.py +++ b/repoapi/models/jbi.py @@ -146,4 +146,6 @@ class JenkinsBuildInfo(models.Model): def jbi_manage(sender, **kwargs): if kwargs["created"]: instance = kwargs["instance"] - get_jbi_files.delay(instance.jobname, instance.buildnumber) + get_jbi_files.delay(instance.pk, + instance.jobname, + instance.buildnumber) diff --git a/repoapi/settings/common.py b/repoapi/settings/common.py index 26a8e0e..d732ea5 100644 --- a/repoapi/settings/common.py +++ b/repoapi/settings/common.py @@ -23,6 +23,7 @@ BASE_DIR = os.path.dirname( # django-jenkins PROJECT_APPS = [ 'repoapi', + 'hotfix', 'panel', ] @@ -142,3 +143,5 @@ CELERY_TASK_SERIALIZER = 'json' CELERY_RESULT_SERIALIZER = 'json' CELERY_ACCEPT_CONTENT = ['json'] CELERY_RESULT_BACKEND = 'djcelery.backends.database:DatabaseBackend' + +HOTFIX_ARTIFACT = 'debian_changelog.txt' diff --git a/repoapi/tasks.py b/repoapi/tasks.py index 6e3813d..4918152 100644 --- a/repoapi/tasks.py +++ b/repoapi/tasks.py @@ -16,8 +16,10 @@ from __future__ import absolute_import import json import logging +from os.path import basename from celery import shared_task from django.conf import settings +from .celery import jbi_parse_hotfix from .utils import jenkins_get_console, jenkins_get_job, jenkins_get_artifact from .utils import jenkins_get_env @@ -25,12 +27,14 @@ logger = logging.getLogger(__name__) @shared_task(ignore_result=True) -def jbi_get_artifact(jobname, buildnumber, artifact_info): - jenkins_get_artifact(jobname, buildnumber, artifact_info) +def jbi_get_artifact(jbi_id, jobname, buildnumber, artifact_info): + path = jenkins_get_artifact(jobname, buildnumber, artifact_info) + if basename(path) == settings.HOTFIX_ARTIFACT: + jbi_parse_hotfix.delay(jbi_id, path) @shared_task(ignore_result=True) -def get_jbi_files(jobname, buildnumber): +def get_jbi_files(jbi_id, jobname, buildnumber): jenkins_get_console(jobname, buildnumber) jenkins_get_env(jobname, buildnumber) path = jenkins_get_job(jobname, buildnumber) @@ -39,6 +43,6 @@ def get_jbi_files(jobname, buildnumber): data = json.load(data_file) logger.debug("job_info:%s", data) for artifact in data['artifacts']: - jbi_get_artifact.delay(jobname, buildnumber, artifact) + jbi_get_artifact.delay(jbi_id, jobname, buildnumber, artifact) else: logger.debug("skip artifacts download for jobname: %s", jobname) diff --git a/t/Dockerfile b/t/Dockerfile index d10c90f..d98c79b 100644 --- a/t/Dockerfile +++ b/t/Dockerfile @@ -8,7 +8,8 @@ FROM docker.mgm.sipwise.com/sipwise-jessie:latest ENV REFRESHED_AT 2016-08-02 RUN apt-get update -RUN apt-get install --assume-yes python2.7 python2.7-dev python-distribute python-pip git screen +RUN apt-get install --assume-yes python2.7 python2.7-dev \ + python-distribute python-pip python-debian git screen # Get pip to download and install requirements: COPY dev.txt test.txt common.txt /tmp/