From 35d2c97d58358b74a007db4b3811b56f3f29a0e5 Mon Sep 17 00:00:00 2001 From: Victor Seva Date: Thu, 14 Jul 2016 17:58:44 +0200 Subject: [PATCH] TT#1734 repoapi: workfront notes * keep track of reviews/commits sent to workfront so just one note will be sent per action * notes will be private * using workfront-post-note script Change-Id: Ibde45086ff4e3c922650f1f290d643a61c42e0ec --- repoapi/migrations/0006_auto_20160714_1359.py | 28 +++ repoapi/models.py | 78 ++++++++ repoapi/settings/dev.py | 19 ++ repoapi/settings/prod.py | 20 ++ repoapi/test/test_workfrontnote.py | 173 ++++++++++++++++++ repoapi/utils.py | 32 +++- 6 files changed, 348 insertions(+), 2 deletions(-) create mode 100644 repoapi/migrations/0006_auto_20160714_1359.py create mode 100644 repoapi/test/test_workfrontnote.py diff --git a/repoapi/migrations/0006_auto_20160714_1359.py b/repoapi/migrations/0006_auto_20160714_1359.py new file mode 100644 index 0000000..25fabb0 --- /dev/null +++ b/repoapi/migrations/0006_auto_20160714_1359.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('repoapi', '0005_jenkinsbuildinfo_git_commit'), + ] + + operations = [ + migrations.CreateModel( + name='WorkfrontNoteInfo', + fields=[ + ('id', models.AutoField( + verbose_name='ID', serialize=False, + auto_created=True, primary_key=True)), + ('workfront_id', models.CharField(max_length=50)), + ('gerrit_change', models.CharField(max_length=50)), + ], + ), + migrations.AlterUniqueTogether( + name='workfrontnoteinfo', + unique_together=set([('workfront_id', 'gerrit_change')]), + ), + ] diff --git a/repoapi/models.py b/repoapi/models.py index c96c1c4..c464ab2 100644 --- a/repoapi/models.py +++ b/repoapi/models.py @@ -15,10 +15,14 @@ from django.db import models from django.db.models import signals +from django.conf import settings from repoapi import utils import logging +import re logger = logging.getLogger(__name__) +workfront_re = re.compile('TT#(\d+)') +commit_re = re.compile('^(\w{7}) ') class JenkinsBuildInfoManager(models.Manager): @@ -151,3 +155,77 @@ def gerrit_repo_manage(sender, **kwargs): gerrit_repo_del(instance) signals.post_save.connect(gerrit_repo_manage, sender=JenkinsBuildInfo) + + +class WorkfrontNoteInfo(models.Model): + workfront_id = models.CharField(max_length=50, null=False) + gerrit_change = models.CharField(max_length=50, null=False) + + class Meta: + unique_together = ["workfront_id", "gerrit_change"] + + @staticmethod + def getIds(git_comment): + """ + parses git_commit_msg searching for Workfront TT# ocurrences + returns a list of IDs + """ + if git_comment: + res = workfront_re.findall(git_comment) + return set(res) + else: + return set() + + @staticmethod + def getCommit(git_comment): + """ + parses git_commit_msg searching for short GIT_COMMIT + """ + if git_comment: + res = commit_re.search(git_comment) + if res: + return res.group(1) + + def __str__(self): + return "%s:%s" % (self.workfront_id, self.gerrit_change) + + +def workfront_note_add(instance, message): + wni = WorkfrontNoteInfo.objects + workfront_ids = WorkfrontNoteInfo.getIds(instance.git_commit_msg) + + for wid in workfront_ids: + if not instance.gerrit_eventtype: + change = WorkfrontNoteInfo.getCommit(instance.git_commit_msg) + url = settings.GITWEB_URL.format(instance.projectname, change) + else: + change = instance.gerrit_change + url = settings.GERRIT_URL.format(instance.gerrit_change) + note, created = wni.get_or_create( + workfront_id=wid, + gerrit_change=change) + if created: + if not utils.workfront_note_send(wid, "%s %s" % (message, url)): + logger.error("remove releated WorkfrontNoteInfo") + note.delete() + + +def workfront_note_manage(sender, **kwargs): + """ + -get-code job is the first in the flow that has the proper + GIT_CHANGE_SUBJECT envVar set, so git_commit_msg is fine + """ + if kwargs["created"]: + instance = kwargs["instance"] + if instance.jobname.endswith("-get-code") and \ + instance.result == "SUCCESS": + if instance.gerrit_eventtype == 'change-merged': + msg = "review merged" + elif instance.gerrit_eventtype == 'patchset-created': + msg = "review created" + else: + msg = "commit created" + workfront_note_add(instance, msg) + + +signals.post_save.connect(workfront_note_manage, sender=JenkinsBuildInfo) diff --git a/repoapi/settings/dev.py b/repoapi/settings/dev.py index 400e62b..23d4413 100644 --- a/repoapi/settings/dev.py +++ b/repoapi/settings/dev.py @@ -156,5 +156,24 @@ SWAGGER_SETTINGS = { }, } +LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'handlers': { + 'console': { + 'class': 'logging.StreamHandler', + }, + }, + 'loggers': { + 'django': { + 'handlers': ['console'], + 'level': os.getenv('DJANGO_LOG_LEVEL', 'DEBUG'), + }, + }, +} + JENKINS_URL = "http://localhost" JENKINS_TOKEN = "sipwise_jenkins_ci" +GERRIT_URL = "https://gerrit.local/{}" +GITWEB_URL = "https://git.local/gitweb/?p={}.git;a=commit;h={}" +WORKFRONT_CREDENTIALS = os.path.join(BASE_DIR, '.workfront.ini') diff --git a/repoapi/settings/prod.py b/repoapi/settings/prod.py index 14b5832..ab5ac33 100644 --- a/repoapi/settings/prod.py +++ b/repoapi/settings/prod.py @@ -147,5 +147,25 @@ SWAGGER_SETTINGS = { }, } +LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'handlers': { + 'console': { + 'class': 'logging.StreamHandler', + }, + }, + 'loggers': { + 'django': { + 'handlers': ['console'], + 'level': os.getenv('DJANGO_LOG_LEVEL', 'INFO'), + }, + }, +} + JENKINS_URL = "https://jenkins.mgm.sipwise.com" JENKINS_TOKEN = "sipwise_jenkins_ci" +GERRIT_URL = "https://gerrit.mgm.sipwise.com/{}" +GITWEB_URL = "https://git.mgm.sipwise.com/gitweb/?p={}.git;a=commit;h={}" +WORKFRONT_CREDENTIALS = os.path.join(BASE_DIR, + '/etc/jenkins_jobs/workfront.ini') diff --git a/repoapi/test/test_workfrontnote.py b/repoapi/test/test_workfrontnote.py new file mode 100644 index 0000000..23f2f91 --- /dev/null +++ b/repoapi/test/test_workfrontnote.py @@ -0,0 +1,173 @@ +# Copyright (C) 2016 The Sipwise Team - http://sipwise.com + +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. + +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. + +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . + +from django.test import TestCase +from django.conf import settings +from repoapi.models import JenkinsBuildInfo, WorkfrontNoteInfo +from mock import patch + + +class WorkfrontNoteTestCase(TestCase): + + def test_getID(self): + res = WorkfrontNoteInfo.getIds("jojo TT#0891 whatever") + self.assertItemsEqual(res, ['0891']) + + def test_getID_multiple(self): + res = WorkfrontNoteInfo.getIds("jojo TT#0891 whatever TT#0001") + self.assertItemsEqual(res, ['0891', '0001']) + + def test_getID_multiple_duplicate(self): + res = WorkfrontNoteInfo.getIds("jojo TT#0891 whatever TT#0001 TT#0891") + self.assertItemsEqual(res, ['0891', '0001']) + + def test_getCommit(self): + res = WorkfrontNoteInfo.getCommit("1234567 TT#67676 whatever") + self.assertEquals(res, "1234567") + + @patch('repoapi.utils.workfront_note_send') + def test_note_gerrit(self, utils): + JenkinsBuildInfo.objects.create( + tag="edc90cd9-37f3-4613-9748-ed05a32031c2", + projectname="kamailio", + jobname="kamailio-gerrit", + buildnumber=897, + result="SUCCESS", + job_url="https://jenkins.mgm.sipwise.com/job/kamailio-gerrit/", + gerrit_patchset="1", + gerrit_change="2054", + gerrit_eventtype="patchset-created", + param_tag="none", + param_branch="master", + param_release="none", + param_distribution="wheezy", + param_ppa="gerrit_MT10339_review2054") + + gri = WorkfrontNoteInfo.objects.filter( + workfront_id="0001", + gerrit_change="2054") + self.assertEquals(gri.count(), 0) + + JenkinsBuildInfo.objects.create( + tag="edc90cd9-37f3-4613-9748-ed05a32031c2", + projectname="kamailio", + jobname="kamailio-get-code", + buildnumber=897, + result="SUCCESS", + job_url="https://jenkins.mgm.sipwise.com/job/kamailio-get-code/", + gerrit_patchset="1", + gerrit_change="2054", + gerrit_eventtype="patchset-created", + param_tag="none", + param_branch="master", + param_release="none", + param_distribution="wheezy", + param_ppa="gerrit_MT10339_review2054", + git_commit_msg="7fg4567 TT#0001 whatever") + + gri = WorkfrontNoteInfo.objects.filter( + workfront_id="0001", + gerrit_change="2054") + self.assertEquals(gri.count(), 1) + msg = "review created %s" % settings.GERRIT_URL.format("2054") + utils.assert_called_once_with("0001", msg) + + @patch('repoapi.utils.workfront_note_send') + def test_note_merge(self, utils): + JenkinsBuildInfo.objects.create( + tag="edc90cd9-37f3-4613-9748-ed05a32031c2", + projectname="kamailio", + jobname="kamailio-gerrit", + buildnumber=897, + result="SUCCESS", + job_url="https://jenkins.mgm.sipwise.com/job/kamailio-gerrit/", + gerrit_patchset="1", + gerrit_change="2054", + gerrit_eventtype="change-merged", + param_tag="none", + param_branch="master", + param_release="none", + param_distribution="wheezy", + param_ppa="gerrit_MT10339_review2054") + + gri = WorkfrontNoteInfo.objects.filter( + workfront_id="0001", + gerrit_change="2054") + self.assertEquals(gri.count(), 0) + + JenkinsBuildInfo.objects.create( + tag="edc90cd9-37f3-4613-9748-ed05a32031c2", + projectname="kamailio", + jobname="kamailio-get-code", + buildnumber=897, + result="SUCCESS", + job_url="https://jenkins.mgm.sipwise.com/job/kamailio-get-code/", + gerrit_patchset="1", + gerrit_change="2054", + gerrit_eventtype="change-merged", + param_tag="none", + param_branch="master", + param_release="none", + param_distribution="wheezy", + param_ppa="gerrit_MT10339_review2054", + git_commit_msg="7fg456 TT#0001 whatever") + + gri = WorkfrontNoteInfo.objects.filter( + workfront_id="0001", + gerrit_change="2054") + self.assertEquals(gri.count(), 1) + msg = "review merged %s" % settings.GERRIT_URL.format("2054") + utils.assert_called_once_with("0001", msg) + + @patch('repoapi.utils.workfront_note_send') + def test_note_commit(self, utils): + JenkinsBuildInfo.objects.create( + tag="edc90cd9-37f3-4613-9748-ed05a32031c2", + projectname="kamailio", + jobname="kamailio-get-code", + buildnumber=897, + result="SUCCESS", + job_url="https://jenkins.mgm.sipwise.com/job/kamailio-get-code/", + param_tag="none", + param_branch="master", + param_release="none", + param_distribution="wheezy", + git_commit_msg="7fg4567 TT#0001 whatever") + + gri = WorkfrontNoteInfo.objects.filter( + workfront_id="0001", + gerrit_change="7fg4567") + self.assertEquals(gri.count(), 1) + + JenkinsBuildInfo.objects.create( + tag="edc90cd9-37f3-4613-9748-ed05a32031c2", + projectname="kamailio", + jobname="kamailio-binaries", + buildnumber=897, + result="SUCCESS", + job_url="https://jenkins.mgm.sipwise.com/job/kamailio-binaries/", + param_tag="none", + param_branch="master", + param_release="none", + param_distribution="wheezy", + git_commit_msg="7fg4567 TT#0001 whatever") + + gri = WorkfrontNoteInfo.objects.filter( + workfront_id="0001", + gerrit_change="7fg4567") + self.assertEquals(gri.count(), 1) + msg = "commit created %s" % settings.GITWEB_URL.format( + "kamailio", "7fg4567") + utils.assert_called_once_with("0001", msg) diff --git a/repoapi/utils.py b/repoapi/utils.py index ce4bb6d..effb799 100644 --- a/repoapi/utils.py +++ b/repoapi/utils.py @@ -12,21 +12,33 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . +from __future__ import print_function import urllib2 import logging +import subprocess +import sys from django.conf import settings logger = logging.getLogger(__name__) +def executeAndReturnOutput(command, env=None): + p = subprocess.Popen(command, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, env=env) + stdoutdata, stderrdata = p.communicate() + print(stdoutdata, file=sys.stdout) + print(stderrdata, file=sys.stderr) + return p.returncode, stdoutdata, stderrdata + + def openurl(URL): req = urllib2.Request(URL) response = urllib2.urlopen(req) if response.code is 200: - print "OK" + print("OK") return 0 else: - print "Error retrieving %s" % URL + print("Error retrieving %s" % URL) return 1 @@ -38,3 +50,19 @@ def jenkins_remove_ppa(repo): logger.info("I would call %s" % url) else: openurl(url) + + +def workfront_note_send(_id, message): + command = [ + "/usr/bin/workfront-post-note", + "--credfile=%s" % settings.WORKFRONT_CREDENTIALS, + "--private", + "--taskid=%s" % _id, + '--message="%s"' % message + ] + logger.debug("workfront-port-note command: %s" % command) + res = executeAndReturnOutput(command) + if res[0] != 0: + logger.error("can't post workfront note. %s. %s" % (res[1], res[2])) + return False + return True