MT#3306 hotfix: add support for mantis

* set default tracker to mantis, None for test/dev

* debian: add missing sipwise-workfront-tools dependency
  Added as Recomends since we are not going to use it
  in the near future

* MANTIS_* settings

Change-Id: Id7b19f23d316a31b362112167edb5c395baf554b
pull/9/head
Victor Seva 4 years ago
parent 3e038304f6
commit 70d70da8c9

2
debian/control vendored

@ -29,5 +29,7 @@ Depends:
uwsgi-plugin-python3,
virtualenv,
${misc:Depends},
Recomends:
sipwise-workfront-tools,
Description: REST API webapp
This package provides repoapi webapp.

4
debian/server.ini vendored

@ -19,3 +19,7 @@ HTTP_PASSWD=fakeHTTPpass
URL=fake
HTTP_USER=fake
HTTP_PASSWD=fakeHTTPpass
[mantis]
URL=fake
TOKEN=fake

@ -27,3 +27,13 @@ class WorkfrontNoteInfoResource(resources.ModelResource):
@admin.register(models.WorkfrontNoteInfo)
class WorkfrontNoteInfoAdmin(ImportExportModelAdmin):
resource_class = WorkfrontNoteInfoResource
class MantisNoteInfoResource(resources.ModelResource):
class Meta:
model = models.MantisNoteInfo
@admin.register(models.MantisNoteInfo)
class MantisNoteInfoAdmin(ImportExportModelAdmin):
resource_class = MantisNoteInfoResource

@ -1,4 +1,4 @@
# Copyright (C) 2020 The Sipwise Team - http://sipwise.com
# Copyright (C) 2020-2022 The Sipwise Team - http://sipwise.com
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free
@ -15,9 +15,15 @@
from django.conf import settings # noqa
from appconf import AppConf
from repoapi.conf import Tracker
class HotfixConf(AppConf):
WF_REGEX = r"TT#(\d+)"
REGEX = {
Tracker.NONE: r"#(\d+)",
Tracker.WORKFRONT: r"TT#(\d+)",
Tracker.MANTIS: r"MT#(\d+)",
}
ARTIFACT = "debian_changelog.txt"
class Meta:

@ -0,0 +1,24 @@
# Copyright (C) 2022 The Sipwise Team - http://sipwise.com
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>.
class Error(Exception):
"""Base class for exceptions in this module."""
pass
class TrackerNotDefined(Error):
pass

@ -0,0 +1,33 @@
# Generated by Django 3.2.15 on 2022-09-13 08:04
from django.db import migrations
from django.db import models
class Migration(migrations.Migration):
dependencies = [
("hotfix", "0001_initial"),
]
operations = [
migrations.CreateModel(
name="MantisNoteInfo",
fields=[
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("projectname", models.CharField(max_length=50)),
("version", models.CharField(max_length=50)),
("mantis_id", models.CharField(max_length=50)),
],
options={
"unique_together": {("mantis_id", "projectname", "version")},
},
),
]

@ -15,28 +15,126 @@
import re
from django.db import models
from natsort import humansorted
from .conf import settings
from .conf import Tracker
from .exceptions import TrackerNotDefined
workfront_re = re.compile(settings.HOTFIX_WF_REGEX)
hotfix_re_release = re.compile(r".+~(mr[0-9]+\.[0-9]+\.[0-9]+.[0-9]+)$")
class WorkfrontNoteInfo(models.Model):
workfront_id = models.CharField(max_length=50, null=False)
class NoteInfo(models.Model):
projectname = models.CharField(max_length=50, null=False)
version = models.CharField(max_length=50, null=False)
tracker_re = re.compile(settings.HOTFIX_REGEX[Tracker.NONE])
class Meta:
unique_together = ["workfront_id", "projectname", "version"]
abstract = True
@staticmethod
def get_target_release(version):
match = hotfix_re_release.search(version)
if match:
return match.group(1)
@property
def target_release(self):
return NoteInfo.get_target_release(self.version)
@staticmethod
def get_model():
if settings.REPOAPI_TRACKER == Tracker.MANTIS:
return MantisNoteInfo
elif settings.REPOAPI_TRACKER == Tracker.WORKFRONT:
return WorkfrontNoteInfo
return NoteInfo
@staticmethod
def getIds(change):
def create(wid, projectname, version):
note, created = NoteInfo.get_or_create(
field_id=wid, projectname=projectname, version=version
)
if created:
msg = "hotfix %s.git %s triggered" % (
note.projectname,
note.version,
)
note.send(msg)
target_release = note.target_release
if target_release:
note.set_target_release()
@classmethod
def get_or_create(cls, defaults=None, **kwargs):
field_id = kwargs.pop("field_id")
if settings.REPOAPI_TRACKER == Tracker.MANTIS:
model = MantisNoteInfo
kwargs["mantis_id"] = field_id
elif settings.REPOAPI_TRACKER == Tracker.WORKFRONT:
model = WorkfrontNoteInfo
kwargs["workfront_id"] = field_id
else:
raise TrackerNotDefined()
return model.objects.get_or_create(defaults, **kwargs)
@classmethod
def getIds(cls, change):
"""
parses text searching for Workfront TT# occurrences
parses text searching for tracker occurrences
returns a list of IDs
"""
if change:
res = workfront_re.findall(change)
res = cls.tracker_re.findall(change)
return set(res)
else:
return set()
class WorkfrontNoteInfo(NoteInfo):
workfront_id = models.CharField(max_length=50, null=False)
tracker_re = re.compile(settings.HOTFIX_REGEX[Tracker.WORKFRONT])
class Meta:
unique_together = ["workfront_id", "projectname", "version"]
def send(self, msg: str):
from repoapi import utils
utils.workfront_note_send(self.workfront_id, msg)
def set_target_release(self):
from repoapi import utils
utils.workfront_set_release_target(
self.workfront_id, self.target_release
)
class MantisNoteInfo(NoteInfo):
mantis_id = models.CharField(max_length=50, null=False)
tracker_re = re.compile(settings.HOTFIX_REGEX[Tracker.MANTIS])
class Meta:
unique_together = ["mantis_id", "projectname", "version"]
def send(self, msg: str):
from repoapi import utils
utils.mantis_note_send(self.mantis_id, msg)
def set_target_release(self):
"""reconstruct value without asking for previous value"""
from repoapi import utils
qs = MantisNoteInfo.objects.filter(mantis_id=self.mantis_id)
values = qs.values_list("version", flat=True)
versions = set()
for val in values:
target_release = self.get_target_release(val)
if target_release:
versions.add(target_release)
if len(versions) > 0:
utils.mantis_set_release_target(
self.mantis_id, ",".join(humansorted(versions))
)

@ -16,8 +16,11 @@ from unittest.mock import call
from unittest.mock import mock_open
from unittest.mock import patch
from django.test import override_settings
from hotfix import models
from hotfix import utils
from hotfix.conf import Tracker
from repoapi.models import JenkinsBuildInfo
from repoapi.test.base import BaseTest
@ -54,26 +57,12 @@ class TestHotfixReleased(BaseTest):
}
return defaults
@patch("builtins.open", mock_open(read_data=debian_changelog))
def test_parse_changelog(self):
ids, changelog = utils.parse_changelog("/tmp/fake.txt")
self.assertCountEqual(ids, ["345", "123"])
self.assertEqual(changelog.full_version, "3.8.7.4+0~mr3.8.7.4")
self.assertEqual(changelog.package, "ngcp-fake")
def test_get_target_release(self):
val = utils.get_target_release("3.8.7.4+0~mr3.8.7.4")
self.assertEqual(val, "mr3.8.7.4")
def test_get_target_release_ko(self):
val = utils.get_target_release("3.8.7.4-1")
self.assertIsNone(val)
@override_settings(REPOAPI_TRACKER=Tracker.WORKFRONT)
@patch("builtins.open", mock_open(read_data=debian_changelog))
@patch("repoapi.utils.dlfile")
@patch("repoapi.utils.workfront_set_release_target")
@patch("repoapi.utils.workfront_note_send")
def test_hotfixreleased(self, wns, wsrt, dlfile):
def test_hotfixreleased_wf(self, wns, wsrt, dlfile):
param = self.get_defaults()
jbi = JenkinsBuildInfo.objects.create(**param)
utils.process_hotfix(str(jbi), jbi.projectname, "/tmp/fake.txt")
@ -102,3 +91,76 @@ class TestHotfixReleased(BaseTest):
[call("345", "mr3.8.7.4"), call("123", "mr3.8.7.4")],
any_order=True,
)
@override_settings(REPOAPI_TRACKER=Tracker.MANTIS)
@patch("builtins.open", mock_open(read_data=debian_changelog))
@patch("repoapi.utils.dlfile")
@patch("repoapi.utils.mantis_set_release_target")
@patch("repoapi.utils.mantis_note_send")
def test_hotfixreleased_mantis(self, mns, msrt, dlfile):
param = self.get_defaults()
jbi = JenkinsBuildInfo.objects.create(**param)
utils.process_hotfix(str(jbi), jbi.projectname, "/tmp/fake.txt")
projectname = "fake"
version = "3.8.7.4+0~mr3.8.7.4"
gri = models.MantisNoteInfo.objects.filter(
projectname=projectname, version=version
)
self.assertEqual(gri.count(), 2)
gri = models.MantisNoteInfo.objects.filter(
mantis_id="8989", projectname=projectname, version=version
)
self.assertEqual(gri.count(), 1)
msg = "hotfix %s.git %s triggered" % (projectname, version)
calls = [
call("8989", msg),
]
gri = models.MantisNoteInfo.objects.filter(
mantis_id="21499", projectname=projectname, version=version
)
self.assertEqual(gri.count(), 1)
msg = "hotfix %s.git %s triggered" % (projectname, version)
calls.append(call("21499", msg))
mns.assert_has_calls(calls, any_order=True)
msrt.assert_has_calls(
[call("8989", "mr3.8.7.4"), call("21499", "mr3.8.7.4")],
any_order=True,
)
@override_settings(REPOAPI_TRACKER=Tracker.MANTIS)
@patch("builtins.open", mock_open(read_data=debian_changelog))
@patch("repoapi.utils.dlfile")
@patch("repoapi.utils.mantis_set_release_target")
@patch("repoapi.utils.mantis_note_send")
def test_hotfixreleased_mantis_versions(self, mns, msrt, dlfile):
param = self.get_defaults()
jbi = JenkinsBuildInfo.objects.create(**param)
projectname = "fake"
version = "3.8.7.4+0~mr3.8.7.4"
other_version = "5.8.7.4+0~mr5.8.7.4"
gri = models.MantisNoteInfo.objects.create(
mantis_id="8989", projectname=projectname, version=other_version
)
utils.process_hotfix(str(jbi), jbi.projectname, "/tmp/fake.txt")
gri = models.MantisNoteInfo.objects.filter(
mantis_id="8989", projectname=projectname
)
self.assertEqual(gri.count(), 2)
msg = "hotfix %s.git %s triggered" % (projectname, version)
calls = [
call("8989", msg),
]
gri = models.MantisNoteInfo.objects.filter(
mantis_id="21499", projectname=projectname, version=version
)
self.assertEqual(gri.count(), 1)
msg = "hotfix %s.git %s triggered" % (projectname, version)
calls.append(call("21499", msg))
mns.assert_has_calls(calls, any_order=True)
msrt.assert_has_calls(
[
call("8989", "mr3.8.7.4,mr5.8.7.4"),
call("21499", "mr3.8.7.4"),
],
any_order=True,
)

@ -0,0 +1,71 @@
# Copyright (C) 2015-2022 The Sipwise Team - http://sipwise.com
#
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>.
from unittest.mock import mock_open
from unittest.mock import patch
from django.test import override_settings
from django.test import SimpleTestCase
from natsort import humansorted
from hotfix import utils
from hotfix.conf import Tracker
debian_changelog = """ngcp-fake (3.8.7.4+0~mr3.8.7.4) unstable; urgency=medium
[ Kirill Solomko ]
* [ee3c706] MT#21499 add mysql replication options
[ Victor Seva ]
* [aabb345] TT#345 fake comment
* [aabb123] MT#8989 TT#123 fake comment
[ Sipwise Jenkins Builder ]
-- whatever <jenkins@sipwise.com> Fri, 22 Jul 2016 17:29:27 +0200
"""
class TestUtils(SimpleTestCase):
@override_settings(REPOAPI_TRACKER=Tracker.NONE)
@patch("builtins.open", mock_open(read_data=debian_changelog))
def test_parse_changelog_none(self):
ids, changelog = utils.parse_changelog("/tmp/fake.txt")
self.assertListEqual(humansorted(ids), ["123", "345", "8989", "21499"])
self.assertEqual(changelog.full_version, "3.8.7.4+0~mr3.8.7.4")
self.assertEqual(changelog.package, "ngcp-fake")
@override_settings(REPOAPI_TRACKER=Tracker.WORKFRONT)
@patch("builtins.open", mock_open(read_data=debian_changelog))
def test_parse_changelog_wf(self):
ids, changelog = utils.parse_changelog("/tmp/fake.txt")
self.assertListEqual(humansorted(ids), ["123", "345"])
self.assertEqual(changelog.full_version, "3.8.7.4+0~mr3.8.7.4")
self.assertEqual(changelog.package, "ngcp-fake")
@override_settings(REPOAPI_TRACKER=Tracker.MANTIS)
@patch("builtins.open", mock_open(read_data=debian_changelog))
def test_parse_changelog_mantis(self):
ids, changelog = utils.parse_changelog("/tmp/fake.txt")
self.assertListEqual(humansorted(ids), ["8989", "21499"])
self.assertEqual(changelog.full_version, "3.8.7.4+0~mr3.8.7.4")
self.assertEqual(changelog.package, "ngcp-fake")
def test_get_target_release(self):
from hotfix.models import NoteInfo
val = NoteInfo.get_target_release("3.8.7.4+0~mr3.8.7.4")
self.assertEqual(val, "mr3.8.7.4")
def test_get_target_release_ko(self):
from hotfix.models import NoteInfo
val = NoteInfo.get_target_release("3.8.7.4-1")
self.assertIsNone(val)

@ -12,52 +12,30 @@
#
# You should have received a copy of the GNU General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>.
import re
import structlog
from .models import WorkfrontNoteInfo
from .models import NoteInfo
from debian.changelog import Changelog
from repoapi import utils
hotfix_re_release = re.compile(r".+~(mr[0-9]+\.[0-9]+\.[0-9]+.[0-9]+)$")
logger = structlog.get_logger(__name__)
def process_hotfix(jbi_info, projectname, path):
model = NoteInfo.get_model()
logger.info("hotfix_released[%s] %s", jbi_info, path)
wids, changelog = parse_changelog(path)
for wid in wids:
create_note(wid, projectname, changelog.full_version)
ids, changelog = parse_changelog(path, model)
for wid in ids:
model.create(wid, projectname, changelog.full_version)
def parse_changelog(path):
def parse_changelog(path, model=None):
if model is None:
model = NoteInfo.get_model()
changelog = Changelog()
with open(path, "r") as file_changelog:
changelog.parse_changelog(file_changelog.read())
set_ids = set()
for block in changelog:
for change in block.changes():
set_ids = set_ids.union(WorkfrontNoteInfo.getIds(change))
set_ids = set_ids.union(model.getIds(change))
return (set_ids, changelog)
def get_target_release(version):
match = hotfix_re_release.search(version)
if match:
return match.group(1)
def create_note(wid, projectname, version):
wni = WorkfrontNoteInfo.objects
note, created = wni.get_or_create(
workfront_id=wid, projectname=projectname, version=version
)
if created:
msg = "hotfix %s.git %s triggered" % (note.projectname, note.version)
utils.workfront_note_send(wid, msg)
target_release = get_target_release(note.version)
if target_release:
utils.workfront_set_release_target(wid, target_release)

@ -28,7 +28,7 @@ class RepoAPIConf(AppConf):
ARTIFACT_JOB_REGEX = [
".*-repos$",
]
TRACKER = Tracker.NONE
TRACKER = Tracker.MANTIS
class Meta:
prefix = "repoapi"

@ -177,6 +177,11 @@ structlog.configure(
)
JENKINS_TOKEN = "sipwise_jenkins_ci"
MANTIS_TOKEN = "fake_mantis_token"
MANTIS_TARGET_RELEASE = {
"id": 77,
"name": "Target Release",
}
CELERY_TASK_SERIALIZER = "json"
CELERY_RESULT_SERIALIZER = "json"

@ -69,6 +69,10 @@ GERRIT_URL = server_config.get("gerrit", "URL")
GERRIT_REST_HTTP_USER = server_config.get("gerrit", "HTTP_USER")
GERRIT_REST_HTTP_PASSWD = server_config.get("gerrit", "HTTP_PASSWD")
MANTIS_URL = server_config.get("mantis", "URL")
MANTIS_TOKEN = server_config.get("mantis", "TOKEN")
MANTIS_TARGET_RELEASE["id"] = 75 # noqa
DOCKER_REGISTRY_URL = server_config.get("server", "DOCKER_REGISTRY_URL")
AUTH_LDAP_SERVER_URI = server_config.get("server", "AUTH_LDAP_SERVER_URI")
AUTH_LDAP_USER_BASE = server_config.get("server", "AUTH_LDAP_USER_BASE")

@ -17,6 +17,7 @@ import os
from pathlib import Path
from .common import * # noqa
from repoapi.conf import Tracker
# pylint: disable=W0401,W0614
@ -58,6 +59,8 @@ GERRIT_REST_HTTP_PASSWD = "verysecrethttppasswd"
GITWEB_URL = "https://git.local/gitweb/?p={}.git;a=commit;h={}"
WORKFRONT_CREDENTIALS = BASE_DIR / ".workfront.ini"
DOCKER_REGISTRY_URL = "https://localhost:5000/v2/{}"
MANTIS_URL = "https://support.local/api/rest/{}"
# fake info
DOCKER_REGISTRY = """
{"repositories":["comx-fs-test-jessie","data-hal-jessie","documentation-jessie","janus-admin-jessie","janus-client-jessie","jenkins-configs","jenkins-configs-jessie","kamailio-config-tests-jessie","libswrate-jessie","libtcap-jessie","lua-ngcp-kamailio","lua-ngcp-kamailio-jenkins","lua-ngcp-kamailio-jessie","ngcp-csc-jessie","ngcp-panel-selenium","ngcp-panel-tests-rest-api-jessie","ngcp-panel-tests-selenium-jessie","ngcp-rate-o-mat-unit-tests-jessie","ngcp-rtcengine-test-jessie","ngcp-rtcengine-tests-selenium-jessie","ngcp-rtcengine-tests-selenium-stretch","ngcp-sipwise-snmp-mibs-jessie","ngcp-snmp-jessie","ngcpcfg-jessie","ossbss-perl-testing-wheezy","puppet-octocatalog-diff","puppet-sipwise-jessie","rate-o-mat-functional-tests-jessie","rate-o-mat-jessie","release-dashboard","repoapi-jessie","repos-scripts-jessie","rtpengine-jessie","sipphone-android","sipwise/ce-trunk","sipwise/mr3.8.10","sipwise/mr3.8.2","sipwise/mr3.8.3","sipwise/mr3.8.4","sipwise/mr3.8.5","sipwise/mr3.8.6","sipwise/mr3.8.7","sipwise/mr3.8.8","sipwise/mr3.8.9","sipwise/mr4.0.1","sipwise/mr4.0.2","sipwise/mr4.1.1","sipwise/mr4.1.2","sipwise/mr4.2.1","sipwise/mr4.2.2","sipwise/mr4.3.1","sipwise/mr4.3.2","sipwise/mr4.4.1","sipwise/mr4.4.2","sipwise/mr4.5.1","sipwise/mr4.5.2","sipwise/mr4.5.3","sipwise/mr4.5.4","sipwise/mr5.0.1","sipwise/mr5.0.2","sipwise/mr5.1.1","sipwise/mr5.1.2","sipwise/mr5.2.1","sipwise/mr5.3.1","sipwise-jessie","sipwise-stretch","sipwise-webpage-soap-docker-jessie","sipwise-webpage-soap-jessie","sipwise-wheezy","system-tools-jessie"]}
@ -107,3 +110,6 @@ JBI_ARTIFACT_JOBS = [
]
REPOAPI_ARTIFACT_JOB_REGEX = []
JBI_ALLOWED_HOSTS = ["jenkins-dev.mgm.sipwise.com"]
# no tracker
REPOAPI_TRACKER = Tracker.NONE

@ -12,6 +12,7 @@
#
# You should have received a copy of the GNU General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>.
import json
import re
import subprocess
from pathlib import Path
@ -28,6 +29,10 @@ JBI_CONSOLE_URL = "{}/job/{}/{}/consoleText"
JBI_BUILD_URL = "{}/job/{}/{}/api/json"
JBI_ARTIFACT_URL = "{}/job/{}/{}/artifact/{}"
JBI_ENVVARS_URL = "{}/job/{}/{}/injectedEnvVars/api/json"
MANTIS_HEADERS = {
"Authorization": settings.MANTIS_TOKEN,
"Content-Type": "application/json",
}
def executeAndReturnOutput(command, env=None):
@ -225,3 +230,43 @@ def is_download_artifacts(jobname):
if re.search(check, jobname) is not None:
return True
return False
def mantis_query(method, url, payload):
logger.bind(
method=method,
url=url,
payload=payload,
)
response = requests.request(
f"{method}", url, headers=MANTIS_HEADERS, data=payload
)
response.raise_for_status()
return response
def mantis_note_send(_id, message):
url = settings.MANTIS_URL.format(f"issues/{_id}/notes")
payload = json.dumps(
{"text": f"{message}", "view_state": {"name": "private"}}
)
mantis_query("POST", url, payload)
def mantis_set_release_target(_id, msg):
url = settings.MANTIS_URL.format(f"issues/{_id}")
cf = settings.MANTIS_TARGET_RELEASE
payload = json.dumps(
{
"custom_fields": [
{
"field": {
"id": cf["id"],
"name": cf["name"],
},
"value": f"{msg}",
},
]
}
)
mantis_query("PATCH", url, payload)

Loading…
Cancel
Save