# Copyright (C) 2017-2024 The Sipwise Team - http://sipwise.com # # This program is free software: you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the Free # Software Foundation, either version 3 of the License, or (at your option) # any later version. # # This program is distributed in the hope that it will be useful, but WITHOUT # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for # more details. # # You should have received a copy of the GNU General Public License along # with this program. If not, see . import datetime import structlog from django.db import models from django.db.models import Q from django.forms.models import model_to_dict from django.utils import timezone from .conf import settings from .exceptions import BuildReleaseUnique from .exceptions import PreviousBuildNotDone from .utils import get_simple_release from .utils import ReleaseConfig from .utils import remove_from_textlist from repoapi.models import JenkinsBuildInfo from repoapi.utils import regex_mrXX from repoapi.utils import regex_mrXX_up from repoapi.utils import regex_mrXXX logger = structlog.get_logger(__name__) build_release_jobs = ",".join(settings.BUILD_RELEASE_JOBS) release_jobs_len = len(build_release_jobs) + 1 class BuildReleaseManager(models.Manager): _jbi = JenkinsBuildInfo.objects def release(self, version, distribution): qs = self.get_queryset() return qs.filter( Q(release=version, distribution=distribution) | Q(release="{}-update".format(version), distribution=distribution) ) def create_build_release(self, uuid, release, fake=False): config = ReleaseConfig(release) qs = self.get_queryset() br = qs.filter( release=config.release, distribution=config.debian_release ) release_ok = config.release if br.exists(): if regex_mrXXX.match(config.branch): msg = f"release[mrX.Y.Z]:{release} has already a build" raise BuildReleaseUnique(msg) elif regex_mrXX.match(config.branch): release_ok = f"{config.release}-update" msg = ( f"release[mrX.Y]:{config.branch} has already a build, " f"set {release_ok} as release" ) logger.info(msg) if not br.last().done: logger.info(f"release:{release} is already building") raise PreviousBuildNotDone(msg) projects = ",".join(config.projects) if fake: start_date = timezone.make_aware(datetime.datetime(1977, 1, 1)) built_projects = ",".join( list(settings.BUILD_RELEASE_JOBS) + config.projects ) else: start_date = timezone.now() built_projects = None return self.create( start_date=start_date, uuid=uuid, tag=config.tag, branch=config.branch, release=release_ok, distribution=config.debian_release, projects=projects, built_projects=built_projects, ) def jbi(self, release_uuid): qs = self._jbi.get_queryset() return qs.filter(param_release_uuid=release_uuid) def release_jobs(self, release_uuid, flat=True): qs = self._jbi.get_queryset() res = qs.filter( jobname__in=settings.BUILD_RELEASE_JOBS, param_release_uuid=release_uuid, ).distinct() if res.exists(): if flat: return res.values_list("jobname", flat=True) else: return res.values("jobname") def release_jobs_full(self, release_uuid): res = dict() jobs = self.release_jobs(release_uuid) if jobs is None: return res for job in jobs: uuids = self.release_jobs_uuids(release_uuid, job) res[job] = [model_to_dict(x) for x in uuids] return res def release_jobs_uuids(self, release_uuid, job): qs = self._jbi.get_queryset() params = { "param_release_uuid": release_uuid, "jobname": job, "tag__isnull": False, } res = qs.filter(**params).distinct() if res.exists(): return res.order_by("-date") def releases_with_builds(self): qs = self.get_queryset() res = set() for br in qs.all(): res.add(get_simple_release(br.release)) return res class BuildRelease(models.Model): uuid = models.CharField(max_length=64, unique=True, null=False) start_date = models.DateTimeField( blank=True, editable=False, default=timezone.now ) tag = models.CharField(max_length=50, null=True, blank=True) branch = models.CharField(max_length=50, null=False) release = models.CharField(max_length=50, null=False, db_index=True) distribution = models.CharField(max_length=50, null=False, editable=False) projects = models.TextField(null=False, editable=False) built_projects = models.TextField(null=True, editable=False) triggered_projects = models.TextField(null=True, editable=False) failed_projects = models.TextField(null=True, editable=False) pool_size = models.SmallIntegerField(default=0, editable=False) triggered_jobs = models.TextField(null=True, editable=False) objects = BuildReleaseManager() class Meta: permissions = [ ("can_trigger", "can trigger build releases"), ("can_trigger_hotfix", "can trigger hotfix builds"), ] def __str__(self): return "%s[%s]" % (self.release, self.uuid) def refresh_projects(self): old_projects = set(self.projects_list) self.projects = ",".join(self.config.projects) new_list = set(self.projects_list) append_list = [item for item in new_list if item not in old_projects] removed_list = [item for item in old_projects if item not in new_list] for project in removed_list: remove_from_textlist(self, "built_projects", project) remove_from_textlist(self, "triggered_projects", project) remove_from_textlist(self, "failed_projects", project) JenkinsBuildInfo.objects.filter( projectname__in=removed_list, param_release_uuid=self.uuid, ).delete() self.save() return (append_list, removed_list) def resume(self): if not self.done: from .tasks import build_resume build_resume.delay(self.id) @property def last_update(self): job = BuildRelease.objects.jbi(self.uuid).order_by("-date").first() if job: return job.date @property def is_update(self): return regex_mrXX_up.match(self.release) is not None @property def done(self): if self.built_projects is None: return False built_len = len(self.built_projects) if self.is_update: return built_len == len(self.projects) else: return built_len == release_jobs_len + len(self.projects) @property def projects_list(self): return [x.strip() for x in self.projects.split(",")] @property def built_projects_list(self): if self.built_projects is not None: return [x.strip() for x in self.built_projects.split(",")] return [] @property def queued_projects_list(self): return [ x for x in self.projects_list if x not in self.built_projects_list ] @property def failed_projects_list(self): if self.failed_projects is not None: return [x.strip() for x in self.failed_projects.split(",")] return [] @property def triggered_projects_list(self): if self.triggered_projects is not None: return [x.strip() for x in self.triggered_projects.split(",")] return [] @property def triggered_jobs_list(self): if self.triggered_jobs is not None: return [x.strip() for x in self.triggered_jobs.split(",")] return [] def append_triggered_job(self, value): if value in self.triggered_jobs_list: return False if self.triggered_jobs is None: self.triggered_jobs = value else: self.triggered_jobs += ",{}".format(value) self.save() return True def append_triggered(self, value): if value in self.triggered_projects_list: return False if self.triggered_projects is None: self.triggered_projects = value else: self.triggered_projects += ",{}".format(value) self.pool_size += 1 self.save() return True def _append_failed(self, value): if value in self.failed_projects_list: return False if self.failed_projects is None: self.failed_projects = value else: self.failed_projects += ",{}".format(value) if self.pool_size > 0: self.pool_size -= 1 self.save() return True def _append_built(self, value): if value in self.built_projects_list: return False if self.built_projects is None: self.built_projects = value else: self.built_projects += ",{}".format(value) failed_projects = self.failed_projects_list if value in failed_projects: failed_projects.remove(value) fp = ",".join(failed_projects) if len(fp) > 0: self.failed_projects = fp else: self.failed_projects = None if self.pool_size > 0: self.pool_size -= 1 self.save() return True def remove_triggered(self, jbi): remove_from_textlist(self, "triggered_projects", jbi.projectname) def append_built(self, jbi): jobname = jbi.jobname if jbi.result == "FAILURE": if jobname.endswith("-piuparts"): return False return self._append_failed(jbi.projectname) is_repos = jobname.endswith("-repos") is_rj = jobname in settings.BUILD_RELEASE_JOBS if is_repos or is_rj: if jbi.result in ["SUCCESS", "UNSTABLE"]: return self._append_built(jbi.projectname) return False @property def branch_or_tag(self): if self.tag: return "tag/{}".format(self.tag) return "branch/{}".format(self.branch) def _next(self, exclude=[]): structlog.contextvars.bind_contextvars(release=f"{self}") if self.built_projects is None: return self.build_deps[0][0] if self.done: return t_list = self.triggered_projects_list built_list = self.built_projects_list deps_missing = [] for grp in self.build_deps: for prj in grp: if prj not in built_list: if prj not in t_list: return prj else: deps_missing.append(prj) else: if len(deps_missing) > 0: logger.info( "release has build_deps missing", deps_missing=deps_missing, ) return None for prj in self.projects_list: if prj in exclude: continue if prj not in built_list and prj not in t_list: return prj @property def next(self): failed_projects = self.failed_projects_list structlog.contextvars.bind_contextvars(release=f"{self}") if any(job in failed_projects for job in settings.BUILD_RELEASE_JOBS): logger.info( "release has failed release_jobs, stop sending jobs", failed_projects=failed_projects, ) return prj = self._next(exclude=failed_projects) if prj in failed_projects and self.config.is_build_dep(prj): return None return prj @property def build_release(self) -> str: if self.release in ["trunk", "trunk-next"]: return "trunk" return self.release @property def build_deps(self) -> list: return self.config.levels_build_deps @property def config(self): if getattr(self, "_config", None) is None: self._config = ReleaseConfig( self.release, distribution=self.distribution ) return self._config