From 7c335c3b427ee498a690f2aec6d680bbbde72ff7 Mon Sep 17 00:00:00 2001 From: Victor Seva Date: Fri, 14 Jul 2017 17:08:28 +0200 Subject: [PATCH] TT#18651 provide beat service for periodic tasks * purge trunk release Change-Id: I783e45de40d6c6aa69db41c00ed8115a8eacecac --- debian/repoapi.repoapi-beat.service | 17 +++++++++++++++ debian/rules | 1 + repoapi/models/jbi.py | 7 +++++++ repoapi/settings/prod.py | 11 ++++++++++ repoapi/tasks.py | 12 ++++++++++- repoapi/test/test_model_queries.py | 11 ++++++++++ repoapi/test/test_tasks.py | 32 +++++++++++++++++++++++++++++ 7 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 debian/repoapi.repoapi-beat.service create mode 100644 repoapi/test/test_tasks.py diff --git a/debian/repoapi.repoapi-beat.service b/debian/repoapi.repoapi-beat.service new file mode 100644 index 0000000..d1ccf9b --- /dev/null +++ b/debian/repoapi.repoapi-beat.service @@ -0,0 +1,17 @@ +[Unit] +Description=Celery Beat Service +After=network.target rabbitmq-server.service + +[Service] +Type=simple +User=www-data +Group=www-data +Environment=DJANGO_SETTINGS_MODULE=repoapi.settings.prod +PIDFile=/var/lib/repoapi/celery-beat.pid +WorkingDirectory=/usr/share/repoapi +ExecStart=/var/lib/repoapi/venv_prod/bin/python ./manage.py celery beat \ + --schedule=/var/lib/repoapi/celery-beat.schedule \ + --pidfile=/var/lib/repoapi/celery-beat.pid --loglevel=INFO + +[Install] +WantedBy=multi-user.target diff --git a/debian/rules b/debian/rules index 291a0de..b90033f 100755 --- a/debian/rules +++ b/debian/rules @@ -31,4 +31,5 @@ override_dh_fixperms: override_dh_systemd_enable: dh_systemd_enable --name=repoapi-worker dh_systemd_enable --name=repoapi-flower + dh_systemd_enable --name=repoapi-beat diff --git a/repoapi/models/jbi.py b/repoapi/models/jbi.py index 2aacd6b..de7af8f 100644 --- a/repoapi/models/jbi.py +++ b/repoapi/models/jbi.py @@ -15,6 +15,7 @@ import logging import re from collections import OrderedDict +from datetime import datetime, timedelta from django.db import models from django.forms.models import model_to_dict @@ -134,6 +135,12 @@ class JenkinsBuildInfoManager(models.Manager): latest_uuid['latest'] = (res.tag == uuid) return latest_uuid + def purge_release(self, release, _timedelta=timedelta(weeks=3)): + _date = datetime.now() - _timedelta + self.get_queryset().filter( + param_release=release, + date__date__lt=_date).delete() + class JenkinsBuildInfo(models.Model): tag = models.CharField(max_length=64, null=True) diff --git a/repoapi/settings/prod.py b/repoapi/settings/prod.py index e20d2e3..998cb99 100644 --- a/repoapi/settings/prod.py +++ b/repoapi/settings/prod.py @@ -18,6 +18,7 @@ import os from ConfigParser import RawConfigParser # pylint: disable=W0401,W0614 from .common import * +from celery.schedules import crontab BASE_DIR = os.path.dirname( os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) @@ -77,6 +78,16 @@ BUILD_KEY_AUTH = True # celery BROKER_URL = server_config.get('server', 'BROKER_URL') +CELERYBEAT_SCHEDULE = { + # Executes every Sunday morning at 7:30 A.M + 'purge-trunk': { + 'task': 'repoapi.tasks.jbi_purge', + 'schedule': crontab(hour=7, minute=30, day_of_week='sunday'), + 'args': ('none', 4), + }, +} +CELERY_TIMEZONE = 'UTC' + JBI_BASEDIR = os.path.join(VAR_DIR, 'jbi_files') JBI_ARTIFACT_JOBS = [ 'release-tools-runner', diff --git a/repoapi/tasks.py b/repoapi/tasks.py index 8d024ed..2c2e519 100644 --- a/repoapi/tasks.py +++ b/repoapi/tasks.py @@ -13,12 +13,13 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . from __future__ import absolute_import - +from datetime import timedelta import json import logging from os.path import basename from celery import shared_task from django.conf import settings +from django.apps import apps from .celery import jbi_parse_hotfix from .utils import jenkins_get_console, jenkins_get_artifact from .utils import jenkins_get_build, jenkins_get_env @@ -46,3 +47,12 @@ def get_jbi_files(jbi_id, jobname, buildnumber): jbi_get_artifact.delay(jbi_id, jobname, buildnumber, artifact) else: logger.debug("skip artifacts download for jobname: %s", jobname) + + +@shared_task(ignore_result=True) +def jbi_purge(release, weeks): + JenkinsBuildInfo = apps.get_model("repoapi", "JenkinsBuildInfo") + JenkinsBuildInfo.objects.purge_release( + release, + timedelta(weeks=weeks)) + logger.info("purged release %s jbi older than %s weeks" % (release, weeks)) diff --git a/repoapi/test/test_model_queries.py b/repoapi/test/test_model_queries.py index 61eab47..88107c4 100644 --- a/repoapi/test/test_model_queries.py +++ b/repoapi/test/test_model_queries.py @@ -16,6 +16,7 @@ from django.utils.dateparse import parse_datetime from repoapi.models import JenkinsBuildInfo from repoapi.test.base import BaseTest +from datetime import datetime, timedelta class JBIQueriesTestCase(BaseTest): @@ -59,3 +60,13 @@ class JBIQueriesTestCase(BaseTest): date = parse_datetime("2015-05-04T17:04:57.802Z") self.assertEquals(JenkinsBuildInfo.objects.latest_uuid( 'mr3.1-fake', 'fake'), {'tag': 'UUID1', 'date': date}) + + def test_purge_release(self): + jbi = JenkinsBuildInfo.objects.get(pk=1) + jbi.date = datetime.now() + jbi.save() + self.assertEquals(JenkinsBuildInfo.objects.count(), 5) + JenkinsBuildInfo.objects.purge_release( + 'mr3.1-fake', + timedelta(weeks=3)) + self.assertEquals(JenkinsBuildInfo.objects.count(), 1) diff --git a/repoapi/test/test_tasks.py b/repoapi/test/test_tasks.py new file mode 100644 index 0000000..eea3c95 --- /dev/null +++ b/repoapi/test/test_tasks.py @@ -0,0 +1,32 @@ +# Copyright (C) 2017 The Sipwise Team - http://sipwise.com + +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. + +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. + +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . + +from repoapi.models import JenkinsBuildInfo +from django.test import override_settings +from repoapi.test.base import BaseTest +from repoapi import tasks +from datetime import datetime, timedelta + + +class TasksTestCase(BaseTest): + fixtures = ['test_model_queries.json'] + + def test_purge(self): + jbi = JenkinsBuildInfo.objects.get(pk=1) + jbi.date = datetime.now() + jbi.save() + self.assertEquals(JenkinsBuildInfo.objects.count(), 5) + tasks.jbi_purge.delay('mr3.1-fake', 3) + self.assertEquals(JenkinsBuildInfo.objects.count(), 1)