TT#18651 provide beat service for periodic tasks

* purge trunk release

Change-Id: I783e45de40d6c6aa69db41c00ed8115a8eacecac

debian/repoapi.repoapi-beat.service

@@ -0,0 +1,17 @@
+[Unit]
+Description=Celery Beat Service
+After=network.target rabbitmq-server.service
+
+[Service]
+Type=simple
+User=www-data
+Group=www-data
+Environment=DJANGO_SETTINGS_MODULE=repoapi.settings.prod
+PIDFile=/var/lib/repoapi/celery-beat.pid
+WorkingDirectory=/usr/share/repoapi
+ExecStart=/var/lib/repoapi/venv_prod/bin/python ./manage.py celery beat \
+		--schedule=/var/lib/repoapi/celery-beat.schedule \
+        --pidfile=/var/lib/repoapi/celery-beat.pid --loglevel=INFO
+
+[Install]
+WantedBy=multi-user.target

debian/rules

@@ -31,4 +31,5 @@ override_dh_fixperms:
 override_dh_systemd_enable:
 	dh_systemd_enable --name=repoapi-worker
 	dh_systemd_enable --name=repoapi-flower
+	dh_systemd_enable --name=repoapi-beat
 

repoapi/models/jbi.py

@@ -15,6 +15,7 @@
 import logging
 import re
 from collections import OrderedDict
+from datetime import datetime, timedelta
 
 from django.db import models
 from django.forms.models import model_to_dict
@@ -134,6 +135,12 @@ class JenkinsBuildInfoManager(models.Manager):
             latest_uuid['latest'] = (res.tag == uuid)
         return latest_uuid
 
+    def purge_release(self, release, _timedelta=timedelta(weeks=3)):
+        _date = datetime.now() - _timedelta
+        self.get_queryset().filter(
+            param_release=release,
+            date__date__lt=_date).delete()
+
 
 class JenkinsBuildInfo(models.Model):
     tag = models.CharField(max_length=64, null=True)

repoapi/settings/prod.py

@@ -18,6 +18,7 @@ import os
 from ConfigParser import RawConfigParser
 # pylint: disable=W0401,W0614
 from .common import *
+from celery.schedules import crontab
 
 BASE_DIR = os.path.dirname(
     os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
@@ -77,6 +78,16 @@ BUILD_KEY_AUTH = True
 
 # celery
 BROKER_URL = server_config.get('server', 'BROKER_URL')
+CELERYBEAT_SCHEDULE = {
+    # Executes every Sunday morning at 7:30 A.M
+    'purge-trunk': {
+        'task': 'repoapi.tasks.jbi_purge',
+        'schedule': crontab(hour=7, minute=30, day_of_week='sunday'),
+        'args': ('none', 4),
+    },
+}
+CELERY_TIMEZONE = 'UTC'
+
 JBI_BASEDIR = os.path.join(VAR_DIR, 'jbi_files')
 JBI_ARTIFACT_JOBS = [
     'release-tools-runner',

repoapi/tasks.py

@@ -13,12 +13,13 @@
 # You should have received a copy of the GNU General Public License along
 # with this program.  If not, see <http://www.gnu.org/licenses/>.
 from __future__ import absolute_import
-
+from datetime import timedelta
 import json
 import logging
 from os.path import basename
 from celery import shared_task
 from django.conf import settings
+from django.apps import apps
 from .celery import jbi_parse_hotfix
 from .utils import jenkins_get_console, jenkins_get_artifact
 from .utils import jenkins_get_build, jenkins_get_env
@@ -46,3 +47,12 @@ def get_jbi_files(jbi_id, jobname, buildnumber):
             jbi_get_artifact.delay(jbi_id, jobname, buildnumber, artifact)
     else:
         logger.debug("skip artifacts download for jobname: %s", jobname)
+
+
+@shared_task(ignore_result=True)
+def jbi_purge(release, weeks):
+    JenkinsBuildInfo = apps.get_model("repoapi", "JenkinsBuildInfo")
+    JenkinsBuildInfo.objects.purge_release(
+        release,
+        timedelta(weeks=weeks))
+    logger.info("purged release %s jbi older than %s weeks" % (release, weeks))

repoapi/test/test_model_queries.py

@@ -16,6 +16,7 @@
 from django.utils.dateparse import parse_datetime
 from repoapi.models import JenkinsBuildInfo
 from repoapi.test.base import BaseTest
+from datetime import datetime, timedelta
 
 
 class JBIQueriesTestCase(BaseTest):
@@ -59,3 +60,13 @@ class JBIQueriesTestCase(BaseTest):
         date = parse_datetime("2015-05-04T17:04:57.802Z")
         self.assertEquals(JenkinsBuildInfo.objects.latest_uuid(
             'mr3.1-fake', 'fake'), {'tag': 'UUID1', 'date': date})
+
+    def test_purge_release(self):
+        jbi = JenkinsBuildInfo.objects.get(pk=1)
+        jbi.date = datetime.now()
+        jbi.save()
+        self.assertEquals(JenkinsBuildInfo.objects.count(), 5)
+        JenkinsBuildInfo.objects.purge_release(
+            'mr3.1-fake',
+            timedelta(weeks=3))
+        self.assertEquals(JenkinsBuildInfo.objects.count(), 1)

repoapi/test/test_tasks.py

@@ -0,0 +1,32 @@
+# Copyright (C) 2017 The Sipwise Team - http://sipwise.com
+
+# This program is free software: you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from repoapi.models import JenkinsBuildInfo
+from django.test import override_settings
+from repoapi.test.base import BaseTest
+from repoapi import tasks
+from datetime import datetime, timedelta
+
+
+class TasksTestCase(BaseTest):
+    fixtures = ['test_model_queries.json']
+
+    def test_purge(self):
+        jbi = JenkinsBuildInfo.objects.get(pk=1)
+        jbi.date = datetime.now()
+        jbi.save()
+        self.assertEquals(JenkinsBuildInfo.objects.count(), 5)
+        tasks.jbi_purge.delay('mr3.1-fake', 3)
+        self.assertEquals(JenkinsBuildInfo.objects.count(), 1)