From a6b7a49f366f43cbc3ea04f47f29e237a9d479b4 Mon Sep 17 00:00:00 2001 From: Victor Seva Date: Fri, 16 Dec 2022 15:53:10 +0100 Subject: [PATCH] MT#55988 buildinfo: fix duration field > [2022-12-16 06:07:01,563: ERROR/ForkPoolWorker-3] Task buildinfo.tasks.parse_buildinfo[bdf0ea9f-5b13-4d08-b23b-aa2ff3a61c40] raised unexpected: DataError('smallint out of range\n') * define datetime field instead of timestamp, that would be filled with the processed value of timestamp Change-Id: Id6c9dac31b5ad9e32890e399c16ff6d9bad06746 --- buildinfo/migrations/0003_duration_fix.py | 18 +++++++++++ .../migrations/0004_buildinfo_datetime.py | 30 +++++++++++++++++++ buildinfo/models.py | 19 ++++++++++-- buildinfo/tests/test_models.py | 2 +- buildinfo/tests/test_utils.py | 20 ++++++++++++- buildinfo/utils.py | 13 ++++++-- 6 files changed, 95 insertions(+), 7 deletions(-) create mode 100644 buildinfo/migrations/0003_duration_fix.py create mode 100644 buildinfo/migrations/0004_buildinfo_datetime.py diff --git a/buildinfo/migrations/0003_duration_fix.py b/buildinfo/migrations/0003_duration_fix.py new file mode 100644 index 0000000..3664f9c --- /dev/null +++ b/buildinfo/migrations/0003_duration_fix.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.15 on 2022-12-16 13:56 +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + + dependencies = [ + ("buildinfo", "0002_ldap_grp_perms"), + ] + + operations = [ + migrations.AlterField( + model_name="buildinfo", + name="duration", + field=models.PositiveIntegerField(), + ), + ] diff --git a/buildinfo/migrations/0004_buildinfo_datetime.py b/buildinfo/migrations/0004_buildinfo_datetime.py new file mode 100644 index 0000000..496ee9a --- /dev/null +++ b/buildinfo/migrations/0004_buildinfo_datetime.py @@ -0,0 +1,30 @@ +# Generated by Django 3.2.15 on 2022-12-16 14:39 +import django.utils.timezone +from django.db import migrations +from django.db import models + +from buildinfo.utils import get_datetime + + +def migrate_timestamps(apps, schema_editor): + BuildInfo = apps.get_model("buildinfo", "BuildInfo") + for info in BuildInfo.objects.all(): + info.datetime = get_datetime(info.timestamp) + + +class Migration(migrations.Migration): + + dependencies = [ + ("buildinfo", "0003_duration_fix"), + ] + + operations = [ + migrations.AddField( + model_name="buildinfo", + name="datetime", + field=models.DateTimeField(default=django.utils.timezone.now), + preserve_default=False, + ), + migrations.RunPython(migrate_timestamps), + migrations.RemoveField(model_name="buildinfo", name="timestamp"), + ] diff --git a/buildinfo/models.py b/buildinfo/models.py index d4725e5..20c9a85 100644 --- a/buildinfo/models.py +++ b/buildinfo/models.py @@ -12,13 +12,27 @@ # # You should have received a copy of the GNU General Public License along # with this program. If not, see . +from typing import Any + from django.db import models +from .utils import datetime +from .utils import get_datetime + + +class BuildInfoManager(models.Manager): + def create(self, **kwargs: Any) -> Any: + if "datetime" in kwargs: + val = kwargs.get("datetime") + if not isinstance(val, datetime): + kwargs["datetime"] = get_datetime(val) + return super().create(**kwargs) + class BuildInfo(models.Model): builton = models.CharField(max_length=50, null=False) - timestamp = models.PositiveBigIntegerField(null=False) - duration = models.PositiveSmallIntegerField(null=False) + datetime = models.DateTimeField(null=False) + duration = models.PositiveIntegerField(null=False) projectname = models.CharField(max_length=100, null=False) buildnumber = models.IntegerField() @@ -29,6 +43,7 @@ class BuildInfo(models.Model): param_release_uuid = models.CharField(max_length=64, null=True, blank=True) param_distribution = models.CharField(max_length=50, null=True, blank=True) param_ppa = models.CharField(max_length=50, null=True, blank=True) + objects = BuildInfoManager() def __str__(self): return ( diff --git a/buildinfo/tests/test_models.py b/buildinfo/tests/test_models.py index 23ed854..1a94dcc 100644 --- a/buildinfo/tests/test_models.py +++ b/buildinfo/tests/test_models.py @@ -28,7 +28,7 @@ class TestBuildInfo(BaseTest): "param_distribution": "wheezy", "param_ppa": "gerrit_MT10339_review2054", "builton": "fake-slave-1", - "timestamp": 0, + "datetime": 1668083617940, "duration": 0, } return defaults diff --git a/buildinfo/tests/test_utils.py b/buildinfo/tests/test_utils.py index 74c130d..6134f8e 100644 --- a/buildinfo/tests/test_utils.py +++ b/buildinfo/tests/test_utils.py @@ -12,9 +12,12 @@ # # You should have received a copy of the GNU General Public License along # with this program. If not, see . +from datetime import datetime from unittest.mock import mock_open from unittest.mock import patch +from django.utils import timezone + from buildinfo import models from buildinfo import utils from repoapi.models import JenkinsBuildInfo @@ -45,6 +48,17 @@ build_info = """{ class TestBuildInfo(BaseTest): + dt = datetime( + 2022, + 11, + 10, + 12, + 33, + 37, + 940000, + tzinfo=timezone.get_current_timezone(), + ) + def get_defaults(self, info=False): defaults = { "projectname": "fake", @@ -66,7 +80,7 @@ class TestBuildInfo(BaseTest): defaults.update( { "builton": "fake-slave-1", - "timestamp": 0, + "datetime": 1668083617940, "duration": 0, } ) @@ -74,6 +88,9 @@ class TestBuildInfo(BaseTest): defaults.update(jbi_defaults) return defaults + def test_get_datetime(self): + self.assertEqual(utils.get_datetime(1668083617940), self.dt) + @patch("builtins.open", mock_open(read_data=build_info)) @patch("repoapi.utils.dlfile") def test_process_buildinfo(self, dlfile): @@ -84,3 +101,4 @@ class TestBuildInfo(BaseTest): utils.process_buildinfo(jbi.pk, "/tmp/fake.txt") qs = models.BuildInfo.objects.filter(builton="jenkins-slave13") self.assertEqual(qs.count(), 1) + self.assertEqual(qs.first().datetime, self.dt) diff --git a/buildinfo/utils.py b/buildinfo/utils.py index 7a8380c..87e0762 100644 --- a/buildinfo/utils.py +++ b/buildinfo/utils.py @@ -13,23 +13,30 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . import json +from datetime import datetime import structlog from django.apps import apps - -from .models import BuildInfo +from django.utils import timezone logger = structlog.get_logger(__name__) +def get_datetime(timestamp: float) -> datetime: + return datetime.fromtimestamp( + timestamp / 1000, tz=timezone.get_current_timezone() + ) + + def process_buildinfo(jbi_id: int, path: str): JenkinsBuildInfo = apps.get_model("repoapi", "JenkinsBuildInfo") jbi = JenkinsBuildInfo.objects.get(pk=jbi_id) + BuildInfo = apps.get_model("buildinfo", "BuildInfo") with open(path, "r") as file: info = json.load(file) BuildInfo.objects.create( builton=info["builtOn"], - timestamp=info["timestamp"], + datetime=info["timestamp"], duration=info["duration"], projectname=jbi.projectname, jobname=jbi.jobname,