TT#17756 docker: support remove tags

* we need to use reference ( docker API it doesn't support tag name )
* fix tag date (value in the first occurrence not last)

Change-Id: I2fa2f7bedaca4cdeeb3a3a2d4a3f4a661d15fad1
changes/76/14076/6
Victor Seva 8 years ago
parent 731dd69df0
commit 629fde1c13

File diff suppressed because one or more lines are too long

@ -0,0 +1,22 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.9 on 2017-06-26 17:22
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('release_dashboard', '0002_dockerimage'),
]
operations = [
migrations.RunSQL(
[("DELETE FROM release_dashboard_dockertag;", None)]),
migrations.AddField(
model_name='dockertag',
name='reference',
field=models.CharField(max_length=150, unique=True),
),
]

@ -91,8 +91,10 @@ class Project(models.Model):
class DockerImageManager(models.Manager):
def images_with_tags(self):
def images_with_tags(self, project=None):
qs = self.get_queryset().filter(dockertag__isnull=False)
if project:
qs = qs.filter(project__name=project)
return qs.distinct()
@ -115,6 +117,7 @@ class DockerTag(models.Model):
name = models.CharField(max_length=50, null=False)
manifests = JSONField(null=False)
image = models.ForeignKey(DockerImage, on_delete=models.CASCADE)
reference = models.CharField(max_length=150, unique=True, null=False)
class Meta:
unique_together = (("name", "image"),)
@ -127,12 +130,11 @@ class DockerTag(models.Model):
if self.manifests is None:
return None
try:
value = self.manifests['history'][-1]['v1Compatibility']
value = self.manifests['history'][0]['v1Compatibility']
time = json.loads(value)
created = time['created'].split('.')
return datetime.strptime(
created[0],
'%Y-%m-%dT%H:%M:%S')
except Exception as e:
logger.error(e)
return None

@ -24,8 +24,16 @@ class ProjectSerializer(serializers.HyperlinkedModelSerializer):
fields = '__all__'
class DockerTagSerializer(serializers.HyperlinkedModelSerializer):
class Meta:
model = models.DockerTag
fields = '__all__'
class DockerImageSerializer(serializers.HyperlinkedModelSerializer):
project = serializers.StringRelatedField()
dockertag_set = DockerTagSerializer(many=True, read_only=True)
class Meta:
model = models.DockerImage

@ -0,0 +1,34 @@
$('button.delete').click(function(e){
// don't send the form
e.preventDefault();
var button = $(this);
var pk = button.attr('pk');
var span = $('span#' + pk);
var div = $('div.list-group-item#row_'+ pk);
function successFunc(data, status) {
span.html('Done');
div.addClass("hidden");
}
function errorFunc(jqXHR, status, error) {
span.html(error);
button.removeAttr("disabled");
}
if(confirm('Are you sure?')) {
$.ajax({
url: '/docker/tag/' + pk + '/',
type: 'DELETE',
success: successFunc,
error: errorFunc,
beforeSend: csrftokenFunc
});
//deactivate button
button.attr("disabled", "disabled");
span.html('processing');
span.show();
}
});

@ -44,10 +44,25 @@ def docker_fetch_info(imagename):
image = DockerImage.objects.get(name=imagename)
tags = docker.get_docker_tags(imagename)
for tagname in tags:
DockerTag.objects.create(
name=tagname,
image=image,
manifests=docker.get_docker_manifests(image.name, tagname))
manifest, digest = docker.get_docker_manifests(image.name, tagname)
if digest:
DockerTag.objects.create(
name=tagname,
image=image,
manifests=manifest,
reference=digest)
@shared_task(ignore_result=True)
def docker_fetch_project(projectname):
DockerImage.objects.filter(project__name=projectname).delete()
images = docker.get_docker_repositories()
project = Project.objects.get(name=projectname)
for imagename in project.filter_docker_images(images):
image = DockerImage.objects.create(name=imagename,
project=project)
logger.debug("%s created" % image)
docker_fetch_info.delay(image.name)
@shared_task(ignore_result=True)
@ -62,3 +77,10 @@ def docker_fetch_all():
project=project)
logger.debug("%s created" % image)
docker_fetch_info.delay(image.name)
@shared_task(ignore_result=True)
def docker_remove_tag(image_name, tag_name):
tag = DockerTag.objects.get(name=tag_name, image__name=image_name)
docker.delete_tag(image_name, tag.reference)
tag.delete()

@ -17,8 +17,16 @@
<tbody>
{% for i in images %}
<tr>
<th><label>{{ i.name }}</label></th>
<td>{{ i.project.name }}</td>
<th><label>
<a href="{% url 'release_dashboard:docker_image_tag' i.project.name i.name%}">
{{ i.name }}
</a>
</label></th>
<td>
<a href="{% url 'release_dashboard:docker_project_images' i.project.name%}">
{{ i.project.name }}
</a>
</td>
<td>
{% for tag in i.dockertag_set.all %}
<a href="{{URL_BASE}}{{i.name}}/manifests/{{tag.name}}">

@ -0,0 +1,14 @@
{% extends "release_dashboard/base.html" %}
{% load staticfiles %}
{% block title %}Build docker images per project{% endblock %}
{% block navlist %}
<li><a href="{% url 'release_dashboard:index'%}">Release Dashboard</a></li>
<li><a href="{% url 'release_dashboard:docker_images'%}">Docker Images</a></li>
{% endblock %}
{% block content %}
{% include "release_dashboard/docker_image_content.html" %}
{% endblock %}
{% block extrajs %}
<script src="{% static "release_dashboard/js/csrf.js" %}"></script>
<script src="{% static "release_dashboard/js/docker_image.js" %}"></script>
{% endblock %}

@ -0,0 +1,61 @@
<div class="container">
<form method="POST" class="form-inline">{% csrf_token %}
<div class="panel panel-default">
<div class="panel panel-default">
<div class="panel-heading">
<h3 class="panel-title">Data</h3>
</div>
<div class="panel-body">
<table class="table table-condensed">
<thead>
<tr>
<th>Image</th>
<th>Project</th>
<th>Tags</th>
</tr>
</thead>
<tbody>
{% for i in images %}
<tr>
<th><label>
<a href="{% url 'release_dashboard:docker_image_tag' i.project.name i.name%}">
{{ i.name }}
</a>
</label></th>
<td>
<a href="{% url 'release_dashboard:docker_project_images' i.project.name%}">
{{ i.project.name }}
</a>
</td>
<td>
<div class="list-group">
{% for tag in i.dockertag_set.all %}
<div id="row_{{tag.pk}}" class="list-group-item">
<a href="{{URL_BASE}}{{i.name}}/manifests/{{tag.name}}">
{% if tag.name|length > 10 %}
<span class="btn btn-warning">
{% else %}
<span class="btn btn-success">
{% endif %}
{{ tag.name|truncatechars:10 }}
</span>
</a>
<div class="pull-right">
<button pk="{{tag.pk}}" class="delete btn btn-danger">Delete</button>
<span id="{{tag.pk}}" class="hidden">Done</span>
</div>
<span class="text-info">
{{tag.date|date:"DATETIME_FORMAT"}}
</span>
</div>
{% endfor %}
</div>
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
</form>
</div>

@ -17,6 +17,8 @@ from django.test import TestCase
from release_dashboard.models import Project, DockerImage, DockerTag
import datetime
diobj = DockerImage.objects
class DockerImageTestCase(TestCase):
@ -24,13 +26,13 @@ class DockerImageTestCase(TestCase):
self.proj = Project.objects.create(name="fake")
def test_create(self):
image = DockerImage.objects.create(
image = diobj.create(
name='fake-jessie', project=self.proj)
self.assertItemsEqual(self.proj.dockerimage_set.all(),
[image, ])
def test_remove_image(self):
image = DockerImage.objects.create(
image = diobj.create(
name='fake-jessie', project=self.proj)
self.assertItemsEqual(self.proj.dockerimage_set.all(),
[image, ])
@ -38,12 +40,12 @@ class DockerImageTestCase(TestCase):
self.assertTrue(Project.objects.filter(name="fake").exists())
def test_remove_project(self):
image = DockerImage.objects.create(
image = diobj.create(
name='fake-jessie', project=self.proj)
self.assertItemsEqual(self.proj.dockerimage_set.all(), [image, ])
self.proj.delete()
self.assertFalse(Project.objects.filter(name="fake").exists())
self.assertFalse(DockerImage.objects.filter(name="fake").exists())
self.assertFalse(diobj.filter(name="fake").exists())
def test_filter_images(self):
images = ['fake-jessie', 'other', 'ngcp-fake', 'fake-more']
@ -52,7 +54,7 @@ class DockerImageTestCase(TestCase):
self.proj.filter_docker_images(images), images_ok)
def test_image_tags(self):
image = DockerImage.objects.create(
image = diobj.create(
name='fake-jessie', project=self.proj)
self.assertItemsEqual(image.tags, [])
DockerTag.objects.create(
@ -63,7 +65,8 @@ class DockerImageTestCase(TestCase):
DockerTag.objects.create(
name='mr5.4',
image=image,
manifests='{}')
manifests='{}',
reference='whatever')
self.assertItemsEqual(image.tags, ['latest', 'mr5.4'])
@ -72,22 +75,33 @@ class DockerImageTest2Case(TestCase):
def setUp(self):
self.images_with_tags = [
DockerImage.objects.get(name='data-hal-jessie'),
DockerImage.objects.get(name='documentation-jessie'),
DockerImage.objects.get(name='ngcp-panel-selenium'),
DockerImage.objects.get(name='ngcp-panel-tests-rest-api-jessie'),
DockerImage.objects.get(name='ngcp-panel-tests-selenium-jessie'),
diobj.get(name='data-hal-jessie'),
diobj.get(name='documentation-jessie'),
diobj.get(name='ngcp-panel-selenium'),
diobj.get(name='ngcp-panel-tests-rest-api-jessie'),
diobj.get(name='ngcp-panel-tests-selenium-jessie'),
]
def test_images_with_tags(self):
self.assertItemsEqual(
DockerImage.objects.images_with_tags(),
diobj.images_with_tags(),
self.images_with_tags)
def test_project_images_with_tags(self):
self.assertItemsEqual(
diobj.images_with_tags('data-hal'),
[diobj.get(name='data-hal-jessie'), ])
self.assertItemsEqual(
diobj.images_with_tags('ngcp-panel'),
[diobj.get(name='ngcp-panel-selenium'),
diobj.get(name='ngcp-panel-tests-rest-api-jessie'),
diobj.get(name='ngcp-panel-tests-selenium-jessie'), ])
self.assertItemsEqual(diobj.images_with_tags('libtcap'), [])
def test_date(self):
tag = DockerTag.objects.get(
name='latest',
image__name='ngcp-panel-tests-selenium-jessie')
self.assertEqual(
tag.date,
datetime.datetime(2016, 11, 07, 20, 30, 25))
datetime.datetime(2017, 6, 21, 16, 3, 37))

@ -0,0 +1,41 @@
# Copyright (C) 2017 The Sipwise Team - http://sipwise.com
# This program is free software: you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option)
# any later version.
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
# You should have received a copy of the GNU General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>.
from django.core.urlresolvers import reverse
from django.conf import settings
from rest_framework import status
from rest_framework.test import APITestCase
from release_dashboard import models
from mock import patch
class TestDockerRest(APITestCase):
fixtures = ['test_model_fixtures', ]
@patch('release_dashboard.utils.docker.delete_docker_info')
def test_deletion(self, ddi):
image_name = 'ngcp-panel-tests-rest-api-jessie'
tag = models.DockerTag.objects.get(
name='latest',
image__name=image_name)
response = self.client.delete(
reverse('dockertag-detail', args=[tag.pk]), format='json')
self.assertEqual(response.status_code, status.HTTP_202_ACCEPTED)
self.assertFalse(
models.DockerTag.objects.filter(
name='latest',
image__name=image_name).exists())
ddi.assert_called_once_with(settings.DOCKER_REGISTRY_URL.format(
'%s/manifests/%s' % (image_name, tag.reference)))

@ -15,8 +15,9 @@
from django.test import TestCase, override_settings
from release_dashboard import tasks
from release_dashboard.models import Project, DockerImage
from release_dashboard.models import Project, DockerImage, DockerTag
from mock import patch, call
import uuid
DOCKER_REST_CATALOG = """
{
@ -53,14 +54,20 @@ def fake_tag(url):
return "{}"
def fake_manifest(url):
return ('{}', uuid.uuid4())
@override_settings(CELERY_EAGER_PROPAGATES_EXCEPTIONS=True)
@override_settings(DOCKER_REGISTRY_URL='{}')
@override_settings(DEBUG=False)
class TasksDockerTestCase(TestCase):
@patch('release_dashboard.utils.docker.get_docker_manifests_info',
side_effect=fake_manifest)
@patch('release_dashboard.utils.docker.get_docker_info',
side_effect=fake_tag)
def test_docker_fetch_info(self, gdi):
def test_docker_fetch_info(self, gdi, gdmi):
proj = Project.objects.create(name="data-hal")
self.assertEquals(proj.name, "data-hal")
image = DockerImage.objects.create(
@ -71,15 +78,41 @@ class TasksDockerTestCase(TestCase):
image = DockerImage.objects.get(name='data-hal-jessie')
calls = [
call("data-hal-jessie/tags/list"),
]
gdi.assert_has_calls(calls)
calls = [
call("data-hal-jessie/manifests/I3a899"),
call("data-hal-jessie/manifests/latest"),
]
gdmi.assert_has_calls(calls)
self.assertItemsEqual(image.tags, ["I3a899", "latest"])
@patch('release_dashboard.utils.docker.get_docker_manifests_info',
side_effect=fake_manifest)
@patch('release_dashboard.utils.docker.get_docker_info',
side_effect=fake_tag)
def test_docker_fetch_project(self, gdi, gdmi):
Project.objects.create(name="data-hal")
result = tasks.docker_fetch_project.delay('data-hal')
self.assertTrue(result.successful())
image = DockerImage.objects.get(name='data-hal-jessie')
calls = [
call("_catalog"),
call("data-hal-jessie/tags/list"),
]
gdi.assert_has_calls(calls)
calls = [
call("data-hal-jessie/manifests/I3a899"),
call("data-hal-jessie/manifests/latest"),
]
gdmi.assert_has_calls(calls)
self.assertItemsEqual(image.tags, ["I3a899", "latest"])
@patch('release_dashboard.utils.docker.get_docker_manifests_info',
side_effect=fake_manifest)
@patch('release_dashboard.utils.docker.get_docker_info',
side_effect=fake_tag)
def test_docker_fetch_all(self, gdi):
def test_docker_fetch_all(self, gdi, gdmi):
result = tasks.docker_fetch_all.delay()
self.assertTrue(result.successful())
proj = Project.objects.get(name="data-hal")
@ -91,10 +124,28 @@ class TasksDockerTestCase(TestCase):
calls = [
call("_catalog"),
call("data-hal-jessie/tags/list"),
call("data-hal-selenium-jessie/tags/list"),
]
gdi.assert_has_calls(calls)
calls = [
call("data-hal-jessie/manifests/I3a899"),
call("data-hal-jessie/manifests/latest"),
call("data-hal-selenium-jessie/tags/list"),
call("data-hal-selenium-jessie/manifests/If53a9"),
call("data-hal-selenium-jessie/manifests/latest"),
]
gdi.assert_has_calls(calls)
gdmi.assert_has_calls(calls)
@patch('release_dashboard.utils.docker.delete_docker_info')
def test_remove_tag(self, ddi):
proj = Project.objects.create(name="data-hal")
image = DockerImage.objects.create(
name='data-hal-jessie', project=proj)
tag = DockerTag.objects.create(
name='latest',
image=image,
reference=uuid.uuid4())
result = tasks.docker_remove_tag.delay('data-hal-jessie', 'latest')
self.assertTrue(result.successful())
ddi.assert_called_once_with('%s/manifests/%s' %
(image.name, tag.reference))
self.assertTrue(image not in image.tags)

@ -39,4 +39,9 @@ urlpatterns = [
name='refresh_docker'),
url(r'^docker/$', docker.docker_images,
name='docker_images'),
url(r'^docker/(?P<project>[^/]+)/$', docker.docker_project_images,
name='docker_project_images'),
url(r'^docker/(?P<project>[^/]+)/(?P<image>[^/]+)$',
docker.docker_image_tags,
name='docker_image_tag'),
]

@ -17,6 +17,7 @@ import logging
import urllib
import requests
import json
import uuid
from django.conf import settings
from repoapi.utils import openurl
@ -47,7 +48,7 @@ def trigger_docker_build(project, branch):
return "{base}/job/build-project-docker/".format(**params)
def get_docker_info(url):
def _get_info(url):
if settings.DEBUG:
logger.debug("Debug mode, would trigger: %s", url)
else:
@ -55,7 +56,28 @@ def get_docker_info(url):
response = requests.get(url)
logger.debug("response: %s" % response)
response.raise_for_status()
return response.text
return response
def get_docker_info(url):
response = _get_info(url)
return response.text
def get_docker_manifests_info(url):
response = _get_info(url)
return (response.text, response.headers['Docker-Content-Digest'])
def delete_docker_info(url):
if settings.DEBUG:
logger.debug("Debug mode, would trigger: %s", url)
else:
logger.debug("trigger: %s", url)
response = requests.delete(url)
logger.debug("response: %s" % response)
response.raise_for_status()
return
def get_docker_repositories():
@ -94,15 +116,21 @@ def get_docker_tags(image):
def get_docker_manifests(image, tag):
if settings.DEBUG:
return '{}'
return ('{}', uuid.uuid4())
else:
dru = settings.DOCKER_REGISTRY_URL
url = dru.format("%s/manifests/%s" % (image, tag))
try:
info = get_docker_info(url)
info, digest = get_docker_manifests_info(url)
logger.debug("response: %s" % info)
result = json.loads(info)
return result
return (result, digest)
except Exception as e:
logger.error('image: %s tag:%s %s' % (image, tag, e))
return None
return (None, None)
def delete_tag(image, reference):
dru = settings.DOCKER_REGISTRY_URL
url = dru.format("%s/manifests/%s" % (image, reference))
delete_docker_info(url)

@ -16,14 +16,17 @@
import logging
import re
from django.shortcuts import render
from django.http import JsonResponse
from django.http import JsonResponse, Http404
from django.views.decorators.http import require_http_methods
from django.conf import settings
from rest_framework import generics, status
from rest_framework.response import Response
from release_dashboard.utils import docker
from release_dashboard.forms.docker import BuildDockerForm
from release_dashboard.forms import docker_projects
from release_dashboard.tasks import docker_fetch_info, docker_fetch_all
from release_dashboard.models import DockerImage
from release_dashboard import tasks
from release_dashboard.models import Project, DockerImage, DockerTag
from release_dashboard import serializers
from . import _projects_versions, _common_versions, _hash_versions
from . import regex_mr
@ -100,7 +103,7 @@ def build_docker_images(request):
def refresh_all(request):
if request.method == "POST":
res = docker_fetch_all.delay()
res = tasks.docker_fetch_all.delay()
return JsonResponse({'url': '/flower/task/%s' % res.id})
else:
projects = []
@ -116,7 +119,7 @@ def refresh_all(request):
@require_http_methods(["POST", ])
def refresh(request, project):
res = docker_fetch_info.delay(project)
res = tasks.docker_fetch_project.delay(project)
return JsonResponse({'url': '/flower/task/%s' % res.id})
@ -129,3 +132,60 @@ def docker_images(request):
}
return render(request, 'release_dashboard/docker_images.html',
context)
@require_http_methods(["GET", ])
def docker_project_images(request, project):
try:
Project.objects.get(name=project)
except Project.DoesNotExist:
raise Http404("Project does not exist")
images = DockerImage.objects.images_with_tags(project)
context = {
'images': images,
'URL_BASE': settings.DOCKER_REGISTRY_URL.format(''),
}
return render(request, 'release_dashboard/docker_images.html',
context)
@require_http_methods(["GET", ])
def docker_image_tags(request, project, image):
try:
proj = Project.objects.get(name=project)
image = DockerImage.objects.get(name=image, project=proj)
except Project.DoesNotExist:
raise Http404("Project does not exist")
except DockerImage.DoesNotExist:
raise Http404("Project does not exist")
context = {
'images': [image, ],
'URL_BASE': settings.DOCKER_REGISTRY_URL.format(''),
}
return render(request, 'release_dashboard/docker_image.html',
context)
class DockerImageList(generics.ListAPIView):
queryset = DockerImage.objects.all()
serializer_class = serializers.DockerImageSerializer
class DockerImageDetail(generics.RetrieveDestroyAPIView):
queryset = DockerImage.objects.all()
serializer_class = serializers.DockerImageSerializer
class DockerTagList(generics.ListAPIView):
queryset = DockerTag.objects.all()
serializer_class = serializers.DockerTagSerializer
class DockerTagDetail(generics.RetrieveDestroyAPIView):
queryset = DockerTag.objects.all()
serializer_class = serializers.DockerTagSerializer
def destroy(self, request, *args, **kwargs):
instance = self.get_object()
tasks.docker_remove_tag.delay(instance.image.name, instance.name)
return Response(status=status.HTTP_202_ACCEPTED)

@ -17,6 +17,7 @@ from django.conf.urls import include, url
from django.contrib import admin
from rest_framework.urlpatterns import format_suffix_patterns
from repoapi import views
from release_dashboard.views import docker
api_patterns = [
url(r'^$', views.api_root, name='index'),
@ -45,6 +46,15 @@ api_patterns = [
'/(?P<project>[^/]+)/(?P<uuid>[^/]+)/$',
views.UUIDInfoList.as_view(),
name='uuidinfo-list'),
url(r'^docker/image/$',
docker.DockerImageList.as_view(),
name='dockerimage-list'),
url(r'^docker/image/(?P<pk>[0-9]+)/$',
docker.DockerImageDetail.as_view(),
name='dockerimage-detail'),
url(r'^docker/tag/(?P<pk>[0-9]+)/$',
docker.DockerTagDetail.as_view(),
name='dockertag-detail'),
]
api_patterns = format_suffix_patterns(api_patterns)

Loading…
Cancel
Save