sipwise
/
repoapi
mirror of https://github.com/sipwise/repoapi.git
1
0
Fork 0
Code Issues Releases Wiki Activity

TT#1906 repoapi: get artifacts for specific jobs

Browse Source 
* set logging from info to debug ( too much noise )
* use shutils to download big files ( just in case )

Change-Id: I35dee767040d64e6b24c743e86b6c2c62020cc0d
changes/52/7252/3
Victor Seva 10 years ago
parent 0c5594ce19
commit f5153bd8ba
7 changed files with 101 additions and 19 deletions
Show Stats Download Patch File Download Diff File

8
repoapi/models/gri.py
Unescape View File

@ -37,7 +37,7 @@ def gerrit_repo_add(instance):
param_ppa=instance.param_ppa,
gerrit_change=instance.gerrit_change)
if created:
logging.info("%s created", ppa)
logging.debug("%s created", ppa)
def gerrit_repo_del(instance):
@ -49,7 +49,7 @@ def gerrit_repo_del(instance):
ppa = gri.get(param_ppa=instance.param_ppa,
gerrit_change=instance.gerrit_change)
ppa.delete()
logger.info("removed %s", ppa)
logger.debug("removed %s", ppa)
except GerritRepoInfo.DoesNotExist:
pass
if gri.filter(param_ppa=instance.param_ppa).count() == 0:
@ -61,7 +61,7 @@ def gerrit_repo_manage(sender, **kwargs):
instance = kwargs["instance"]
if instance.jobname.endswith("-repos") and \
instance.result == "SUCCESS":
logger.info("we need to count this %s", instance.param_ppa)
logger.debug("we need to count this %s", instance.param_ppa)
if instance.gerrit_eventtype == "patchset-created":
gerrit_repo_add(instance)
elif instance.gerrit_eventtype == "change-merged":
@ -69,5 +69,5 @@ def gerrit_repo_manage(sender, **kwargs):
elif instance.jobname.endswith("-cleanup") and \
instance.result == "SUCCESS" and \
instance.gerrit_eventtype == "change-abandoned":
logger.info("we need to count this %s", instance.param_ppa)
logger.debug("we need to count this %s", instance.param_ppa)
gerrit_repo_del(instance)

3
repoapi/models/jbi.py
Unescape View File

@ -16,9 +16,6 @@ import logging
import re
from django.db import models
from django.conf import settings
from repoapi import utils
from repoapi.tasks import get_jbi_files
logger = logging.getLogger(__name__)

3
repoapi/settings/prod.py
Unescape View File

@ -59,3 +59,6 @@ WORKFRONT_CREDENTIALS = os.path.join(BASE_DIR,
# celery
BROKER_URL = 'amqp://guest:guest@localhost'
JBI_BASEDIR = os.path.join(VAR_DIR, 'jbi_files')
JBI_ARTIFACT_JOBS = [
'release-tools-runner',
]

4
repoapi/settings/test.py
Unescape View File

@ -67,4 +67,8 @@ WORKFRONT_CREDENTIALS = os.path.join(BASE_DIR, '.workfront.ini')
# celery
BROKER_BACKEND = 'memory'
CELERY_ALWAYS_EAGER = True
# CELERY_EAGER_PROPAGATES_EXCEPTIONS = True
JBI_BASEDIR = os.path.join(RESULTS_DIR, 'jbi_files')
JBI_ARTIFACT_JOBS = [
'fake-release-tools-runner',
]

24
repoapi/tasks.py
Unescape View File

@ -14,11 +14,29 @@
# with this program. If not, see <http://www.gnu.org/licenses/>.
from __future__ import absolute_import
import json
import logging
from celery import shared_task
from .utils import jenkins_get_console, jenkins_get_job
from django.conf import settings
from .utils import jenkins_get_console, jenkins_get_job, jenkins_get_artifact
logger = logging.getLogger(__name__)
@shared_task
@shared_task(ignore_result=True)
def jbi_get_artifact(jobname, buildnumber, artifact_info):
jenkins_get_artifact(jobname, buildnumber, artifact_info)
@shared_task(ignore_result=True)
def get_jbi_files(jobname, buildnumber):
jenkins_get_console(jobname, buildnumber)
jenkins_get_job(jobname, buildnumber)
path = jenkins_get_job(jobname, buildnumber)
if jobname in settings.JBI_ARTIFACT_JOBS:
with open(path) as data_file:
data = json.load(data_file)
logger.debug("job_info:%s", data)
for artifact in data['artifacts']:
jbi_get_artifact.delay(jobname, buildnumber, artifact)
else:
logger.debug("skip artifacts download for jobname: %s", jobname)

53
repoapi/test/test_jbi_info.py
Unescape View File

@ -19,8 +19,20 @@ import shutil
from django.test import TestCase
from django.conf import settings
from repoapi.models import JenkinsBuildInfo
from repoapi.utils import JBI_CONSOLE_URL, JBI_JOB_URL
from mock import patch, call
from repoapi.utils import JBI_CONSOLE_URL, JBI_JOB_URL, JBI_ARTIFACT_URL
from mock import patch, call, mock_open
artifacts_json = """{
"artifacts": [
{
"displayPath": "builddeps.list",
"fileName": "builddeps.list",
"relativePath": "builddeps.list"
}
]
}"""
class TestJBICelery(TestCase):
@ -49,13 +61,15 @@ class TestJBICelery(TestCase):
if os.path.exists(settings.JBI_BASEDIR):
shutil.rmtree(settings.JBI_BASEDIR)
@patch('__builtin__.open', mock_open(read_data=artifacts_json))
@patch('repoapi.utils.dlfile')
def test_jbi_path_creation(self, dlfile):
param = self.get_defaults()
jbi = JenkinsBuildInfo.objects.create(**param)
base_path = os.path.join(settings.JBI_BASEDIR,
jbi.jobname, str(jbi.buildnumber))
self.assertTrue(os.path.exists(settings.JBI_BASEDIR), settings.JBI_BASEDIR)
self.assertTrue(
os.path.exists(settings.JBI_BASEDIR), settings.JBI_BASEDIR)
self.assertTrue(os.path.exists(base_path))
path = os.path.join(base_path, 'console.txt')
url = JBI_CONSOLE_URL.format(
@ -63,7 +77,7 @@ class TestJBICelery(TestCase):
jbi.jobname,
jbi.buildnumber
)
calls = [call(url, path),]
calls = [call(url, path), ]
url = JBI_JOB_URL.format(
settings.JENKINS_URL,
jbi.jobname,
@ -72,3 +86,34 @@ class TestJBICelery(TestCase):
path = os.path.join(base_path, 'job.json')
calls.append(call(url, path))
dlfile.assert_has_calls(calls)
url = JBI_ARTIFACT_URL.format(
settings.JENKINS_URL,
jbi.jobname,
jbi.buildnumber,
"builddeps.list"
)
artifact_base_path = os.path.join(base_path, 'artifact')
path = os.path.join(artifact_base_path, 'builddeps.list')
self.assertNotIn(call(url, path), dlfile.call_args_list)
@patch('__builtin__.open', mock_open(read_data=artifacts_json))
@patch('repoapi.utils.dlfile')
def test_jbi_artifact(self, dlfile):
param = self.get_defaults()
param['jobname'] = 'fake-release-tools-runner'
jbi = JenkinsBuildInfo.objects.create(**param)
base_path = os.path.join(settings.JBI_BASEDIR,
jbi.jobname, str(jbi.buildnumber))
self.assertTrue(
os.path.exists(settings.JBI_BASEDIR), settings.JBI_BASEDIR)
self.assertTrue(os.path.exists(base_path))
url = JBI_ARTIFACT_URL.format(
settings.JENKINS_URL,
jbi.jobname,
jbi.buildnumber,
"builddeps.list"
)
artifact_base_path = os.path.join(base_path, 'artifact')
path = os.path.join(artifact_base_path, 'builddeps.list')
calls = [call(url, path), ]
dlfile.assert_has_calls(calls)

25
repoapi/utils.py
Unescape View File

@ -17,6 +17,7 @@ from __future__ import absolute_import
from distutils.dir_util import mkpath
import logging
import os
import shutil
import subprocess
import urllib2
from django.conf import settings
@ -25,6 +26,7 @@ logger = logging.getLogger(__name__)
JBI_CONSOLE_URL = "{}/job/{}/{}/consoleText"
JBI_JOB_URL = "{}/job/{}/{}/api/json"
JBI_ARTIFACT_URL = "{}/job/{}/{}/artifact/{}"
def executeAndReturnOutput(command, env=None):
@ -43,7 +45,7 @@ def dlfile(url, path):
remote_file = urllib2.urlopen(url)
logger.debug("url:[%s]", url)
with open(path, "wb") as local_file:
local_file.write(remote_file.read())
shutil.copyfileobj(remote_file, local_file)
def openurl(url):
@ -63,7 +65,7 @@ def jenkins_remove_ppa(repo):
"token=%s&repository=%s" % \
(settings.JENKINS_URL, settings.JENKINS_TOKEN, repo)
if settings.DEBUG:
logger.info("I would call %s", url)
logger.debug("I would call %s", url)
else:
openurl(url)
@ -71,8 +73,9 @@ def jenkins_remove_ppa(repo):
def _jenkins_get(url, base_path, filename):
mkpath(base_path)
path = os.path.join(base_path, filename)
logger.info("url:[%s] path[%s]", url, path)
logger.debug("url:[%s] path[%s]", url, path)
dlfile(url, path)
return path
def jenkins_get_console(jobname, buildnumber):
@ -83,7 +86,7 @@ def jenkins_get_console(jobname, buildnumber):
)
base_path = os.path.join(settings.JBI_BASEDIR,
jobname, str(buildnumber))
_jenkins_get(url, base_path, 'console.txt')
return _jenkins_get(url, base_path, 'console.txt')
def jenkins_get_job(jobname, buildnumber):
@ -94,7 +97,19 @@ def jenkins_get_job(jobname, buildnumber):
)
base_path = os.path.join(settings.JBI_BASEDIR,
jobname, str(buildnumber))
_jenkins_get(url, base_path, 'job.json')
return _jenkins_get(url, base_path, 'job.json')
def jenkins_get_artifact(jobname, buildnumber, artifact_info):
url = JBI_ARTIFACT_URL.format(
settings.JENKINS_URL,
jobname,
buildnumber,
artifact_info["relativePath"]
)
base_path = os.path.join(settings.JBI_BASEDIR,
jobname, str(buildnumber), 'artifact')
return _jenkins_get(url, base_path, artifact_info["fileName"])
def workfront_note_send(_id, message):

Write Preview
Loading…
Cancel
Save