From 34221aee0a116d5488cdd3663f7b423e5ee2c60f Mon Sep 17 00:00:00 2001 From: Victor Seva Date: Thu, 15 Oct 2020 17:40:33 +0200 Subject: [PATCH] TT#7211 add authentication on jenkins requests * rename repoapi.utils.openurl to a less confusing name open_jenkins_url Change-Id: I0d3fee994d0fb4af5a315b4c1d1a42dcf067504a --- build/test/test_utils.py | 2 +- build/utils.py | 6 ++--- release_dashboard/utils/build.py | 6 ++--- release_dashboard/utils/docker.py | 4 +-- repoapi/utils.py | 44 ++++++++++++++++++++----------- 5 files changed, 38 insertions(+), 24 deletions(-) diff --git a/build/test/test_utils.py b/build/test/test_utils.py index fbd74d1..6964a7c 100644 --- a/build/test/test_utils.py +++ b/build/test/test_utils.py @@ -174,7 +174,7 @@ class ReleaseConfigTestCase(SimpleTestCase): self.assertListEqual(["ngcp-schema"], values) -@patch("build.utils.openurl") +@patch("build.utils.open_jenkins_url") class TriggerBuild(SimpleTestCase): def test_project_build(self, openurl): params = { diff --git a/build/utils.py b/build/utils.py index 1bd2378..91c2cbd 100644 --- a/build/utils.py +++ b/build/utils.py @@ -24,7 +24,7 @@ from yaml import Loader from . import exceptions as err from .conf import settings -from repoapi.utils import openurl +from repoapi.utils import open_jenkins_url logger = logging.getLogger(__name__) @@ -79,7 +79,7 @@ def trigger_copy_deps(release, internal, release_uuid, uuid=None): if settings.DEBUG: logger.info("Debug mode, would trigger: %s", url) else: - openurl(url) + open_jenkins_url(url) return "{base}/job/{job}/".format(**params) @@ -123,7 +123,7 @@ def trigger_build( if settings.DEBUG: logger.info("Debug mode, would trigger: %s", url) else: - openurl(url) + open_jenkins_url(url) return "{base}/job/{job}/".format(**params) diff --git a/release_dashboard/utils/build.py b/release_dashboard/utils/build.py index 8828ded..a733e54 100644 --- a/release_dashboard/utils/build.py +++ b/release_dashboard/utils/build.py @@ -18,7 +18,7 @@ from requests.auth import HTTPBasicAuth from ..conf import settings from ..models import Project -from repoapi.utils import openurl +from repoapi.utils import open_jenkins_url logger = logging.getLogger(__name__) @@ -62,7 +62,7 @@ def trigger_hotfix(project, branch, user, push="yes"): if settings.DEBUG: logger.warn("Debug mode, would trigger: %s", url) else: - openurl(url) + open_jenkins_url(url) return "%s/job/release-tools-runner/" % settings.JENKINS_URL @@ -109,7 +109,7 @@ def trigger_build( if settings.DEBUG: logger.debug("Debug mode, would trigger: %s", url) else: - openurl(url) + open_jenkins_url(url) return "{base}/job/{job}/".format(**params) diff --git a/release_dashboard/utils/docker.py b/release_dashboard/utils/docker.py index e434abb..f02e209 100644 --- a/release_dashboard/utils/docker.py +++ b/release_dashboard/utils/docker.py @@ -20,7 +20,7 @@ import uuid import requests from ..conf import settings -from repoapi.utils import openurl +from repoapi.utils import open_jenkins_url logger = logging.getLogger(__name__) @@ -50,7 +50,7 @@ def trigger_docker_build(project, branch): if settings.DEBUG: logger.debug("Debug mode, would trigger: %s", url) else: - openurl(url) + open_jenkins_url(url) return "{base}/job/build-project-docker/".format(**params) diff --git a/repoapi/utils.py b/repoapi/utils.py index 3385251..ce1a2e9 100644 --- a/repoapi/utils.py +++ b/repoapi/utils.py @@ -17,9 +17,11 @@ import os import re import shutil import subprocess -import urllib.request from distutils.dir_util import mkpath +import requests +from requests.auth import HTTPBasicAuth + from .conf import settings logger = logging.getLogger(__name__) @@ -40,28 +42,40 @@ def executeAndReturnOutput(command, env=None): return proc.returncode, stdoutdata, stderrdata +def get_jenkins_response(url): + auth = HTTPBasicAuth( + settings.JENKINS_HTTP_USER, settings.JENKINS_HTTP_PASSWD + ) + response = requests.get(url, auth=auth) + response.raise_for_status() + return response + + def dlfile(url, path): if settings.DEBUG: logger.info("I would call %s", url) else: - remote_file = urllib.request.urlopen(url) + auth = HTTPBasicAuth( + settings.JENKINS_HTTP_USER, settings.JENKINS_HTTP_PASSWD + ) logger.debug("url:[%s]", url) - with open(path, "wb") as local_file: - shutil.copyfileobj(remote_file, local_file) + with requests.get(url, auth=auth, stream=True) as req: + with open(path, "wb") as local_file: + shutil.copyfileobj(req.raw, local_file) -def openurl(url): - req = urllib.request.Request(url) +def open_jenkins_url(url): logger.debug("Trying to retrieve url: [%s]", url) try: - response = urllib.request.urlopen(req) - if 199 < response.getcode() < 300: - logger.debug("OK[%d] URL[%s]", response.getcode(), url) - return True - except urllib.error.HTTPError as e: - logger.error("Error[%d] retrieving URL[%s]", e.getcode(), url) - except Exception: - logger.error("Fatal error retrieving URL[%s]", url) + res = get_jenkins_response(url) + logger.debug("OK[{}] URL[{}]".format(res.status_code, url)) + return True + except requests.HTTPError as e: + logger.error( + "Error[{}] retrieving URL[{}]: {}".format(res.status_code, url, e) + ) + except Exception as e: + logger.error("Fatal error retrieving URL[{}]: {}".format(url, e)) return False @@ -75,7 +89,7 @@ def jenkins_remove_ppa(repo): if settings.DEBUG: logger.debug("I would call %s", url) else: - openurl(url) + open_jenkins_url(url) def _jenkins_get(url, base_path, filename):