From 4e76d15eda8dddbc6ab58032ee6b1b619df8dfe0 Mon Sep 17 00:00:00 2001 From: "Sphicas, Phil (ps3910)" Date: Tue, 17 Sep 2019 01:24:02 -0700 Subject: [PATCH] Allow proxy_server use for chart tarball downloads Currently the chart `source` schema allows for a proxy server to be specified, but it is only used for git repos. This patchset allows the `proxy_server` to also be used for tarball url sources. Change-Id: I6f90d056fa46f596b1fb248b6c596c58b6513d64 --- armada/handlers/armada.py | 11 +++++--- armada/utils/source.py | 17 +++++++++---- .../documents/v1/document-authoring.rst | 25 +++++++++++-------- .../documents/v2/document-authoring.rst | 25 +++++++++++-------- 4 files changed, 47 insertions(+), 31 deletions(-) diff --git a/armada/handlers/armada.py b/armada/handlers/armada.py index 471b839f..68af09ab 100644 --- a/armada/handlers/armada.py +++ b/armada/handlers/armada.py @@ -122,6 +122,7 @@ class Armada(object): location = chart_source.get('location') ct_type = chart_source.get('type') subpath = chart_source.get('subpath', '.') + proxy_server = chart_source.get('proxy_server') if ct_type == 'local': chart['source_dir'] = (location, subpath) @@ -129,15 +130,18 @@ class Armada(object): source_key = (ct_type, location) if source_key not in self.chart_cache: - LOG.info('Downloading tarball from: %s', location) + LOG.info( + "Downloading tarball from: %s / proxy %s", location, + proxy_server or "not set") if not CONF.certs: LOG.warn( 'Disabling server validation certs to extract charts') - tarball_dir = source.get_tarball(location, verify=False) + tarball_dir = source.get_tarball( + location, verify=False, proxy_server=proxy_server) else: tarball_dir = source.get_tarball( - location, verify=CONF.certs) + location, verify=CONF.certs, proxy_server=proxy_server) self.chart_cache[source_key] = tarball_dir chart['source_dir'] = (self.chart_cache.get(source_key), subpath) elif ct_type == 'git': @@ -146,7 +150,6 @@ class Armada(object): if source_key not in self.chart_cache: auth_method = chart_source.get('auth_method') - proxy_server = chart_source.get('proxy_server') logstr = 'Cloning repo: {} from branch: {}'.format( location, reference) diff --git a/armada/utils/source.py b/armada/utils/source.py index 31120fe3..ed42215b 100644 --- a/armada/utils/source.py +++ b/armada/utils/source.py @@ -113,21 +113,28 @@ def git_clone(repo_url, ref='master', proxy_server=None, auth_method=None): return temp_dir -def get_tarball(tarball_url, verify=False): - tarball_path = download_tarball(tarball_url, verify=verify) +def get_tarball(tarball_url, verify=False, proxy_server=None): + tarball_path = download_tarball( + tarball_url, verify=verify, proxy_server=proxy_server) return extract_tarball(tarball_path) -def download_tarball(tarball_url, verify=False): +def download_tarball(tarball_url, verify=False, proxy_server=None): ''' Downloads a tarball to /tmp and returns the path ''' try: if not verify: urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - + kwargs = {} + if proxy_server: + kwargs['proxies'] = { + 'http': proxy_server, + 'https': proxy_server, + 'ftp': proxy_server + } tarball_filename = tempfile.mkstemp(prefix='armada')[1] - response = requests.get(tarball_url, verify=verify) + response = requests.get(tarball_url, verify=verify, **kwargs) with open(tarball_filename, 'wb') as f: f.write(response.content) diff --git a/doc/source/operations/documents/v1/document-authoring.rst b/doc/source/operations/documents/v1/document-authoring.rst index c441510c..94a2f9ef 100644 --- a/doc/source/operations/documents/v1/document-authoring.rst +++ b/doc/source/operations/documents/v1/document-authoring.rst @@ -333,17 +333,19 @@ Delete Source ^^^^^^ -+-------------+----------+-----------------------------------------------------------------------------------+ -| keyword | type | action | -+=============+==========+===================================================================================+ -| type | string | source to build the chart: ``git``, ``local``, or ``tar`` | -+-------------+----------+-----------------------------------------------------------------------------------+ -| location | string | ``url`` or ``path`` to the chart's parent directory | -+-------------+----------+-----------------------------------------------------------------------------------+ -| subpath | string | (optional) relative path to target chart from parent (``.`` if not specified) | -+-------------+----------+-----------------------------------------------------------------------------------+ -| reference | string | (optional) branch, commit, or reference in the repo (``master`` if not specified) | -+-------------+----------+-----------------------------------------------------------------------------------+ ++-----------------+----------+-----------------------------------------------------------------------------------+ +| keyword | type | action | ++=================+==========+===================================================================================+ +| type | string | source to build the chart: ``git``, ``local``, or ``tar`` | ++-----------------+----------+-----------------------------------------------------------------------------------+ +| location | string | ``url`` or ``path`` to the chart's parent directory | ++-----------------+----------+-----------------------------------------------------------------------------------+ +| subpath | string | (optional) relative path to target chart from parent (``.`` if not specified) | ++-----------------+----------+-----------------------------------------------------------------------------------+ +| reference | string | (optional) branch, commit, or reference in the repo (``master`` if not specified) | ++-----------------+----------+-----------------------------------------------------------------------------------+ +| proxy\_server | string | (optional) proxy server URL for downloading ``git`` or ``tar`` charts | ++-----------------+----------+-----------------------------------------------------------------------------------+ Source Example ^^^^^^^^^^^^^^ @@ -420,6 +422,7 @@ Source Example location: https://localhost:8879/charts/chart-0.1.0.tgz subpath: mariadb reference: null + proxy_server: http://my.proxy.server:8888 diff --git a/doc/source/operations/documents/v2/document-authoring.rst b/doc/source/operations/documents/v2/document-authoring.rst index 841d265c..52b19b40 100644 --- a/doc/source/operations/documents/v2/document-authoring.rst +++ b/doc/source/operations/documents/v2/document-authoring.rst @@ -397,17 +397,19 @@ Delete Source ^^^^^^ -+-------------+----------+-----------------------------------------------------------------------------------+ -| keyword | type | action | -+=============+==========+===================================================================================+ -| type | string | source to build the chart: ``git``, ``local``, or ``tar`` | -+-------------+----------+-----------------------------------------------------------------------------------+ -| location | string | ``url`` or ``path`` to the chart's parent directory | -+-------------+----------+-----------------------------------------------------------------------------------+ -| subpath | string | (optional) relative path to target chart from parent (``.`` if not specified) | -+-------------+----------+-----------------------------------------------------------------------------------+ -| reference | string | (optional) branch, commit, or reference in the repo (``master`` if not specified) | -+-------------+----------+-----------------------------------------------------------------------------------+ ++-----------------+----------+-----------------------------------------------------------------------------------+ +| keyword | type | action | ++=================+==========+===================================================================================+ +| type | string | source to build the chart: ``git``, ``local``, or ``tar`` | ++-----------------+----------+-----------------------------------------------------------------------------------+ +| location | string | ``url`` or ``path`` to the chart's parent directory | ++-----------------+----------+-----------------------------------------------------------------------------------+ +| subpath | string | (optional) relative path to target chart from parent (``.`` if not specified) | ++-----------------+----------+-----------------------------------------------------------------------------------+ +| reference | string | (optional) branch, commit, or reference in the repo (``master`` if not specified) | ++-----------------+----------+-----------------------------------------------------------------------------------+ +| proxy\_server | string | (optional) proxy server URL for downloading ``git`` or ``tar`` charts | ++-----------------+----------+-----------------------------------------------------------------------------------+ Source Example ^^^^^^^^^^^^^^ @@ -430,6 +432,7 @@ Source Example source: type: git location: https://github.com/namespace/repo + proxy_server: http://my.proxy.server:8888 # type local ---