From 9cb2c5a5caf7d0d842b8894f7428f34774748740 Mon Sep 17 00:00:00 2001 From: Mark Burnett Date: Mon, 25 Jun 2018 06:24:01 -0500 Subject: [PATCH] Add local file cache for Builder tarball fetch Change-Id: I4741b50c4a4b505f4f70ff2cbc5e9689b9f28b7b --- .../promenade/templates/deployment-api.yaml | 4 ++ promenade/builder.py | 38 +++++++++++++++---- promenade/templates/include/utils.sh | 6 +-- requirements-direct.txt | 1 + requirements-frozen.txt | 34 +++++++++-------- tools/g2/manifests/resiliency.json | 2 +- 6 files changed, 57 insertions(+), 28 deletions(-) diff --git a/charts/promenade/templates/deployment-api.yaml b/charts/promenade/templates/deployment-api.yaml index e218dadf..0415fed1 100644 --- a/charts/promenade/templates/deployment-api.yaml +++ b/charts/promenade/templates/deployment-api.yaml @@ -82,9 +82,13 @@ spec: mountPath: /etc/promenade/promenade.conf subPath: promenade.conf readOnly: true + - name: cache + mountPath: /tmp/cache volumes: - name: promenade-etc configMap: name: promenade-etc defaultMode: 0444 + - name: cache + emptyDir: {} {{- end }} diff --git a/promenade/builder.py b/promenade/builder.py index 394e14f7..b65704a2 100644 --- a/promenade/builder.py +++ b/promenade/builder.py @@ -1,4 +1,7 @@ from . import logging, renderer +from beaker.cache import CacheManager +from beaker.util import parse_cache_config_options + import io import itertools import os @@ -10,6 +13,18 @@ __all__ = ['Builder'] LOG = logging.getLogger(__name__) +# Ignore bandit false positive: +# B108:hardcoded_tmp_directory +# This cache needs to be shared by all forks within the same container, and so +# must be at a well-known location. +CACHE_OPTS = { + 'cache.type': 'file', + 'cache.data_dir': '/tmp/cache/data', # nosec + 'cache.lock_dir': '/tmp/cache/lock', # nosec +} + +CACHE = CacheManager(**parse_cache_config_options(CACHE_OPTS)) + class Builder: def __init__(self, config, *, validators=False): @@ -30,8 +45,8 @@ class Builder: if 'content' in file_spec: data = file_spec['content'] elif 'tar_url' in file_spec: - data = _fetch_tar_content( - url=file_spec['tar_url'], path=file_spec['tar_path']) + data = _fetch_tar_content(file_spec['tar_url'], + file_spec['tar_path']) self._file_cache[path] = { 'path': path, 'data': data, @@ -111,17 +126,24 @@ class Builder: sub_config, template='scripts/validate-join.sh') -def _fetch_tar_content(*, url, path): - LOG.debug('Fetching url=%s (tar path=%s)', url, path) - response = requests.get(url) - response.raise_for_status() - LOG.debug('Finished downloading url=%s (tar path=%s)', url, path) - f = io.BytesIO(response.content) +@CACHE.cache('fetch_tarball_content', expire=72 * 3600) +def _fetch_tar_content(url, path): + content = _fetch_tar_url(url) + f = io.BytesIO(content) tf = tarfile.open(fileobj=f, mode='r') buf_reader = tf.extractfile(path) return buf_reader.read() +@CACHE.cache('fetch_tarball_url', expire=72 * 3600) +def _fetch_tar_url(url): + LOG.debug('Fetching url=%s', url) + response = requests.get(url) + response.raise_for_status() + LOG.debug('Finished downloading url=%s', url) + return response.content + + def _join_name(node_name): return 'join-%s.sh' % node_name diff --git a/promenade/templates/include/utils.sh b/promenade/templates/include/utils.sh index 048a2798..76d4fb3b 100644 --- a/promenade/templates/include/utils.sh +++ b/promenade/templates/include/utils.sh @@ -195,19 +195,19 @@ function wait_for_pod_termination { end=$(($(date +%s) + $SEC)) while true; do - POD_PHASE=$(kubectl --request-timeout 10s --namespace $NAMESPACE get -o jsonpath="${POD_PHASE_JSONPATH}" pod $POD_NAME) + POD_PHASE=$(kubectl --request-timeout 120s --namespace $NAMESPACE get -o jsonpath="${POD_PHASE_JSONPATH}" pod $POD_NAME) if [ "x$POD_PHASE" = "xSucceeded" ]; then log Pod $POD_NAME succeeded. break elif [ "x$POD_PHASE" = "xFailed" ]; then log Pod $POD_NAME failed. - kubectl --request-timeout 10s --namespace $NAMESPACE get -o yaml pod $POD_NAME 1>&2 + kubectl --request-timeout 120s --namespace $NAMESPACE get -o yaml pod $POD_NAME 1>&2 fail else now=$(date +%s) if [ $now -gt $end ]; then log Pod did not terminate before timeout. - kubectl --request-timeout 10s --namespace $NAMESPACE get -o yaml pod $POD_NAME 1>&2 + kubectl --request-timeout 120s --namespace $NAMESPACE get -o yaml pod $POD_NAME 1>&2 fail fi sleep 1 diff --git a/requirements-direct.txt b/requirements-direct.txt index a4cc9e6d..2b6b8067 100644 --- a/requirements-direct.txt +++ b/requirements-direct.txt @@ -1,3 +1,4 @@ +beaker==1.9.1 click==6.7 falcon==1.2.0 jinja2==2.9.6 diff --git a/requirements-frozen.txt b/requirements-frozen.txt index 0ab93999..df62d13c 100644 --- a/requirements-frozen.txt +++ b/requirements-frozen.txt @@ -1,12 +1,14 @@ alembic==0.8.2 -amqp==2.2.2 -Babel==2.5.3 +amqp==2.3.2 +Babel==2.6.0 +Beaker==1.9.1 cachetools==2.1.0 certifi==2018.4.16 chardet==3.0.4 click==6.7 -cliff==2.11.0 -cmd2==0.8.5 +cliff==2.12.1 +cmd2==0.9.1 +colorama==0.3.9 contextlib2==0.5.5 debtcollector==1.19.0 git+https://git.openstack.org/openstack/airship-deckhand@177675e96fffcda9799c68bbce831424c1167020#egg=deckhand @@ -19,7 +21,7 @@ fasteners==0.14.1 fixtures==3.0.0 flake8==2.6.2 futurist==1.7.0 -google-auth==1.4.1 +google-auth==1.5.0 greenlet==0.4.13 hacking==1.1.0 idna==2.6 @@ -30,7 +32,7 @@ jsonpath-ng==1.4.3 jsonschema==2.6.0 keystoneauth1==3.2.0 keystonemiddleware==4.17.0 -kombu==4.1.0 +kombu==4.2.1 kubernetes==3.0.0 linecache2==1.0.0 Mako==1.0.7 @@ -45,15 +47,15 @@ oslo.cache==1.30.1 oslo.concurrency==3.27.0 oslo.config==6.2.1 oslo.context==2.19.2 -oslo.db==4.38.0 +oslo.db==4.39.0 oslo.i18n==3.20.0 oslo.log==3.38.1 -oslo.messaging==6.2.0 +oslo.messaging==6.5.0 oslo.middleware==3.35.0 oslo.policy==1.22.1 -oslo.serialization==2.25.0 +oslo.serialization==2.27.0 oslo.service==1.31.2 -oslo.utils==3.36.2 +oslo.utils==3.36.3 Paste==2.0.3 PasteDeploy==1.5.2 pbr==3.0.1 @@ -61,18 +63,18 @@ ply==3.11 positional==1.2.1 prettytable==0.7.2 psycopg2==2.7.4 -pyasn1==0.4.2 +pyasn1==0.4.3 pyasn1-modules==0.2.1 pycadf==2.7.0 pycodestyle==2.0.0 pyflakes==1.2.3 pyinotify==0.9.6 pyparsing==2.2.0 -pyperclip==1.6.0 +pyperclip==1.6.2 python-barbicanclient==4.6.0 python-dateutil==2.7.3 python-editor==1.0.3 -python-keystoneclient==3.16.0 +python-keystoneclient==3.17.0 python-memcached==1.59 python-mimeparse==1.6.0 pytz==2018.4 @@ -83,7 +85,7 @@ rfc3986==1.1.0 Routes==2.4.1 rsa==3.4.2 six==1.11.0 -SQLAlchemy==1.2.7 +SQLAlchemy==1.2.8 sqlalchemy-migrate==0.11.0 sqlparse==0.2.4 statsd==3.2.2 @@ -95,11 +97,11 @@ testscenarios==0.5.0 testtools==2.3.0 traceback2==1.4.0 unittest2==1.1.0 -urllib3==1.22 +urllib3==1.23 uWSGI==2.0.15 vine==1.1.4 wcwidth==0.1.7 -WebOb==1.8.1 +WebOb==1.8.2 websocket-client==0.40.0 Werkzeug==0.14.1 wrapt==1.10.11 diff --git a/tools/g2/manifests/resiliency.json b/tools/g2/manifests/resiliency.json index 6f12fd16..151f0bb4 100644 --- a/tools/g2/manifests/resiliency.json +++ b/tools/g2/manifests/resiliency.json @@ -109,7 +109,7 @@ "name": "Check full etcd cluster", "script": "check-etcd-health.sh", "arguments": [ - "-w", "30", + "-w", "60", "-e", "kubernetes n0 n0 n1 n2 n3", "-e", "calico n0 n0 n1 n2 n3" ]