Add local file cache for Builder tarball fetch

Change-Id: I4741b50c4a4b505f4f70ff2cbc5e9689b9f28b7b
This commit is contained in:
Mark Burnett 2018-06-25 06:24:01 -05:00
parent 57093d29ce
commit 9cb2c5a5ca
6 changed files with 57 additions and 28 deletions

View File

@ -82,9 +82,13 @@ spec:
mountPath: /etc/promenade/promenade.conf
subPath: promenade.conf
readOnly: true
- name: cache
mountPath: /tmp/cache
volumes:
- name: promenade-etc
configMap:
name: promenade-etc
defaultMode: 0444
- name: cache
emptyDir: {}
{{- end }}

View File

@ -1,4 +1,7 @@
from . import logging, renderer
from beaker.cache import CacheManager
from beaker.util import parse_cache_config_options
import io
import itertools
import os
@ -10,6 +13,18 @@ __all__ = ['Builder']
LOG = logging.getLogger(__name__)
# Ignore bandit false positive:
# B108:hardcoded_tmp_directory
# This cache needs to be shared by all forks within the same container, and so
# must be at a well-known location.
CACHE_OPTS = {
'cache.type': 'file',
'cache.data_dir': '/tmp/cache/data', # nosec
'cache.lock_dir': '/tmp/cache/lock', # nosec
}
CACHE = CacheManager(**parse_cache_config_options(CACHE_OPTS))
class Builder:
def __init__(self, config, *, validators=False):
@ -30,8 +45,8 @@ class Builder:
if 'content' in file_spec:
data = file_spec['content']
elif 'tar_url' in file_spec:
data = _fetch_tar_content(
url=file_spec['tar_url'], path=file_spec['tar_path'])
data = _fetch_tar_content(file_spec['tar_url'],
file_spec['tar_path'])
self._file_cache[path] = {
'path': path,
'data': data,
@ -111,17 +126,24 @@ class Builder:
sub_config, template='scripts/validate-join.sh')
def _fetch_tar_content(*, url, path):
LOG.debug('Fetching url=%s (tar path=%s)', url, path)
response = requests.get(url)
response.raise_for_status()
LOG.debug('Finished downloading url=%s (tar path=%s)', url, path)
f = io.BytesIO(response.content)
@CACHE.cache('fetch_tarball_content', expire=72 * 3600)
def _fetch_tar_content(url, path):
content = _fetch_tar_url(url)
f = io.BytesIO(content)
tf = tarfile.open(fileobj=f, mode='r')
buf_reader = tf.extractfile(path)
return buf_reader.read()
@CACHE.cache('fetch_tarball_url', expire=72 * 3600)
def _fetch_tar_url(url):
LOG.debug('Fetching url=%s', url)
response = requests.get(url)
response.raise_for_status()
LOG.debug('Finished downloading url=%s', url)
return response.content
def _join_name(node_name):
return 'join-%s.sh' % node_name

View File

@ -195,19 +195,19 @@ function wait_for_pod_termination {
end=$(($(date +%s) + $SEC))
while true; do
POD_PHASE=$(kubectl --request-timeout 10s --namespace $NAMESPACE get -o jsonpath="${POD_PHASE_JSONPATH}" pod $POD_NAME)
POD_PHASE=$(kubectl --request-timeout 120s --namespace $NAMESPACE get -o jsonpath="${POD_PHASE_JSONPATH}" pod $POD_NAME)
if [ "x$POD_PHASE" = "xSucceeded" ]; then
log Pod $POD_NAME succeeded.
break
elif [ "x$POD_PHASE" = "xFailed" ]; then
log Pod $POD_NAME failed.
kubectl --request-timeout 10s --namespace $NAMESPACE get -o yaml pod $POD_NAME 1>&2
kubectl --request-timeout 120s --namespace $NAMESPACE get -o yaml pod $POD_NAME 1>&2
fail
else
now=$(date +%s)
if [ $now -gt $end ]; then
log Pod did not terminate before timeout.
kubectl --request-timeout 10s --namespace $NAMESPACE get -o yaml pod $POD_NAME 1>&2
kubectl --request-timeout 120s --namespace $NAMESPACE get -o yaml pod $POD_NAME 1>&2
fail
fi
sleep 1

View File

@ -1,3 +1,4 @@
beaker==1.9.1
click==6.7
falcon==1.2.0
jinja2==2.9.6

View File

@ -1,12 +1,14 @@
alembic==0.8.2
amqp==2.2.2
Babel==2.5.3
amqp==2.3.2
Babel==2.6.0
Beaker==1.9.1
cachetools==2.1.0
certifi==2018.4.16
chardet==3.0.4
click==6.7
cliff==2.11.0
cmd2==0.8.5
cliff==2.12.1
cmd2==0.9.1
colorama==0.3.9
contextlib2==0.5.5
debtcollector==1.19.0
git+https://git.openstack.org/openstack/airship-deckhand@177675e96fffcda9799c68bbce831424c1167020#egg=deckhand
@ -19,7 +21,7 @@ fasteners==0.14.1
fixtures==3.0.0
flake8==2.6.2
futurist==1.7.0
google-auth==1.4.1
google-auth==1.5.0
greenlet==0.4.13
hacking==1.1.0
idna==2.6
@ -30,7 +32,7 @@ jsonpath-ng==1.4.3
jsonschema==2.6.0
keystoneauth1==3.2.0
keystonemiddleware==4.17.0
kombu==4.1.0
kombu==4.2.1
kubernetes==3.0.0
linecache2==1.0.0
Mako==1.0.7
@ -45,15 +47,15 @@ oslo.cache==1.30.1
oslo.concurrency==3.27.0
oslo.config==6.2.1
oslo.context==2.19.2
oslo.db==4.38.0
oslo.db==4.39.0
oslo.i18n==3.20.0
oslo.log==3.38.1
oslo.messaging==6.2.0
oslo.messaging==6.5.0
oslo.middleware==3.35.0
oslo.policy==1.22.1
oslo.serialization==2.25.0
oslo.serialization==2.27.0
oslo.service==1.31.2
oslo.utils==3.36.2
oslo.utils==3.36.3
Paste==2.0.3
PasteDeploy==1.5.2
pbr==3.0.1
@ -61,18 +63,18 @@ ply==3.11
positional==1.2.1
prettytable==0.7.2
psycopg2==2.7.4
pyasn1==0.4.2
pyasn1==0.4.3
pyasn1-modules==0.2.1
pycadf==2.7.0
pycodestyle==2.0.0
pyflakes==1.2.3
pyinotify==0.9.6
pyparsing==2.2.0
pyperclip==1.6.0
pyperclip==1.6.2
python-barbicanclient==4.6.0
python-dateutil==2.7.3
python-editor==1.0.3
python-keystoneclient==3.16.0
python-keystoneclient==3.17.0
python-memcached==1.59
python-mimeparse==1.6.0
pytz==2018.4
@ -83,7 +85,7 @@ rfc3986==1.1.0
Routes==2.4.1
rsa==3.4.2
six==1.11.0
SQLAlchemy==1.2.7
SQLAlchemy==1.2.8
sqlalchemy-migrate==0.11.0
sqlparse==0.2.4
statsd==3.2.2
@ -95,11 +97,11 @@ testscenarios==0.5.0
testtools==2.3.0
traceback2==1.4.0
unittest2==1.1.0
urllib3==1.22
urllib3==1.23
uWSGI==2.0.15
vine==1.1.4
wcwidth==0.1.7
WebOb==1.8.1
WebOb==1.8.2
websocket-client==0.40.0
Werkzeug==0.14.1
wrapt==1.10.11

View File

@ -109,7 +109,7 @@
"name": "Check full etcd cluster",
"script": "check-etcd-health.sh",
"arguments": [
"-w", "30",
"-w", "60",
"-e", "kubernetes n0 n0 n1 n2 n3",
"-e", "calico n0 n0 n1 n2 n3"
]