bug(wait): fixing how we wait on chart and group

- added abilty to choose labels to wait on
- created wait object

Change-Id: Ia3b6f7bd7b6ef15779b087c613d69f4f6a7b41e9
This commit is contained in:
gardlt 2017-10-30 19:24:57 +00:00
parent be8e86351f
commit a99fc4ad6c
6 changed files with 103 additions and 52 deletions

View File

@ -213,8 +213,8 @@ class Armada(object):
release[1])
for entry in self.config[const.KEYWORD_ARMADA][const.KEYWORD_GROUPS]:
chart_wait = self.wait
chart_wait = self.wait
desc = entry.get('description', 'A Chart Group')
chart_group = entry.get(const.KEYWORD_CHARTS, [])
test_charts = entry.get('test_charts', False)
@ -227,6 +227,7 @@ class Armada(object):
for gchart in chart_group:
chart = dotify(gchart['chart'])
values = gchart.get('chart').get('values', {})
wait_values = gchart.get('chart').get('wait', {})
test_chart = gchart.get('chart').get('test', False)
pre_actions = {}
post_actions = {}
@ -242,7 +243,7 @@ class Armada(object):
if chart_wait:
if chart_timeout == DEFAULT_TIMEOUT:
chart_timeout = getattr(
chart, 'timeout', DEFAULT_TIMEOUT)
chart, 'timeout', chart_timeout)
chartbuilder = ChartBuilder(chart)
protoc_chart = chartbuilder.get_helm_chart()
@ -302,10 +303,16 @@ class Armada(object):
timeout=chart_timeout)
if chart_wait:
# TODO(gardlt): after v0.7.1 depricate timeout values
if not wait_values.get('timeout', None):
wait_values['timeout'] = chart_timeout
self.tiller.k8s.wait_until_ready(
release=prefix_chart,
labels=wait_values.get('labels', ''),
namespace=chart.namespace,
timeout=chart_timeout)
timeout=wait_values.get('timeout', DEFAULT_TIMEOUT)
)
msg['upgraded'].append(prefix_chart)
@ -322,10 +329,14 @@ class Armada(object):
timeout=chart_timeout)
if chart_wait:
if not wait_values.get('timeout', None):
wait_values['timeout'] = chart_timeout
self.tiller.k8s.wait_until_ready(
release=prefix_chart,
labels=wait_values.get('labels', ''),
namespace=chart.namespace,
timeout=chart_timeout)
timeout=wait_values.get('timeout', 3600))
msg['installed'].append(prefix_chart)

View File

@ -14,12 +14,15 @@
import re
import time
from kubernetes import client, config, watch
from kubernetes.client.rest import ApiException
from oslo_config import cfg
from oslo_log import log as logging
from armada.utils.release import label_selectors
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
@ -191,6 +194,7 @@ class K8s(object):
def wait_until_ready(self,
release=None,
namespace='default',
labels='',
timeout=300,
sleep=15):
'''
@ -200,9 +204,9 @@ class K8s(object):
LOG.debug("Wait on %s for %s sec", namespace, timeout)
label_selector = ''
# FIXME(gardlt): this requires a label schema from OSH
if release is not None:
label_selector = 'release_group={}'.format(release)
if labels:
label_selector = label_selectors(labels)
valid_state = ['Succeeded', 'Running']
@ -213,19 +217,27 @@ class K8s(object):
self.is_pods_ready(label_selector=label_selector, timeout=timeout)
pod_ready = []
not_ready = []
for pod in self.client.list_pod_for_all_namespaces(
label_selector=label_selector).items:
p_state = pod.status.phase
p_name = pod.metadata.name
if p_state in valid_state:
pod_ready.append(True)
continue
pod_ready.append(False)
not_ready.append(p_name)
LOG.debug('%s', p_state)
if time.time() > wait_timeout or all(pod_ready):
LOG.debug("Pod States %s", pod_ready)
break
if time.time() > wait_timeout and not all(pod_ready):
LOG.exception(
'Failed to bring up release %s: %s', release, not_ready)
break
else:
LOG.debug('time: %s pod %s', wait_timeout, pod_ready)

View File

@ -32,7 +32,7 @@ from armada.const import STATUS_DEPLOYED, STATUS_FAILED
from armada.exceptions import tiller_exceptions as ex
from armada.handlers.k8s import K8s
from armada.utils.release import release_prefix
from armada.utils.release import label_selectors
TILLER_PORT = 44134
TILLER_VERSION = b'2.5.0'
@ -479,13 +479,8 @@ class Tiller(object):
'''
label_selector = ''
if not resource_type == 'job':
label_selector = 'release_name={}'.format(release_name)
if resource_labels is not None:
label_selector = ",".join(
["%s=%s" % (k, v) for k, v in resource_labels.items()])
label_selector = label_selectors(resource_labels)
if 'job' in resource_type:
LOG.info("Deleting %s in namespace: %s", resource_name, namespace)
@ -509,7 +504,7 @@ class Tiller(object):
resource_name, resource_type)
def rolling_upgrade_pod_deployment(self, name, release_name, namespace,
labels, action_type, chart,
resource_labels, action_type, chart,
disable_hooks, values):
'''
update statefullsets (daemon, stateful)
@ -518,11 +513,11 @@ class Tiller(object):
if action_type == 'daemonset':
LOG.info('Updating: %s', action_type)
label_selector = 'release_name={}'.format(release_name)
if labels is not None:
label_selector = ",".join(
["%s=%s" % (k, v) for k, v in labels.items()])
label_selector = ''
if resource_labels is not None:
label_selector = label_selectors(resource_labels)
get_daemonset = self.k8s.get_namespace_daemonset(
namespace=namespace, label=label_selector)
@ -547,8 +542,8 @@ class Tiller(object):
namespace=namespace, template=template)
# delete pods
self.delete_resources(release_name, name, 'pod', labels,
namespace)
self.delete_resources(release_name, name, 'pod',
resource_labels, namespace)
elif action_type == 'statefulset':
pass

View File

@ -18,3 +18,13 @@ def release_prefix(prefix, chart):
how to attach prefix to chart
'''
return "{}-{}".format(prefix, chart)
def label_selectors(labels):
"""
:param labels: dictionary containing k, v
:return: string of k8s labels
"""
return ",".join(
["%s=%s" % (k, v) for k, v in labels.items()])

View File

@ -75,32 +75,40 @@ Example
armada/Chart/v1
---------------
.. DANGER::
DEPRICATION: ``timeout`` key-value will be removed timeout will be defiend
under ``wait`` object.
Chart
^^^^^
+-----------------+----------+---------------------------------------------------------------------------+
| keyword | type | action |
+=================+==========+===========================================================================+
| chart\_name | string | name for the chart |
+-----------------+----------+---------------------------------------------------------------------------+
| release\_name | string | name of the release |
+-----------------+----------+---------------------------------------------------------------------------+
| namespace | string | namespace of your chart |
+-----------------+----------+---------------------------------------------------------------------------+
| timeout | int | time (in seconds) allotted for chart to deploy when 'wait' flag is set |
+-----------------+----------+---------------------------------------------------------------------------+
| test | bool | run pre-defined helm tests helm in a chart |
+-----------------+----------+---------------------------------------------------------------------------+
| install | object | install the chart into your Kubernetes cluster |
+-----------------+----------+---------------------------------------------------------------------------+
| update | object | update the chart managed by the armada yaml |
+-----------------+----------+---------------------------------------------------------------------------+
| values | object | override any default values in the charts |
+-----------------+----------+---------------------------------------------------------------------------+
| source | object | provide a path to a ``git repo``, ``local dir``, or ``tarball url`` chart |
+-----------------+----------+---------------------------------------------------------------------------+
| dependencies | object | reference any chart dependencies before install |
+-----------------+----------+---------------------------------------------------------------------------+
+-----------------+----------+---------------------------------------------------------------------------------------+
| keyword | type | action |
+=================+==========+=======================================================================================+
| chart\_name | string | name for the chart |
+-----------------+----------+---------------------------------------------------------------------------------------+
| release\_name | string | name of the release |
+-----------------+----------+---------------------------------------------------------------------------------------+
| namespace | string | namespace of your chart |
+-----------------+----------+---------------------------------------------------------------------------------------+
| wait | object | contains wait information such as (timeout, lables) |
+-----------------+----------+---------------------------------------------------------------------------------------+
| test | bool | run pre-defined helm tests helm in a chart |
+-----------------+----------+---------------------------------------------------------------------------------------+
| install | object | install the chart into your Kubernetes cluster |
+-----------------+----------+---------------------------------------------------------------------------------------+
| update | object | update the chart managed by the armada yaml |
+-----------------+----------+---------------------------------------------------------------------------------------+
| values | object | override any default values in the charts |
+-----------------+----------+---------------------------------------------------------------------------------------+
| source | object | provide a path to a ``git repo``, ``local dir``, or ``tarball url`` chart |
+-----------------+----------+---------------------------------------------------------------------------------------+
| dependencies | object | reference any chart dependencies before install |
+-----------------+----------+---------------------------------------------------------------------------------------+
| timeout | int | time (in seconds) allotted for chart to deploy when 'wait' flag is set (DEPRICATED) |
+-----------------+----------+---------------------------------------------------------------------------------------+
Update - Pre or Post
^^^^^^^^^^^^^^^^^^^^
@ -167,7 +175,10 @@ Example
chart_name: blog-1
release_name: blog-1
namespace: default
timeout: 100
wait:
timeout: 100
labels:
component: blog
install:
no_hook: false
upgrade:
@ -190,7 +201,8 @@ Example
chart_name: blog-1
release_name: blog-1
namespace: default
timeout: 100
wait:
timeout: 100
install:
no_hook: false
upgrade:
@ -213,7 +225,8 @@ Example
chart_name: blog-1
release_name: blog-1
namespace: default
timeout: 100
wait:
timeout: 100
install:
no_hook: false
upgrade:
@ -257,7 +270,8 @@ Example
chart_name: blog-1
release_name: blog-1
namespace: default
timeout: 100
wait:
timeout: 100
install:
no_hook: false
upgrade:

View File

@ -23,7 +23,10 @@ data:
chart_name: mariadb
release: mariadb
namespace: openstack
timeout: 3600
wait:
timeout: 3600
labels:
release_group: armada-mariadb
install:
no_hooks: false
upgrade:
@ -45,7 +48,10 @@ data:
chart_name: memcached
release: memcached
namespace: openstack
timeout: 100
wait:
timeout: 100
labels:
release_group: armada-memcached
install:
no_hooks: false
upgrade:
@ -68,7 +74,10 @@ data:
test: true
release: keystone
namespace: openstack
timeout: 100
wait:
timeout: 100
labels:
release_group: armada-keystone
install:
no_hooks: false
upgrade: