bug(wait): fixing how we wait on chart and group
- added abilty to choose labels to wait on - created wait object Change-Id: Ia3b6f7bd7b6ef15779b087c613d69f4f6a7b41e9
This commit is contained in:
parent
be8e86351f
commit
a99fc4ad6c
|
@ -213,8 +213,8 @@ class Armada(object):
|
||||||
release[1])
|
release[1])
|
||||||
|
|
||||||
for entry in self.config[const.KEYWORD_ARMADA][const.KEYWORD_GROUPS]:
|
for entry in self.config[const.KEYWORD_ARMADA][const.KEYWORD_GROUPS]:
|
||||||
chart_wait = self.wait
|
|
||||||
|
|
||||||
|
chart_wait = self.wait
|
||||||
desc = entry.get('description', 'A Chart Group')
|
desc = entry.get('description', 'A Chart Group')
|
||||||
chart_group = entry.get(const.KEYWORD_CHARTS, [])
|
chart_group = entry.get(const.KEYWORD_CHARTS, [])
|
||||||
test_charts = entry.get('test_charts', False)
|
test_charts = entry.get('test_charts', False)
|
||||||
|
@ -227,6 +227,7 @@ class Armada(object):
|
||||||
for gchart in chart_group:
|
for gchart in chart_group:
|
||||||
chart = dotify(gchart['chart'])
|
chart = dotify(gchart['chart'])
|
||||||
values = gchart.get('chart').get('values', {})
|
values = gchart.get('chart').get('values', {})
|
||||||
|
wait_values = gchart.get('chart').get('wait', {})
|
||||||
test_chart = gchart.get('chart').get('test', False)
|
test_chart = gchart.get('chart').get('test', False)
|
||||||
pre_actions = {}
|
pre_actions = {}
|
||||||
post_actions = {}
|
post_actions = {}
|
||||||
|
@ -242,7 +243,7 @@ class Armada(object):
|
||||||
if chart_wait:
|
if chart_wait:
|
||||||
if chart_timeout == DEFAULT_TIMEOUT:
|
if chart_timeout == DEFAULT_TIMEOUT:
|
||||||
chart_timeout = getattr(
|
chart_timeout = getattr(
|
||||||
chart, 'timeout', DEFAULT_TIMEOUT)
|
chart, 'timeout', chart_timeout)
|
||||||
|
|
||||||
chartbuilder = ChartBuilder(chart)
|
chartbuilder = ChartBuilder(chart)
|
||||||
protoc_chart = chartbuilder.get_helm_chart()
|
protoc_chart = chartbuilder.get_helm_chart()
|
||||||
|
@ -302,10 +303,16 @@ class Armada(object):
|
||||||
timeout=chart_timeout)
|
timeout=chart_timeout)
|
||||||
|
|
||||||
if chart_wait:
|
if chart_wait:
|
||||||
|
# TODO(gardlt): after v0.7.1 depricate timeout values
|
||||||
|
if not wait_values.get('timeout', None):
|
||||||
|
wait_values['timeout'] = chart_timeout
|
||||||
|
|
||||||
self.tiller.k8s.wait_until_ready(
|
self.tiller.k8s.wait_until_ready(
|
||||||
release=prefix_chart,
|
release=prefix_chart,
|
||||||
|
labels=wait_values.get('labels', ''),
|
||||||
namespace=chart.namespace,
|
namespace=chart.namespace,
|
||||||
timeout=chart_timeout)
|
timeout=wait_values.get('timeout', DEFAULT_TIMEOUT)
|
||||||
|
)
|
||||||
|
|
||||||
msg['upgraded'].append(prefix_chart)
|
msg['upgraded'].append(prefix_chart)
|
||||||
|
|
||||||
|
@ -322,10 +329,14 @@ class Armada(object):
|
||||||
timeout=chart_timeout)
|
timeout=chart_timeout)
|
||||||
|
|
||||||
if chart_wait:
|
if chart_wait:
|
||||||
|
if not wait_values.get('timeout', None):
|
||||||
|
wait_values['timeout'] = chart_timeout
|
||||||
|
|
||||||
self.tiller.k8s.wait_until_ready(
|
self.tiller.k8s.wait_until_ready(
|
||||||
release=prefix_chart,
|
release=prefix_chart,
|
||||||
|
labels=wait_values.get('labels', ''),
|
||||||
namespace=chart.namespace,
|
namespace=chart.namespace,
|
||||||
timeout=chart_timeout)
|
timeout=wait_values.get('timeout', 3600))
|
||||||
|
|
||||||
msg['installed'].append(prefix_chart)
|
msg['installed'].append(prefix_chart)
|
||||||
|
|
||||||
|
|
|
@ -14,12 +14,15 @@
|
||||||
|
|
||||||
import re
|
import re
|
||||||
import time
|
import time
|
||||||
|
|
||||||
from kubernetes import client, config, watch
|
from kubernetes import client, config, watch
|
||||||
from kubernetes.client.rest import ApiException
|
from kubernetes.client.rest import ApiException
|
||||||
|
|
||||||
from oslo_config import cfg
|
from oslo_config import cfg
|
||||||
from oslo_log import log as logging
|
from oslo_log import log as logging
|
||||||
|
|
||||||
|
from armada.utils.release import label_selectors
|
||||||
|
|
||||||
|
|
||||||
LOG = logging.getLogger(__name__)
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
CONF = cfg.CONF
|
CONF = cfg.CONF
|
||||||
|
@ -191,6 +194,7 @@ class K8s(object):
|
||||||
def wait_until_ready(self,
|
def wait_until_ready(self,
|
||||||
release=None,
|
release=None,
|
||||||
namespace='default',
|
namespace='default',
|
||||||
|
labels='',
|
||||||
timeout=300,
|
timeout=300,
|
||||||
sleep=15):
|
sleep=15):
|
||||||
'''
|
'''
|
||||||
|
@ -200,9 +204,9 @@ class K8s(object):
|
||||||
LOG.debug("Wait on %s for %s sec", namespace, timeout)
|
LOG.debug("Wait on %s for %s sec", namespace, timeout)
|
||||||
|
|
||||||
label_selector = ''
|
label_selector = ''
|
||||||
# FIXME(gardlt): this requires a label schema from OSH
|
|
||||||
if release is not None:
|
if labels:
|
||||||
label_selector = 'release_group={}'.format(release)
|
label_selector = label_selectors(labels)
|
||||||
|
|
||||||
valid_state = ['Succeeded', 'Running']
|
valid_state = ['Succeeded', 'Running']
|
||||||
|
|
||||||
|
@ -213,19 +217,27 @@ class K8s(object):
|
||||||
self.is_pods_ready(label_selector=label_selector, timeout=timeout)
|
self.is_pods_ready(label_selector=label_selector, timeout=timeout)
|
||||||
|
|
||||||
pod_ready = []
|
pod_ready = []
|
||||||
|
not_ready = []
|
||||||
for pod in self.client.list_pod_for_all_namespaces(
|
for pod in self.client.list_pod_for_all_namespaces(
|
||||||
label_selector=label_selector).items:
|
label_selector=label_selector).items:
|
||||||
p_state = pod.status.phase
|
p_state = pod.status.phase
|
||||||
|
p_name = pod.metadata.name
|
||||||
if p_state in valid_state:
|
if p_state in valid_state:
|
||||||
pod_ready.append(True)
|
pod_ready.append(True)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
pod_ready.append(False)
|
pod_ready.append(False)
|
||||||
|
not_ready.append(p_name)
|
||||||
|
|
||||||
LOG.debug('%s', p_state)
|
LOG.debug('%s', p_state)
|
||||||
|
|
||||||
if time.time() > wait_timeout or all(pod_ready):
|
if time.time() > wait_timeout or all(pod_ready):
|
||||||
LOG.debug("Pod States %s", pod_ready)
|
LOG.debug("Pod States %s", pod_ready)
|
||||||
break
|
break
|
||||||
|
if time.time() > wait_timeout and not all(pod_ready):
|
||||||
|
LOG.exception(
|
||||||
|
'Failed to bring up release %s: %s', release, not_ready)
|
||||||
|
break
|
||||||
else:
|
else:
|
||||||
LOG.debug('time: %s pod %s', wait_timeout, pod_ready)
|
LOG.debug('time: %s pod %s', wait_timeout, pod_ready)
|
||||||
|
|
||||||
|
|
|
@ -32,7 +32,7 @@ from armada.const import STATUS_DEPLOYED, STATUS_FAILED
|
||||||
from armada.exceptions import tiller_exceptions as ex
|
from armada.exceptions import tiller_exceptions as ex
|
||||||
from armada.handlers.k8s import K8s
|
from armada.handlers.k8s import K8s
|
||||||
from armada.utils.release import release_prefix
|
from armada.utils.release import release_prefix
|
||||||
|
from armada.utils.release import label_selectors
|
||||||
|
|
||||||
TILLER_PORT = 44134
|
TILLER_PORT = 44134
|
||||||
TILLER_VERSION = b'2.5.0'
|
TILLER_VERSION = b'2.5.0'
|
||||||
|
@ -479,13 +479,8 @@ class Tiller(object):
|
||||||
'''
|
'''
|
||||||
|
|
||||||
label_selector = ''
|
label_selector = ''
|
||||||
|
|
||||||
if not resource_type == 'job':
|
|
||||||
label_selector = 'release_name={}'.format(release_name)
|
|
||||||
|
|
||||||
if resource_labels is not None:
|
if resource_labels is not None:
|
||||||
label_selector = ",".join(
|
label_selector = label_selectors(resource_labels)
|
||||||
["%s=%s" % (k, v) for k, v in resource_labels.items()])
|
|
||||||
|
|
||||||
if 'job' in resource_type:
|
if 'job' in resource_type:
|
||||||
LOG.info("Deleting %s in namespace: %s", resource_name, namespace)
|
LOG.info("Deleting %s in namespace: %s", resource_name, namespace)
|
||||||
|
@ -509,7 +504,7 @@ class Tiller(object):
|
||||||
resource_name, resource_type)
|
resource_name, resource_type)
|
||||||
|
|
||||||
def rolling_upgrade_pod_deployment(self, name, release_name, namespace,
|
def rolling_upgrade_pod_deployment(self, name, release_name, namespace,
|
||||||
labels, action_type, chart,
|
resource_labels, action_type, chart,
|
||||||
disable_hooks, values):
|
disable_hooks, values):
|
||||||
'''
|
'''
|
||||||
update statefullsets (daemon, stateful)
|
update statefullsets (daemon, stateful)
|
||||||
|
@ -518,11 +513,11 @@ class Tiller(object):
|
||||||
if action_type == 'daemonset':
|
if action_type == 'daemonset':
|
||||||
|
|
||||||
LOG.info('Updating: %s', action_type)
|
LOG.info('Updating: %s', action_type)
|
||||||
label_selector = 'release_name={}'.format(release_name)
|
|
||||||
|
|
||||||
if labels is not None:
|
label_selector = ''
|
||||||
label_selector = ",".join(
|
|
||||||
["%s=%s" % (k, v) for k, v in labels.items()])
|
if resource_labels is not None:
|
||||||
|
label_selector = label_selectors(resource_labels)
|
||||||
|
|
||||||
get_daemonset = self.k8s.get_namespace_daemonset(
|
get_daemonset = self.k8s.get_namespace_daemonset(
|
||||||
namespace=namespace, label=label_selector)
|
namespace=namespace, label=label_selector)
|
||||||
|
@ -547,8 +542,8 @@ class Tiller(object):
|
||||||
namespace=namespace, template=template)
|
namespace=namespace, template=template)
|
||||||
|
|
||||||
# delete pods
|
# delete pods
|
||||||
self.delete_resources(release_name, name, 'pod', labels,
|
self.delete_resources(release_name, name, 'pod',
|
||||||
namespace)
|
resource_labels, namespace)
|
||||||
|
|
||||||
elif action_type == 'statefulset':
|
elif action_type == 'statefulset':
|
||||||
pass
|
pass
|
||||||
|
|
|
@ -18,3 +18,13 @@ def release_prefix(prefix, chart):
|
||||||
how to attach prefix to chart
|
how to attach prefix to chart
|
||||||
'''
|
'''
|
||||||
return "{}-{}".format(prefix, chart)
|
return "{}-{}".format(prefix, chart)
|
||||||
|
|
||||||
|
|
||||||
|
def label_selectors(labels):
|
||||||
|
"""
|
||||||
|
:param labels: dictionary containing k, v
|
||||||
|
|
||||||
|
:return: string of k8s labels
|
||||||
|
"""
|
||||||
|
return ",".join(
|
||||||
|
["%s=%s" % (k, v) for k, v in labels.items()])
|
||||||
|
|
|
@ -75,32 +75,40 @@ Example
|
||||||
armada/Chart/v1
|
armada/Chart/v1
|
||||||
---------------
|
---------------
|
||||||
|
|
||||||
|
.. DANGER::
|
||||||
|
|
||||||
|
DEPRICATION: ``timeout`` key-value will be removed timeout will be defiend
|
||||||
|
under ``wait`` object.
|
||||||
|
|
||||||
|
|
||||||
Chart
|
Chart
|
||||||
^^^^^
|
^^^^^
|
||||||
|
|
||||||
+-----------------+----------+---------------------------------------------------------------------------+
|
+-----------------+----------+---------------------------------------------------------------------------------------+
|
||||||
| keyword | type | action |
|
| keyword | type | action |
|
||||||
+=================+==========+===========================================================================+
|
+=================+==========+=======================================================================================+
|
||||||
| chart\_name | string | name for the chart |
|
| chart\_name | string | name for the chart |
|
||||||
+-----------------+----------+---------------------------------------------------------------------------+
|
+-----------------+----------+---------------------------------------------------------------------------------------+
|
||||||
| release\_name | string | name of the release |
|
| release\_name | string | name of the release |
|
||||||
+-----------------+----------+---------------------------------------------------------------------------+
|
+-----------------+----------+---------------------------------------------------------------------------------------+
|
||||||
| namespace | string | namespace of your chart |
|
| namespace | string | namespace of your chart |
|
||||||
+-----------------+----------+---------------------------------------------------------------------------+
|
+-----------------+----------+---------------------------------------------------------------------------------------+
|
||||||
| timeout | int | time (in seconds) allotted for chart to deploy when 'wait' flag is set |
|
| wait | object | contains wait information such as (timeout, lables) |
|
||||||
+-----------------+----------+---------------------------------------------------------------------------+
|
+-----------------+----------+---------------------------------------------------------------------------------------+
|
||||||
| test | bool | run pre-defined helm tests helm in a chart |
|
| test | bool | run pre-defined helm tests helm in a chart |
|
||||||
+-----------------+----------+---------------------------------------------------------------------------+
|
+-----------------+----------+---------------------------------------------------------------------------------------+
|
||||||
| install | object | install the chart into your Kubernetes cluster |
|
| install | object | install the chart into your Kubernetes cluster |
|
||||||
+-----------------+----------+---------------------------------------------------------------------------+
|
+-----------------+----------+---------------------------------------------------------------------------------------+
|
||||||
| update | object | update the chart managed by the armada yaml |
|
| update | object | update the chart managed by the armada yaml |
|
||||||
+-----------------+----------+---------------------------------------------------------------------------+
|
+-----------------+----------+---------------------------------------------------------------------------------------+
|
||||||
| values | object | override any default values in the charts |
|
| values | object | override any default values in the charts |
|
||||||
+-----------------+----------+---------------------------------------------------------------------------+
|
+-----------------+----------+---------------------------------------------------------------------------------------+
|
||||||
| source | object | provide a path to a ``git repo``, ``local dir``, or ``tarball url`` chart |
|
| source | object | provide a path to a ``git repo``, ``local dir``, or ``tarball url`` chart |
|
||||||
+-----------------+----------+---------------------------------------------------------------------------+
|
+-----------------+----------+---------------------------------------------------------------------------------------+
|
||||||
| dependencies | object | reference any chart dependencies before install |
|
| dependencies | object | reference any chart dependencies before install |
|
||||||
+-----------------+----------+---------------------------------------------------------------------------+
|
+-----------------+----------+---------------------------------------------------------------------------------------+
|
||||||
|
| timeout | int | time (in seconds) allotted for chart to deploy when 'wait' flag is set (DEPRICATED) |
|
||||||
|
+-----------------+----------+---------------------------------------------------------------------------------------+
|
||||||
|
|
||||||
Update - Pre or Post
|
Update - Pre or Post
|
||||||
^^^^^^^^^^^^^^^^^^^^
|
^^^^^^^^^^^^^^^^^^^^
|
||||||
|
@ -167,7 +175,10 @@ Example
|
||||||
chart_name: blog-1
|
chart_name: blog-1
|
||||||
release_name: blog-1
|
release_name: blog-1
|
||||||
namespace: default
|
namespace: default
|
||||||
timeout: 100
|
wait:
|
||||||
|
timeout: 100
|
||||||
|
labels:
|
||||||
|
component: blog
|
||||||
install:
|
install:
|
||||||
no_hook: false
|
no_hook: false
|
||||||
upgrade:
|
upgrade:
|
||||||
|
@ -190,7 +201,8 @@ Example
|
||||||
chart_name: blog-1
|
chart_name: blog-1
|
||||||
release_name: blog-1
|
release_name: blog-1
|
||||||
namespace: default
|
namespace: default
|
||||||
timeout: 100
|
wait:
|
||||||
|
timeout: 100
|
||||||
install:
|
install:
|
||||||
no_hook: false
|
no_hook: false
|
||||||
upgrade:
|
upgrade:
|
||||||
|
@ -213,7 +225,8 @@ Example
|
||||||
chart_name: blog-1
|
chart_name: blog-1
|
||||||
release_name: blog-1
|
release_name: blog-1
|
||||||
namespace: default
|
namespace: default
|
||||||
timeout: 100
|
wait:
|
||||||
|
timeout: 100
|
||||||
install:
|
install:
|
||||||
no_hook: false
|
no_hook: false
|
||||||
upgrade:
|
upgrade:
|
||||||
|
@ -257,7 +270,8 @@ Example
|
||||||
chart_name: blog-1
|
chart_name: blog-1
|
||||||
release_name: blog-1
|
release_name: blog-1
|
||||||
namespace: default
|
namespace: default
|
||||||
timeout: 100
|
wait:
|
||||||
|
timeout: 100
|
||||||
install:
|
install:
|
||||||
no_hook: false
|
no_hook: false
|
||||||
upgrade:
|
upgrade:
|
||||||
|
|
|
@ -23,7 +23,10 @@ data:
|
||||||
chart_name: mariadb
|
chart_name: mariadb
|
||||||
release: mariadb
|
release: mariadb
|
||||||
namespace: openstack
|
namespace: openstack
|
||||||
timeout: 3600
|
wait:
|
||||||
|
timeout: 3600
|
||||||
|
labels:
|
||||||
|
release_group: armada-mariadb
|
||||||
install:
|
install:
|
||||||
no_hooks: false
|
no_hooks: false
|
||||||
upgrade:
|
upgrade:
|
||||||
|
@ -45,7 +48,10 @@ data:
|
||||||
chart_name: memcached
|
chart_name: memcached
|
||||||
release: memcached
|
release: memcached
|
||||||
namespace: openstack
|
namespace: openstack
|
||||||
timeout: 100
|
wait:
|
||||||
|
timeout: 100
|
||||||
|
labels:
|
||||||
|
release_group: armada-memcached
|
||||||
install:
|
install:
|
||||||
no_hooks: false
|
no_hooks: false
|
||||||
upgrade:
|
upgrade:
|
||||||
|
@ -68,7 +74,10 @@ data:
|
||||||
test: true
|
test: true
|
||||||
release: keystone
|
release: keystone
|
||||||
namespace: openstack
|
namespace: openstack
|
||||||
timeout: 100
|
wait:
|
||||||
|
timeout: 100
|
||||||
|
labels:
|
||||||
|
release_group: armada-keystone
|
||||||
install:
|
install:
|
||||||
no_hooks: false
|
no_hooks: false
|
||||||
upgrade:
|
upgrade:
|
||||||
|
|
Loading…
Reference in New Issue