From a99fc4ad6c7d5b31eb05c6959960c6baf39e801a Mon Sep 17 00:00:00 2001 From: gardlt Date: Mon, 30 Oct 2017 19:24:57 +0000 Subject: [PATCH] bug(wait): fixing how we wait on chart and group - added abilty to choose labels to wait on - created wait object Change-Id: Ia3b6f7bd7b6ef15779b087c613d69f4f6a7b41e9 --- armada/handlers/armada.py | 19 ++++-- armada/handlers/k8s.py | 20 ++++-- armada/handlers/tiller.py | 23 +++---- armada/utils/release.py | 10 +++ .../operations/guide-build-armada-yaml.rst | 68 +++++++++++-------- examples/keystone-manifest.yaml | 15 +++- 6 files changed, 103 insertions(+), 52 deletions(-) diff --git a/armada/handlers/armada.py b/armada/handlers/armada.py index 15fa7c29..66cabf4c 100644 --- a/armada/handlers/armada.py +++ b/armada/handlers/armada.py @@ -213,8 +213,8 @@ class Armada(object): release[1]) for entry in self.config[const.KEYWORD_ARMADA][const.KEYWORD_GROUPS]: - chart_wait = self.wait + chart_wait = self.wait desc = entry.get('description', 'A Chart Group') chart_group = entry.get(const.KEYWORD_CHARTS, []) test_charts = entry.get('test_charts', False) @@ -227,6 +227,7 @@ class Armada(object): for gchart in chart_group: chart = dotify(gchart['chart']) values = gchart.get('chart').get('values', {}) + wait_values = gchart.get('chart').get('wait', {}) test_chart = gchart.get('chart').get('test', False) pre_actions = {} post_actions = {} @@ -242,7 +243,7 @@ class Armada(object): if chart_wait: if chart_timeout == DEFAULT_TIMEOUT: chart_timeout = getattr( - chart, 'timeout', DEFAULT_TIMEOUT) + chart, 'timeout', chart_timeout) chartbuilder = ChartBuilder(chart) protoc_chart = chartbuilder.get_helm_chart() @@ -302,10 +303,16 @@ class Armada(object): timeout=chart_timeout) if chart_wait: + # TODO(gardlt): after v0.7.1 depricate timeout values + if not wait_values.get('timeout', None): + wait_values['timeout'] = chart_timeout + self.tiller.k8s.wait_until_ready( release=prefix_chart, + labels=wait_values.get('labels', ''), namespace=chart.namespace, - timeout=chart_timeout) + timeout=wait_values.get('timeout', DEFAULT_TIMEOUT) + ) msg['upgraded'].append(prefix_chart) @@ -322,10 +329,14 @@ class Armada(object): timeout=chart_timeout) if chart_wait: + if not wait_values.get('timeout', None): + wait_values['timeout'] = chart_timeout + self.tiller.k8s.wait_until_ready( release=prefix_chart, + labels=wait_values.get('labels', ''), namespace=chart.namespace, - timeout=chart_timeout) + timeout=wait_values.get('timeout', 3600)) msg['installed'].append(prefix_chart) diff --git a/armada/handlers/k8s.py b/armada/handlers/k8s.py index b37e9e99..03413ffb 100644 --- a/armada/handlers/k8s.py +++ b/armada/handlers/k8s.py @@ -14,12 +14,15 @@ import re import time + from kubernetes import client, config, watch from kubernetes.client.rest import ApiException - from oslo_config import cfg from oslo_log import log as logging +from armada.utils.release import label_selectors + + LOG = logging.getLogger(__name__) CONF = cfg.CONF @@ -191,6 +194,7 @@ class K8s(object): def wait_until_ready(self, release=None, namespace='default', + labels='', timeout=300, sleep=15): ''' @@ -200,9 +204,9 @@ class K8s(object): LOG.debug("Wait on %s for %s sec", namespace, timeout) label_selector = '' - # FIXME(gardlt): this requires a label schema from OSH - if release is not None: - label_selector = 'release_group={}'.format(release) + + if labels: + label_selector = label_selectors(labels) valid_state = ['Succeeded', 'Running'] @@ -213,19 +217,27 @@ class K8s(object): self.is_pods_ready(label_selector=label_selector, timeout=timeout) pod_ready = [] + not_ready = [] for pod in self.client.list_pod_for_all_namespaces( label_selector=label_selector).items: p_state = pod.status.phase + p_name = pod.metadata.name if p_state in valid_state: pod_ready.append(True) continue pod_ready.append(False) + not_ready.append(p_name) + LOG.debug('%s', p_state) if time.time() > wait_timeout or all(pod_ready): LOG.debug("Pod States %s", pod_ready) break + if time.time() > wait_timeout and not all(pod_ready): + LOG.exception( + 'Failed to bring up release %s: %s', release, not_ready) + break else: LOG.debug('time: %s pod %s', wait_timeout, pod_ready) diff --git a/armada/handlers/tiller.py b/armada/handlers/tiller.py index 7248255a..ec898f9c 100644 --- a/armada/handlers/tiller.py +++ b/armada/handlers/tiller.py @@ -32,7 +32,7 @@ from armada.const import STATUS_DEPLOYED, STATUS_FAILED from armada.exceptions import tiller_exceptions as ex from armada.handlers.k8s import K8s from armada.utils.release import release_prefix - +from armada.utils.release import label_selectors TILLER_PORT = 44134 TILLER_VERSION = b'2.5.0' @@ -479,13 +479,8 @@ class Tiller(object): ''' label_selector = '' - - if not resource_type == 'job': - label_selector = 'release_name={}'.format(release_name) - if resource_labels is not None: - label_selector = ",".join( - ["%s=%s" % (k, v) for k, v in resource_labels.items()]) + label_selector = label_selectors(resource_labels) if 'job' in resource_type: LOG.info("Deleting %s in namespace: %s", resource_name, namespace) @@ -509,7 +504,7 @@ class Tiller(object): resource_name, resource_type) def rolling_upgrade_pod_deployment(self, name, release_name, namespace, - labels, action_type, chart, + resource_labels, action_type, chart, disable_hooks, values): ''' update statefullsets (daemon, stateful) @@ -518,11 +513,11 @@ class Tiller(object): if action_type == 'daemonset': LOG.info('Updating: %s', action_type) - label_selector = 'release_name={}'.format(release_name) - if labels is not None: - label_selector = ",".join( - ["%s=%s" % (k, v) for k, v in labels.items()]) + label_selector = '' + + if resource_labels is not None: + label_selector = label_selectors(resource_labels) get_daemonset = self.k8s.get_namespace_daemonset( namespace=namespace, label=label_selector) @@ -547,8 +542,8 @@ class Tiller(object): namespace=namespace, template=template) # delete pods - self.delete_resources(release_name, name, 'pod', labels, - namespace) + self.delete_resources(release_name, name, 'pod', + resource_labels, namespace) elif action_type == 'statefulset': pass diff --git a/armada/utils/release.py b/armada/utils/release.py index c622e95e..084292d1 100644 --- a/armada/utils/release.py +++ b/armada/utils/release.py @@ -18,3 +18,13 @@ def release_prefix(prefix, chart): how to attach prefix to chart ''' return "{}-{}".format(prefix, chart) + + +def label_selectors(labels): + """ + :param labels: dictionary containing k, v + + :return: string of k8s labels + """ + return ",".join( + ["%s=%s" % (k, v) for k, v in labels.items()]) diff --git a/docs/source/operations/guide-build-armada-yaml.rst b/docs/source/operations/guide-build-armada-yaml.rst index 9948df50..72522d07 100644 --- a/docs/source/operations/guide-build-armada-yaml.rst +++ b/docs/source/operations/guide-build-armada-yaml.rst @@ -75,32 +75,40 @@ Example armada/Chart/v1 --------------- +.. DANGER:: + + DEPRICATION: ``timeout`` key-value will be removed timeout will be defiend + under ``wait`` object. + + Chart ^^^^^ -+-----------------+----------+---------------------------------------------------------------------------+ -| keyword | type | action | -+=================+==========+===========================================================================+ -| chart\_name | string | name for the chart | -+-----------------+----------+---------------------------------------------------------------------------+ -| release\_name | string | name of the release | -+-----------------+----------+---------------------------------------------------------------------------+ -| namespace | string | namespace of your chart | -+-----------------+----------+---------------------------------------------------------------------------+ -| timeout | int | time (in seconds) allotted for chart to deploy when 'wait' flag is set | -+-----------------+----------+---------------------------------------------------------------------------+ -| test | bool | run pre-defined helm tests helm in a chart | -+-----------------+----------+---------------------------------------------------------------------------+ -| install | object | install the chart into your Kubernetes cluster | -+-----------------+----------+---------------------------------------------------------------------------+ -| update | object | update the chart managed by the armada yaml | -+-----------------+----------+---------------------------------------------------------------------------+ -| values | object | override any default values in the charts | -+-----------------+----------+---------------------------------------------------------------------------+ -| source | object | provide a path to a ``git repo``, ``local dir``, or ``tarball url`` chart | -+-----------------+----------+---------------------------------------------------------------------------+ -| dependencies | object | reference any chart dependencies before install | -+-----------------+----------+---------------------------------------------------------------------------+ ++-----------------+----------+---------------------------------------------------------------------------------------+ +| keyword | type | action | ++=================+==========+=======================================================================================+ +| chart\_name | string | name for the chart | ++-----------------+----------+---------------------------------------------------------------------------------------+ +| release\_name | string | name of the release | ++-----------------+----------+---------------------------------------------------------------------------------------+ +| namespace | string | namespace of your chart | ++-----------------+----------+---------------------------------------------------------------------------------------+ +| wait | object | contains wait information such as (timeout, lables) | ++-----------------+----------+---------------------------------------------------------------------------------------+ +| test | bool | run pre-defined helm tests helm in a chart | ++-----------------+----------+---------------------------------------------------------------------------------------+ +| install | object | install the chart into your Kubernetes cluster | ++-----------------+----------+---------------------------------------------------------------------------------------+ +| update | object | update the chart managed by the armada yaml | ++-----------------+----------+---------------------------------------------------------------------------------------+ +| values | object | override any default values in the charts | ++-----------------+----------+---------------------------------------------------------------------------------------+ +| source | object | provide a path to a ``git repo``, ``local dir``, or ``tarball url`` chart | ++-----------------+----------+---------------------------------------------------------------------------------------+ +| dependencies | object | reference any chart dependencies before install | ++-----------------+----------+---------------------------------------------------------------------------------------+ +| timeout | int | time (in seconds) allotted for chart to deploy when 'wait' flag is set (DEPRICATED) | ++-----------------+----------+---------------------------------------------------------------------------------------+ Update - Pre or Post ^^^^^^^^^^^^^^^^^^^^ @@ -167,7 +175,10 @@ Example chart_name: blog-1 release_name: blog-1 namespace: default - timeout: 100 + wait: + timeout: 100 + labels: + component: blog install: no_hook: false upgrade: @@ -190,7 +201,8 @@ Example chart_name: blog-1 release_name: blog-1 namespace: default - timeout: 100 + wait: + timeout: 100 install: no_hook: false upgrade: @@ -213,7 +225,8 @@ Example chart_name: blog-1 release_name: blog-1 namespace: default - timeout: 100 + wait: + timeout: 100 install: no_hook: false upgrade: @@ -257,7 +270,8 @@ Example chart_name: blog-1 release_name: blog-1 namespace: default - timeout: 100 + wait: + timeout: 100 install: no_hook: false upgrade: diff --git a/examples/keystone-manifest.yaml b/examples/keystone-manifest.yaml index 0869c90d..ee1a21e4 100644 --- a/examples/keystone-manifest.yaml +++ b/examples/keystone-manifest.yaml @@ -23,7 +23,10 @@ data: chart_name: mariadb release: mariadb namespace: openstack - timeout: 3600 + wait: + timeout: 3600 + labels: + release_group: armada-mariadb install: no_hooks: false upgrade: @@ -45,7 +48,10 @@ data: chart_name: memcached release: memcached namespace: openstack - timeout: 100 + wait: + timeout: 100 + labels: + release_group: armada-memcached install: no_hooks: false upgrade: @@ -68,7 +74,10 @@ data: test: true release: keystone namespace: openstack - timeout: 100 + wait: + timeout: 100 + labels: + release_group: armada-keystone install: no_hooks: false upgrade: