airship
/
armada
Code Issues Proposed changes

bug(wait): fixing how we wait on chart and group 

- added abilty to choose labels to wait on
- created wait object

Change-Id: Ia3b6f7bd7b6ef15779b087c613d69f4f6a7b41e9
This commit is contained in:
gardlt 2017-10-30 19:24:57 +00:00
parent be8e86351f
commit a99fc4ad6c
6 changed files with 103 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
armada/handlers/armada.py
View File

@ -213,8 +213,8 @@ class Armada(object):
release[1]) release[1])
for entry in self.config[const.KEYWORD_ARMADA][const.KEYWORD_GROUPS]: for entry in self.config[const.KEYWORD_ARMADA][const.KEYWORD_GROUPS]:
chart_wait = self.wait
chart_wait = self.wait
desc = entry.get('description', 'A Chart Group') desc = entry.get('description', 'A Chart Group')
chart_group = entry.get(const.KEYWORD_CHARTS, []) chart_group = entry.get(const.KEYWORD_CHARTS, [])
test_charts = entry.get('test_charts', False) test_charts = entry.get('test_charts', False)
@ -227,6 +227,7 @@ class Armada(object):
for gchart in chart_group: for gchart in chart_group:
chart = dotify(gchart['chart']) chart = dotify(gchart['chart'])
values = gchart.get('chart').get('values', {}) values = gchart.get('chart').get('values', {})
wait_values = gchart.get('chart').get('wait', {})
test_chart = gchart.get('chart').get('test', False) test_chart = gchart.get('chart').get('test', False)
pre_actions = {} pre_actions = {}
post_actions = {} post_actions = {}
@ -242,7 +243,7 @@ class Armada(object):
if chart_wait: if chart_wait:
if chart_timeout == DEFAULT_TIMEOUT: if chart_timeout == DEFAULT_TIMEOUT:
chart_timeout = getattr( chart_timeout = getattr(
chart, 'timeout', DEFAULT_TIMEOUT) chart, 'timeout', chart_timeout)
chartbuilder = ChartBuilder(chart) chartbuilder = ChartBuilder(chart)
protoc_chart = chartbuilder.get_helm_chart() protoc_chart = chartbuilder.get_helm_chart()
@ -302,10 +303,16 @@ class Armada(object):
timeout=chart_timeout) timeout=chart_timeout)
if chart_wait: if chart_wait:
# TODO(gardlt): after v0.7.1 depricate timeout values
if not wait_values.get('timeout', None):
wait_values['timeout'] = chart_timeout
self.tiller.k8s.wait_until_ready( self.tiller.k8s.wait_until_ready(
release=prefix_chart, release=prefix_chart,
labels=wait_values.get('labels', ''),
namespace=chart.namespace, namespace=chart.namespace,
timeout=chart_timeout) timeout=wait_values.get('timeout', DEFAULT_TIMEOUT)
)
msg['upgraded'].append(prefix_chart) msg['upgraded'].append(prefix_chart)
@ -322,10 +329,14 @@ class Armada(object):
timeout=chart_timeout) timeout=chart_timeout)
if chart_wait: if chart_wait:
if not wait_values.get('timeout', None):
wait_values['timeout'] = chart_timeout
self.tiller.k8s.wait_until_ready( self.tiller.k8s.wait_until_ready(
release=prefix_chart, release=prefix_chart,
labels=wait_values.get('labels', ''),
namespace=chart.namespace, namespace=chart.namespace,
timeout=chart_timeout) timeout=wait_values.get('timeout', 3600))
msg['installed'].append(prefix_chart) msg['installed'].append(prefix_chart)

20
armada/handlers/k8s.py
View File

@ -14,12 +14,15 @@
import re import re
import time import time
from kubernetes import client, config, watch from kubernetes import client, config, watch
from kubernetes.client.rest import ApiException from kubernetes.client.rest import ApiException
from oslo_config import cfg from oslo_config import cfg
from oslo_log import log as logging from oslo_log import log as logging
from armada.utils.release import label_selectors
LOG = logging.getLogger(__name__) LOG = logging.getLogger(__name__)
CONF = cfg.CONF CONF = cfg.CONF
@ -191,6 +194,7 @@ class K8s(object):
def wait_until_ready(self, def wait_until_ready(self,
release=None, release=None,
namespace='default', namespace='default',
labels='',
timeout=300, timeout=300,
sleep=15): sleep=15):
''' '''
@ -200,9 +204,9 @@ class K8s(object):
LOG.debug("Wait on %s for %s sec", namespace, timeout) LOG.debug("Wait on %s for %s sec", namespace, timeout)
label_selector = '' label_selector = ''
# FIXME(gardlt): this requires a label schema from OSH
if release is not None: if labels:
label_selector = 'release_group={}'.format(release) label_selector = label_selectors(labels)
valid_state = ['Succeeded', 'Running'] valid_state = ['Succeeded', 'Running']
@ -213,19 +217,27 @@ class K8s(object):
self.is_pods_ready(label_selector=label_selector, timeout=timeout) self.is_pods_ready(label_selector=label_selector, timeout=timeout)
pod_ready = [] pod_ready = []
not_ready = []
for pod in self.client.list_pod_for_all_namespaces( for pod in self.client.list_pod_for_all_namespaces(
label_selector=label_selector).items: label_selector=label_selector).items:
p_state = pod.status.phase p_state = pod.status.phase
p_name = pod.metadata.name
if p_state in valid_state: if p_state in valid_state:
pod_ready.append(True) pod_ready.append(True)
continue continue
pod_ready.append(False) pod_ready.append(False)
not_ready.append(p_name)
LOG.debug('%s', p_state) LOG.debug('%s', p_state)
if time.time() > wait_timeout or all(pod_ready): if time.time() > wait_timeout or all(pod_ready):
LOG.debug("Pod States %s", pod_ready) LOG.debug("Pod States %s", pod_ready)
break break
if time.time() > wait_timeout and not all(pod_ready):
LOG.exception(
'Failed to bring up release %s: %s', release, not_ready)
break
else: else:
LOG.debug('time: %s pod %s', wait_timeout, pod_ready) LOG.debug('time: %s pod %s', wait_timeout, pod_ready)

23
armada/handlers/tiller.py
View File

@ -32,7 +32,7 @@ from armada.const import STATUS_DEPLOYED, STATUS_FAILED
from armada.exceptions import tiller_exceptions as ex from armada.exceptions import tiller_exceptions as ex
from armada.handlers.k8s import K8s from armada.handlers.k8s import K8s
from armada.utils.release import release_prefix from armada.utils.release import release_prefix
from armada.utils.release import label_selectors
TILLER_PORT = 44134 TILLER_PORT = 44134
TILLER_VERSION = b'2.5.0' TILLER_VERSION = b'2.5.0'
@ -479,13 +479,8 @@ class Tiller(object):
''' '''
label_selector = '' label_selector = ''
if not resource_type == 'job':
label_selector = 'release_name={}'.format(release_name)
if resource_labels is not None: if resource_labels is not None:
label_selector = ",".join( label_selector = label_selectors(resource_labels)
["%s=%s" % (k, v) for k, v in resource_labels.items()])
if 'job' in resource_type: if 'job' in resource_type:
LOG.info("Deleting %s in namespace: %s", resource_name, namespace) LOG.info("Deleting %s in namespace: %s", resource_name, namespace)
@ -509,7 +504,7 @@ class Tiller(object):
resource_name, resource_type) resource_name, resource_type)
def rolling_upgrade_pod_deployment(self, name, release_name, namespace, def rolling_upgrade_pod_deployment(self, name, release_name, namespace,
labels, action_type, chart, resource_labels, action_type, chart,
disable_hooks, values): disable_hooks, values):
''' '''
update statefullsets (daemon, stateful) update statefullsets (daemon, stateful)
@ -518,11 +513,11 @@ class Tiller(object):
if action_type == 'daemonset': if action_type == 'daemonset':
LOG.info('Updating: %s', action_type) LOG.info('Updating: %s', action_type)
label_selector = 'release_name={}'.format(release_name)
if labels is not None: label_selector = ''
label_selector = ",".join(
["%s=%s" % (k, v) for k, v in labels.items()]) if resource_labels is not None:
label_selector = label_selectors(resource_labels)
get_daemonset = self.k8s.get_namespace_daemonset( get_daemonset = self.k8s.get_namespace_daemonset(
namespace=namespace, label=label_selector) namespace=namespace, label=label_selector)
@ -547,8 +542,8 @@ class Tiller(object):
namespace=namespace, template=template) namespace=namespace, template=template)
# delete pods # delete pods
self.delete_resources(release_name, name, 'pod', labels, self.delete_resources(release_name, name, 'pod',
namespace) resource_labels, namespace)
elif action_type == 'statefulset': elif action_type == 'statefulset':
pass pass

10
armada/utils/release.py
View File

@ -18,3 +18,13 @@ def release_prefix(prefix, chart):
how to attach prefix to chart how to attach prefix to chart
''' '''
return "{}-{}".format(prefix, chart) return "{}-{}".format(prefix, chart)
def label_selectors(labels):
"""
:param labels: dictionary containing k, v
:return: string of k8s labels
"""
return ",".join(
["%s=%s" % (k, v) for k, v in labels.items()])

68
docs/source/operations/guide-build-armada-yaml.rst
View File

@ -75,32 +75,40 @@ Example
armada/Chart/v1 armada/Chart/v1
--------------- ---------------
.. DANGER::
DEPRICATION: ``timeout`` key-value will be removed timeout will be defiend
under ``wait`` object.
Chart Chart
^^^^^ ^^^^^
+-----------------+----------+---------------------------------------------------------------------------+ +-----------------+----------+---------------------------------------------------------------------------------------+
| keyword | type | action | | keyword | type | action |
+=================+==========+===========================================================================+ +=================+==========+=======================================================================================+
| chart\_name | string | name for the chart | | chart\_name | string | name for the chart |
+-----------------+----------+---------------------------------------------------------------------------+ +-----------------+----------+---------------------------------------------------------------------------------------+
| release\_name | string | name of the release | | release\_name | string | name of the release |
+-----------------+----------+---------------------------------------------------------------------------+ +-----------------+----------+---------------------------------------------------------------------------------------+
| namespace | string | namespace of your chart | | namespace | string | namespace of your chart |
+-----------------+----------+---------------------------------------------------------------------------+ +-----------------+----------+---------------------------------------------------------------------------------------+
| timeout | int | time (in seconds) allotted for chart to deploy when 'wait' flag is set | | wait | object | contains wait information such as (timeout, lables) |
+-----------------+----------+---------------------------------------------------------------------------+ +-----------------+----------+---------------------------------------------------------------------------------------+
| test | bool | run pre-defined helm tests helm in a chart | | test | bool | run pre-defined helm tests helm in a chart |
+-----------------+----------+---------------------------------------------------------------------------+ +-----------------+----------+---------------------------------------------------------------------------------------+
| install | object | install the chart into your Kubernetes cluster | | install | object | install the chart into your Kubernetes cluster |
+-----------------+----------+---------------------------------------------------------------------------+ +-----------------+----------+---------------------------------------------------------------------------------------+
| update | object | update the chart managed by the armada yaml | | update | object | update the chart managed by the armada yaml |
+-----------------+----------+---------------------------------------------------------------------------+ +-----------------+----------+---------------------------------------------------------------------------------------+
| values | object | override any default values in the charts | | values | object | override any default values in the charts |
+-----------------+----------+---------------------------------------------------------------------------+ +-----------------+----------+---------------------------------------------------------------------------------------+
| source | object | provide a path to a ``git repo``, ``local dir``, or ``tarball url`` chart | | source | object | provide a path to a ``git repo``, ``local dir``, or ``tarball url`` chart |
+-----------------+----------+---------------------------------------------------------------------------+ +-----------------+----------+---------------------------------------------------------------------------------------+
| dependencies | object | reference any chart dependencies before install | | dependencies | object | reference any chart dependencies before install |
+-----------------+----------+---------------------------------------------------------------------------+ +-----------------+----------+---------------------------------------------------------------------------------------+
| timeout | int | time (in seconds) allotted for chart to deploy when 'wait' flag is set (DEPRICATED) |
+-----------------+----------+---------------------------------------------------------------------------------------+
Update - Pre or Post Update - Pre or Post
^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^
@ -167,7 +175,10 @@ Example
chart_name: blog-1 chart_name: blog-1
release_name: blog-1 release_name: blog-1
namespace: default namespace: default
timeout: 100 wait:
timeout: 100
labels:
component: blog
install: install:
no_hook: false no_hook: false
upgrade: upgrade:
@ -190,7 +201,8 @@ Example
chart_name: blog-1 chart_name: blog-1
release_name: blog-1 release_name: blog-1
namespace: default namespace: default
timeout: 100 wait:
timeout: 100
install: install:
no_hook: false no_hook: false
upgrade: upgrade:
@ -213,7 +225,8 @@ Example
chart_name: blog-1 chart_name: blog-1
release_name: blog-1 release_name: blog-1
namespace: default namespace: default
timeout: 100 wait:
timeout: 100
install: install:
no_hook: false no_hook: false
upgrade: upgrade:
@ -257,7 +270,8 @@ Example
chart_name: blog-1 chart_name: blog-1
release_name: blog-1 release_name: blog-1
namespace: default namespace: default
timeout: 100 wait:
timeout: 100
install: install:
no_hook: false no_hook: false
upgrade: upgrade:

15
examples/keystone-manifest.yaml
View File

@ -23,7 +23,10 @@ data:
chart_name: mariadb chart_name: mariadb
release: mariadb release: mariadb
namespace: openstack namespace: openstack
timeout: 3600 wait:
timeout: 3600
labels:
release_group: armada-mariadb
install: install:
no_hooks: false no_hooks: false
upgrade: upgrade:
@ -45,7 +48,10 @@ data:
chart_name: memcached chart_name: memcached
release: memcached release: memcached
namespace: openstack namespace: openstack
timeout: 100 wait:
timeout: 100
labels:
release_group: armada-memcached
install: install:
no_hooks: false no_hooks: false
upgrade: upgrade:
@ -68,7 +74,10 @@ data:
test: true test: true
release: keystone release: keystone
namespace: openstack namespace: openstack
timeout: 100 wait:
timeout: 100
labels:
release_group: armada-keystone
install: install:
no_hooks: false no_hooks: false
upgrade: upgrade: