fix(timeout): timeout logic error during apply
Change-Id: If317829d83e285a8e87bb8ba9878a0eee27b5c34
This commit is contained in:
parent
dc508d5012
commit
331b443e58
|
@ -269,16 +269,18 @@ class Armada(object):
|
||||||
for chart_entry in cg_charts:
|
for chart_entry in cg_charts:
|
||||||
chart = chart_entry.get('chart', {})
|
chart = chart_entry.get('chart', {})
|
||||||
namespace = chart.get('namespace')
|
namespace = chart.get('namespace')
|
||||||
namespaces_seen.add(namespace)
|
|
||||||
release = chart.get('release')
|
release = chart.get('release')
|
||||||
values = chart.get('values', {})
|
values = chart.get('values', {})
|
||||||
pre_actions = {}
|
pre_actions = {}
|
||||||
post_actions = {}
|
post_actions = {}
|
||||||
|
|
||||||
|
wait_timeout = self.timeout
|
||||||
|
wait_labels = {}
|
||||||
|
|
||||||
release_name = release_prefix(prefix, release)
|
release_name = release_prefix(prefix, release)
|
||||||
|
|
||||||
# Retrieve appropriate timeout value
|
# Retrieve appropriate timeout value
|
||||||
wait_timeout = self.timeout
|
|
||||||
if wait_timeout <= 0:
|
if wait_timeout <= 0:
|
||||||
# TODO(MarshM): chart's `data.timeout` should be deprecated
|
# TODO(MarshM): chart's `data.timeout` should be deprecated
|
||||||
chart_timeout = chart.get('timeout', 0)
|
chart_timeout = chart.get('timeout', 0)
|
||||||
|
@ -296,6 +298,9 @@ class Armada(object):
|
||||||
DEFAULT_CHART_TIMEOUT)
|
DEFAULT_CHART_TIMEOUT)
|
||||||
wait_timeout = DEFAULT_CHART_TIMEOUT
|
wait_timeout = DEFAULT_CHART_TIMEOUT
|
||||||
|
|
||||||
|
# Track namespaces + labels touched
|
||||||
|
namespaces_seen.add((namespace, tuple(wait_labels.items())))
|
||||||
|
|
||||||
# Naively take largest timeout to apply at end
|
# Naively take largest timeout to apply at end
|
||||||
# TODO(MarshM) better handling of timeout/timer
|
# TODO(MarshM) better handling of timeout/timer
|
||||||
cg_max_timeout = max(wait_timeout, cg_max_timeout)
|
cg_max_timeout = max(wait_timeout, cg_max_timeout)
|
||||||
|
@ -437,20 +442,26 @@ class Armada(object):
|
||||||
|
|
||||||
# After all Charts are applied, we should wait for the entire
|
# After all Charts are applied, we should wait for the entire
|
||||||
# ChartGroup to become healthy by looking at the namespaces seen
|
# ChartGroup to become healthy by looking at the namespaces seen
|
||||||
# TODO(MarshM): Need to restrict to only charts we processed
|
# TODO(MarshM): Need to restrict to only releases we processed
|
||||||
# TODO(MarshM): Need to determine a better timeout
|
# TODO(MarshM): Need to determine a better timeout
|
||||||
|
# (not cg_max_timeout)
|
||||||
|
if cg_max_timeout <= 0:
|
||||||
|
cg_max_timeout = DEFAULT_CHART_TIMEOUT
|
||||||
deadline = time.time() + cg_max_timeout
|
deadline = time.time() + cg_max_timeout
|
||||||
for ns in namespaces_seen:
|
for (ns, labels) in namespaces_seen:
|
||||||
|
labels_dict = dict(labels)
|
||||||
timer = int(round(deadline - time.time()))
|
timer = int(round(deadline - time.time()))
|
||||||
LOG.info('Final wait for healthy namespace (%s), '
|
LOG.info('Final wait for healthy namespace (%s), label=(%s), '
|
||||||
'timeout remaining: %ss.', ns, timer)
|
'timeout remaining: %ss.', ns, labels_dict, timer)
|
||||||
if timer <= 0:
|
if timer <= 0:
|
||||||
reason = 'Timeout expired waiting on namespace: %s' % (ns)
|
reason = ('Timeout expired waiting on namespace: %s, '
|
||||||
|
'label: %s' % (ns, labels_dict))
|
||||||
LOG.error(reason)
|
LOG.error(reason)
|
||||||
raise ArmadaTimeoutException(reason)
|
raise ArmadaTimeoutException(reason)
|
||||||
|
|
||||||
self.tiller.k8s.wait_until_ready(
|
self.tiller.k8s.wait_until_ready(
|
||||||
namespace=ns,
|
namespace=ns,
|
||||||
|
labels=labels_dict,
|
||||||
k8s_wait_attempts=self.k8s_wait_attempts,
|
k8s_wait_attempts=self.k8s_wait_attempts,
|
||||||
k8s_wait_attempt_sleep=self.k8s_wait_attempt_sleep,
|
k8s_wait_attempt_sleep=self.k8s_wait_attempt_sleep,
|
||||||
timeout=timer)
|
timeout=timer)
|
||||||
|
|
|
@ -307,7 +307,7 @@ class K8s(object):
|
||||||
return True
|
return True
|
||||||
|
|
||||||
def _wait_one_time(self, namespace, label_selector, timeout=100):
|
def _wait_one_time(self, namespace, label_selector, timeout=100):
|
||||||
LOG.debug('Starting to wait: namespace=%s, label_selector=%s, '
|
LOG.debug('Starting to wait: namespace=%s, label_selector=(%s), '
|
||||||
'timeout=%s', namespace, label_selector, timeout)
|
'timeout=%s', namespace, label_selector, timeout)
|
||||||
ready_pods = {}
|
ready_pods = {}
|
||||||
modified_pods = set()
|
modified_pods = set()
|
||||||
|
|
Loading…
Reference in New Issue