summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSean Eagan <sean.eagan@att.com>2019-01-08 15:06:36 -0600
committerSean Eagan <sean.eagan@att.com>2019-01-28 13:19:09 -0600
commitc31a961bf1f66d570cb7fe6f3f1a03b07ecd8734 (patch)
treea000864d8a361412aef525152daa41499ecea5e6
parent52f29ddf732f50fe457c518587c0fc115c8eb188 (diff)
Automate deletion of test pods
When running helm tests for a chart release multiple times in a site, if the previous test pod is not deleted, then the test pod creation can fail due to a name conflict. Armada/helm support immediate test pod cleanup, but using this means that upon test failure, the test pod logs will not be available for debugging purposes. Due to this, the recommended approach for deleting test pods in Armada has been using `upgrade.pre.delete` actions. So chart authors can accomplish test pod deletion using this feature, however, it often takes awhile, usually not until they test upgrading the chart for chart authors to realize that this is necessary and to get it implemented. This patchset automates deletion of test pods directly before running tests by using the `wait.labels` field in the chart doc when they exist to find all pods in the release and then using their annotations to determine if they are test pods and deleting them if so. A later patchset is planned to implement defaulting of the wait labels when they are not defined. Change-Id: I2092f448acb88b5ade3b31b397f9c874c0061668
Notes
Notes (review): Code-Review+1: Nishant Kumar <nishant.e.kumar@ericsson.com> Code-Review+2: Bryan Strassner <strassner.bryan@gmail.com> Code-Review+2: Scott Hussey <sthussey@att.com> Workflow+1: Scott Hussey <sthussey@att.com> Verified+2: Zuul Submitted-by: Zuul Submitted-at: Tue, 29 Jan 2019 17:29:12 +0000 Reviewed-on: https://review.openstack.org/629313 Project: openstack/airship-armada Branch: refs/heads/master
-rw-r--r--armada/api/controller/test.py7
-rw-r--r--armada/cli/test.py11
-rw-r--r--armada/const.py2
-rw-r--r--armada/handlers/chart_deploy.py5
-rw-r--r--armada/handlers/test.py65
-rw-r--r--armada/handlers/tiller.py4
-rw-r--r--armada/handlers/wait.py27
-rw-r--r--armada/tests/unit/handlers/test_armada.py7
-rw-r--r--armada/tests/unit/handlers/test_test.py115
-rw-r--r--armada/tests/unit/handlers/test_tiller.py12
-rw-r--r--armada/utils/helm.py40
-rw-r--r--armada/utils/release.py4
-rw-r--r--doc/source/operations/guide-build-armada-yaml.rst11
13 files changed, 202 insertions, 108 deletions
diff --git a/armada/api/controller/test.py b/armada/api/controller/test.py
index d1804c5..c2467cc 100644
--- a/armada/api/controller/test.py
+++ b/armada/api/controller/test.py
@@ -39,7 +39,7 @@ class TestReleasesReleaseNameController(api.BaseResource):
39 with self.get_tiller(req, resp) as tiller: 39 with self.get_tiller(req, resp) as tiller:
40 cleanup = req.get_param_as_bool('cleanup') 40 cleanup = req.get_param_as_bool('cleanup')
41 41
42 test_handler = Test(release, tiller, cleanup=cleanup) 42 test_handler = Test({}, release, tiller, cleanup=cleanup)
43 success = test_handler.test_release_for_success() 43 success = test_handler.test_release_for_success()
44 44
45 if success: 45 if success:
@@ -136,15 +136,14 @@ class TestReleasesManifestController(api.BaseResource):
136 cleanup = req.get_param_as_bool('cleanup') 136 cleanup = req.get_param_as_bool('cleanup')
137 enable_all = req.get_param_as_bool('enable_all') 137 enable_all = req.get_param_as_bool('enable_all')
138 cg_test_charts = group.get('test_charts') 138 cg_test_charts = group.get('test_charts')
139 test_values = chart.get('test', {})
140 139
141 test_handler = Test( 140 test_handler = Test(
141 chart,
142 release_name, 142 release_name,
143 tiller, 143 tiller,
144 cg_test_charts=cg_test_charts, 144 cg_test_charts=cg_test_charts,
145 cleanup=cleanup, 145 cleanup=cleanup,
146 enable_all=enable_all, 146 enable_all=enable_all)
147 test_values=test_values)
148 147
149 if test_handler.test_enabled: 148 if test_handler.test_enabled:
150 success = test_handler.test_release_for_success() 149 success = test_handler.test_release_for_success()
diff --git a/armada/cli/test.py b/armada/cli/test.py
index 998426a..14efb83 100644
--- a/armada/cli/test.py
+++ b/armada/cli/test.py
@@ -124,7 +124,10 @@ class TestChartManifest(CliAction):
124 124
125 if self.release: 125 if self.release:
126 if not self.ctx.obj.get('api', False): 126 if not self.ctx.obj.get('api', False):
127 test_handler = Test(self.release, tiller, cleanup=self.cleanup) 127 test_handler = Test({},
128 self.release,
129 tiller,
130 cleanup=self.cleanup)
128 test_handler.test_release_for_success() 131 test_handler.test_release_for_success()
129 else: 132 else:
130 client = self.ctx.obj.get('CLIENT') 133 client = self.ctx.obj.get('CLIENT')
@@ -156,14 +159,12 @@ class TestChartManifest(CliAction):
156 release_name = release_prefixer( 159 release_name = release_prefixer(
157 prefix, chart.get('release')) 160 prefix, chart.get('release'))
158 if release_name in known_release_names: 161 if release_name in known_release_names:
159 test_values = chart.get('test', {})
160
161 test_handler = Test( 162 test_handler = Test(
163 chart,
162 release_name, 164 release_name,
163 tiller, 165 tiller,
164 cleanup=self.cleanup, 166 cleanup=self.cleanup,
165 enable_all=self.enable_all, 167 enable_all=self.enable_all)
166 test_values=test_values)
167 168
168 if test_handler.test_enabled: 169 if test_handler.test_enabled:
169 test_handler.test_release_for_success() 170 test_handler.test_release_for_success()
diff --git a/armada/const.py b/armada/const.py
index 30d3149..e253881 100644
--- a/armada/const.py
+++ b/armada/const.py
@@ -27,8 +27,6 @@ DEFAULT_CHART_TIMEOUT = 900
27 27
28# Tiller 28# Tiller
29DEFAULT_TILLER_TIMEOUT = 300 29DEFAULT_TILLER_TIMEOUT = 300
30HELM_HOOK_ANNOTATION = 'helm.sh/hook'
31HELM_TEST_HOOKS = ['test-success', 'test-failure']
32STATUS_UNKNOWN = 'UNKNOWN' 30STATUS_UNKNOWN = 'UNKNOWN'
33STATUS_DEPLOYED = 'DEPLOYED' 31STATUS_DEPLOYED = 'DEPLOYED'
34STATUS_DELETED = 'DELETED' 32STATUS_DELETED = 'DELETED'
diff --git a/armada/handlers/chart_deploy.py b/armada/handlers/chart_deploy.py
index 76a3b12..b160f16 100644
--- a/armada/handlers/chart_deploy.py
+++ b/armada/handlers/chart_deploy.py
@@ -232,12 +232,11 @@ class ChartDeploy(object):
232 just_deployed = ('install' in result) or ('upgrade' in result) 232 just_deployed = ('install' in result) or ('upgrade' in result)
233 last_test_passed = old_release and r.get_last_test_result(old_release) 233 last_test_passed = old_release and r.get_last_test_result(old_release)
234 234
235 test_values = chart.get('test')
236 test_handler = Test( 235 test_handler = Test(
236 chart,
237 release_name, 237 release_name,
238 self.tiller, 238 self.tiller,
239 cg_test_charts=cg_test_all_charts, 239 cg_test_charts=cg_test_all_charts)
240 test_values=test_values)
241 240
242 run_test = test_handler.test_enabled and (just_deployed or 241 run_test = test_handler.test_enabled and (just_deployed or
243 not last_test_passed) 242 not last_test_passed)
diff --git a/armada/handlers/test.py b/armada/handlers/test.py
index aa7fe03..94c9479 100644
--- a/armada/handlers/test.py
+++ b/armada/handlers/test.py
@@ -16,10 +16,9 @@ from oslo_log import log as logging
16 16
17from armada import const 17from armada import const
18 18
19TESTRUN_STATUS_UNKNOWN = 0 19from armada.handlers.wait import get_wait_labels
20TESTRUN_STATUS_SUCCESS = 1 20from armada.utils.release import label_selectors
21TESTRUN_STATUS_FAILURE = 2 21from armada.utils.helm import get_test_suite_run_success, is_test_pod
22TESTRUN_STATUS_RUNNING = 3
23 22
24LOG = logging.getLogger(__name__) 23LOG = logging.getLogger(__name__)
25 24
@@ -27,33 +26,37 @@ LOG = logging.getLogger(__name__)
27class Test(object): 26class Test(object):
28 27
29 def __init__(self, 28 def __init__(self,
29 chart,
30 release_name, 30 release_name,
31 tiller, 31 tiller,
32 cg_test_charts=None, 32 cg_test_charts=None,
33 cleanup=None, 33 cleanup=None,
34 enable_all=False, 34 enable_all=False):
35 test_values=None):
36 """Initialize a test handler to run Helm tests corresponding to a 35 """Initialize a test handler to run Helm tests corresponding to a
37 release. 36 release.
38 37
38 :param chart: The armada chart document
39 :param release_name: Name of a Helm release 39 :param release_name: Name of a Helm release
40 :param tiller: Tiller object 40 :param tiller: Tiller object
41 :param cg_test_charts: Chart group `test_charts` key 41 :param cg_test_charts: Chart group `test_charts` key
42 :param cleanup: Triggers cleanup; overrides `test.options.cleanup` 42 :param cleanup: Triggers cleanup; overrides `test.options.cleanup`
43 :param enable_all: Run tests regardless of the value of `test.enabled` 43 :param enable_all: Run tests regardless of the value of `test.enabled`
44 :param test_values: Test values retrieved from a chart's `test` key
45 44
45 :type chart: dict
46 :type release_name: str 46 :type release_name: str
47 :type tiller: Tiller object 47 :type tiller: Tiller object
48 :type cg_test_charts: bool 48 :type cg_test_charts: bool
49 :type cleanup: bool 49 :type cleanup: bool
50 :type enable_all: bool 50 :type enable_all: bool
51 :type test_values: dict or bool (deprecated)
52 """ 51 """
53 52
53 self.chart = chart
54 self.release_name = release_name 54 self.release_name = release_name
55 self.tiller = tiller 55 self.tiller = tiller
56 self.cleanup = cleanup 56 self.cleanup = cleanup
57 self.k8s_timeout = const.DEFAULT_K8S_TIMEOUT
58
59 test_values = self.chart.get('test', None)
57 60
58 # NOTE(drewwalters96): Support the chart_group `test_charts` key until 61 # NOTE(drewwalters96): Support the chart_group `test_charts` key until
59 # its deprecation period ends. The `test.enabled`, `enable_all` flag, 62 # its deprecation period ends. The `test.enabled`, `enable_all` flag,
@@ -106,10 +109,16 @@ class Test(object):
106 """ 109 """
107 LOG.info('RUNNING: %s tests', self.release_name) 110 LOG.info('RUNNING: %s tests', self.release_name)
108 111
112 try:
113 self.delete_test_pods()
114 except Exception:
115 LOG.exception("Exception when deleting test pods for release: %s",
116 self.release_name)
117
109 test_suite_run = self.tiller.test_release( 118 test_suite_run = self.tiller.test_release(
110 self.release_name, timeout=timeout, cleanup=self.cleanup) 119 self.release_name, timeout=timeout, cleanup=self.cleanup)
111 120
112 success = self.get_test_suite_run_success(test_suite_run) 121 success = get_test_suite_run_success(test_suite_run)
113 if success: 122 if success:
114 LOG.info('PASSED: %s', self.release_name) 123 LOG.info('PASSED: %s', self.release_name)
115 else: 124 else:
@@ -117,7 +126,37 @@ class Test(object):
117 126
118 return success 127 return success
119 128
120 @classmethod 129 def delete_test_pods(self):
121 def get_test_suite_run_success(self, test_suite_run): 130 """Deletes any existing test pods for the release, as identified by the
122 return all( 131 wait labels for the chart, to avoid test pod name conflicts when
123 r.status == TESTRUN_STATUS_SUCCESS for r in test_suite_run.results) 132 creating the new test pod as well as just for general cleanup since
133 the new test pod should supercede it.
134 """
135 labels = get_wait_labels(self.chart)
136
137 # Guard against labels being left empty, so we don't delete other
138 # chart's test pods.
139 if labels:
140 label_selector = label_selectors(labels)
141
142 namespace = self.chart['namespace']
143
144 list_args = {
145 'namespace': namespace,
146 'label_selector': label_selector,
147 'timeout_seconds': self.k8s_timeout
148 }
149
150 pod_list = self.tiller.k8s.client.list_namespaced_pod(**list_args)
151 test_pods = (pod for pod in pod_list.items if is_test_pod(pod))
152
153 if test_pods:
154 LOG.info(
155 'Found existing test pods for release with '
156 'namespace=%s, labels=(%s)', namespace, label_selector)
157
158 for test_pod in test_pods:
159 pod_name = test_pod.metadata.name
160 LOG.info('Deleting existing test pod: %s', pod_name)
161 self.tiller.k8s.delete_pod_action(
162 pod_name, namespace, timeout=self.k8s_timeout)
diff --git a/armada/handlers/tiller.py b/armada/handlers/tiller.py
index d95ea3c..29c2b6c 100644
--- a/armada/handlers/tiller.py
+++ b/armada/handlers/tiller.py
@@ -32,7 +32,7 @@ from oslo_log import log as logging
32from armada import const 32from armada import const
33from armada.exceptions import tiller_exceptions as ex 33from armada.exceptions import tiller_exceptions as ex
34from armada.handlers.k8s import K8s 34from armada.handlers.k8s import K8s
35from armada.handlers import test 35from armada.utils import helm
36from armada.utils.release import label_selectors, get_release_status 36from armada.utils.release import label_selectors, get_release_status
37 37
38TILLER_VERSION = b'2.12.1' 38TILLER_VERSION = b'2.12.1'
@@ -502,7 +502,7 @@ class Tiller(object):
502 502
503 failed = 0 503 failed = 0
504 for test_message in test_message_stream: 504 for test_message in test_message_stream:
505 if test_message.status == test.TESTRUN_STATUS_FAILURE: 505 if test_message.status == helm.TESTRUN_STATUS_FAILURE:
506 failed += 1 506 failed += 1
507 LOG.info(test_message.msg) 507 LOG.info(test_message.msg)
508 if failed: 508 if failed:
diff --git a/armada/handlers/wait.py b/armada/handlers/wait.py
index 7a27945..0c80438 100644
--- a/armada/handlers/wait.py
+++ b/armada/handlers/wait.py
@@ -21,6 +21,7 @@ import time
21from oslo_log import log as logging 21from oslo_log import log as logging
22 22
23from armada import const 23from armada import const
24from armada.utils.helm import is_test_pod
24from armada.utils.release import label_selectors 25from armada.utils.release import label_selectors
25from armada.exceptions import k8s_exceptions 26from armada.exceptions import k8s_exceptions
26from armada.exceptions import manifest_exceptions 27from armada.exceptions import manifest_exceptions
@@ -32,6 +33,11 @@ LOG = logging.getLogger(__name__)
32ROLLING_UPDATE_STRATEGY_TYPE = 'RollingUpdate' 33ROLLING_UPDATE_STRATEGY_TYPE = 'RollingUpdate'
33 34
34 35
36def get_wait_labels(chart):
37 wait_config = chart.get('wait', {})
38 return wait_config.get('labels', {})
39
40
35# TODO: Validate this object up front in armada validate flow. 41# TODO: Validate this object up front in armada validate flow.
36class ChartWait(): 42class ChartWait():
37 43
@@ -46,7 +52,7 @@ class ChartWait():
46 self.k8s_wait_attempt_sleep = max(k8s_wait_attempt_sleep, 1) 52 self.k8s_wait_attempt_sleep = max(k8s_wait_attempt_sleep, 1)
47 53
48 resources = self.wait_config.get('resources') 54 resources = self.wait_config.get('resources')
49 labels = self.wait_config.get('labels', {}) 55 labels = get_wait_labels(self.chart)
50 56
51 if resources is not None: 57 if resources is not None:
52 waits = [] 58 waits = []
@@ -349,25 +355,16 @@ class PodWait(ResourceWait):
349 355
350 def include_resource(self, resource): 356 def include_resource(self, resource):
351 pod = resource 357 pod = resource
352 annotations = pod.metadata.annotations 358 include = not is_test_pod(pod)
353
354 # Retrieve pod's Helm test hooks
355 test_hooks = None
356 if annotations:
357 hook_string = annotations.get(const.HELM_HOOK_ANNOTATION)
358 if hook_string:
359 hooks = hook_string.split(',')
360 test_hooks = [h for h in hooks if h in const.HELM_TEST_HOOKS]
361 359
362 # NOTE(drewwalters96): Test pods may cause wait operations to fail 360 # NOTE(drewwalters96): Test pods may cause wait operations to fail
363 # when old resources remain from previous upgrades/tests. Indicate that 361 # when old resources remain from previous upgrades/tests. Indicate that
364 # test pods should not be included in wait operations. 362 # test pods should not be included in wait operations.
365 if test_hooks: 363 if not include:
366 LOG.debug('Pod %s will be skipped during wait operations.', 364 LOG.debug('Test pod %s will be skipped during wait operations.',
367 pod.metadata.name) 365 pod.metadata.name)
368 return False 366
369 else: 367 return include
370 return True
371 368
372 def is_resource_ready(self, resource): 369 def is_resource_ready(self, resource):
373 pod = resource 370 pod = resource
diff --git a/armada/tests/unit/handlers/test_armada.py b/armada/tests/unit/handlers/test_armada.py
index d1a35dc..110e78c 100644
--- a/armada/tests/unit/handlers/test_armada.py
+++ b/armada/tests/unit/handlers/test_armada.py
@@ -17,7 +17,7 @@ import yaml
17 17
18from armada import const 18from armada import const
19from armada.handlers import armada 19from armada.handlers import armada
20from armada.handlers.test import TESTRUN_STATUS_SUCCESS, TESTRUN_STATUS_FAILURE 20from armada.utils.helm import TESTRUN_STATUS_SUCCESS, TESTRUN_STATUS_FAILURE
21from armada.tests.unit import base 21from armada.tests.unit import base
22from armada.tests.test_utils import AttrDict, makeMockThreadSafe 22from armada.tests.test_utils import AttrDict, makeMockThreadSafe
23from armada.utils.release import release_prefixer, get_release_status 23from armada.utils.release import release_prefixer, get_release_status
@@ -459,13 +459,12 @@ class ArmadaHandlerTestCase(base.ArmadaTestCase):
459 wait=native_wait_enabled, 459 wait=native_wait_enabled,
460 timeout=mock.ANY)) 460 timeout=mock.ANY))
461 461
462 test_chart_override = chart.get('test')
463 expected_test_constructor_calls.append( 462 expected_test_constructor_calls.append(
464 mock.call( 463 mock.call(
464 chart,
465 release_name, 465 release_name,
466 m_tiller, 466 m_tiller,
467 cg_test_charts=cg_test_all_charts, 467 cg_test_charts=cg_test_all_charts))
468 test_values=test_chart_override))
469 468
470 any_order = not chart_group['sequenced'] 469 any_order = not chart_group['sequenced']
471 # Verify that at least 1 release is either installed or updated. 470 # Verify that at least 1 release is either installed or updated.
diff --git a/armada/tests/unit/handlers/test_test.py b/armada/tests/unit/handlers/test_test.py
index 6459ed3..9bf4544 100644
--- a/armada/tests/unit/handlers/test_test.py
+++ b/armada/tests/unit/handlers/test_test.py
@@ -18,6 +18,7 @@ from armada.handlers import test
18from armada.handlers import tiller 18from armada.handlers import tiller
19from armada.tests.unit import base 19from armada.tests.unit import base
20from armada.tests.test_utils import AttrDict 20from armada.tests.test_utils import AttrDict
21from armada.utils import helm
21 22
22 23
23class TestHandlerTestCase(base.ArmadaTestCase): 24class TestHandlerTestCase(base.ArmadaTestCase):
@@ -33,7 +34,7 @@ class TestHandlerTestCase(base.ArmadaTestCase):
33 tiller_obj.test_release.return_value = AttrDict( 34 tiller_obj.test_release.return_value = AttrDict(
34 **{'results': results}) 35 **{'results': results})
35 36
36 test_handler = test.Test(release, tiller_obj) 37 test_handler = test.Test({}, release, tiller_obj)
37 success = test_handler.test_release_for_success() 38 success = test_handler.test_release_for_success()
38 39
39 self.assertEqual(expected_success, success) 40 self.assertEqual(expected_success, success)
@@ -45,24 +46,24 @@ class TestHandlerTestCase(base.ArmadaTestCase):
45 46
46 def test_unknown(self): 47 def test_unknown(self):
47 self._test_test_release_for_success(False, [ 48 self._test_test_release_for_success(False, [
48 AttrDict(**{'status': test.TESTRUN_STATUS_SUCCESS}), 49 AttrDict(**{'status': helm.TESTRUN_STATUS_SUCCESS}),
49 AttrDict(**{'status': test.TESTRUN_STATUS_UNKNOWN}) 50 AttrDict(**{'status': helm.TESTRUN_STATUS_UNKNOWN})
50 ]) 51 ])
51 52
52 def test_success(self): 53 def test_success(self):
53 self._test_test_release_for_success( 54 self._test_test_release_for_success(
54 True, [AttrDict(**{'status': test.TESTRUN_STATUS_SUCCESS})]) 55 True, [AttrDict(**{'status': helm.TESTRUN_STATUS_SUCCESS})])
55 56
56 def test_failure(self): 57 def test_failure(self):
57 self._test_test_release_for_success(False, [ 58 self._test_test_release_for_success(False, [
58 AttrDict(**{'status': test.TESTRUN_STATUS_SUCCESS}), 59 AttrDict(**{'status': helm.TESTRUN_STATUS_SUCCESS}),
59 AttrDict(**{'status': test.TESTRUN_STATUS_FAILURE}) 60 AttrDict(**{'status': helm.TESTRUN_STATUS_FAILURE})
60 ]) 61 ])
61 62
62 def test_running(self): 63 def test_running(self):
63 self._test_test_release_for_success(False, [ 64 self._test_test_release_for_success(False, [
64 AttrDict(**{'status': test.TESTRUN_STATUS_SUCCESS}), 65 AttrDict(**{'status': helm.TESTRUN_STATUS_SUCCESS}),
65 AttrDict(**{'status': test.TESTRUN_STATUS_RUNNING}) 66 AttrDict(**{'status': helm.TESTRUN_STATUS_RUNNING})
66 ]) 67 ])
67 68
68 def test_cg_disabled(self): 69 def test_cg_disabled(self):
@@ -70,7 +71,10 @@ class TestHandlerTestCase(base.ArmadaTestCase):
70 tests. 71 tests.
71 """ 72 """
72 test_handler = test.Test( 73 test_handler = test.Test(
73 release_name='release', tiller=mock.Mock(), cg_test_charts=False) 74 chart={},
75 release_name='release',
76 tiller=mock.Mock(),
77 cg_test_charts=False)
74 78
75 assert test_handler.test_enabled is False 79 assert test_handler.test_enabled is False
76 80
@@ -79,10 +83,10 @@ class TestHandlerTestCase(base.ArmadaTestCase):
79 tests and the deprecated, boolean `test` key is enabled. 83 tests and the deprecated, boolean `test` key is enabled.
80 """ 84 """
81 test_handler = test.Test( 85 test_handler = test.Test(
86 chart={'test': True},
82 release_name='release', 87 release_name='release',
83 tiller=mock.Mock(), 88 tiller=mock.Mock(),
84 cg_test_charts=False, 89 cg_test_charts=False)
85 test_values=True)
86 90
87 assert test_handler.test_enabled is True 91 assert test_handler.test_enabled is True
88 92
@@ -90,13 +94,13 @@ class TestHandlerTestCase(base.ArmadaTestCase):
90 """Test that tests are enabled when a chart group disables all 94 """Test that tests are enabled when a chart group disables all
91 tests and the `test.enabled` key is False. 95 tests and the `test.enabled` key is False.
92 """ 96 """
93 test_values = {'enabled': True}
94
95 test_handler = test.Test( 97 test_handler = test.Test(
98 chart={'test': {
99 'enabled': True
100 }},
96 release_name='release', 101 release_name='release',
97 tiller=mock.Mock(), 102 tiller=mock.Mock(),
98 cg_test_charts=False, 103 cg_test_charts=False)
99 test_values=test_values)
100 104
101 assert test_handler.test_enabled is True 105 assert test_handler.test_enabled is True
102 106
@@ -105,10 +109,10 @@ class TestHandlerTestCase(base.ArmadaTestCase):
105 tests and the deprecated, boolean `test` key is disabled. 109 tests and the deprecated, boolean `test` key is disabled.
106 """ 110 """
107 test_handler = test.Test( 111 test_handler = test.Test(
112 chart={'test': False},
108 release_name='release', 113 release_name='release',
109 tiller=mock.Mock(), 114 tiller=mock.Mock(),
110 cg_test_charts=True, 115 cg_test_charts=True)
111 test_values=False)
112 116
113 assert test_handler.test_enabled is False 117 assert test_handler.test_enabled is False
114 118
@@ -116,13 +120,13 @@ class TestHandlerTestCase(base.ArmadaTestCase):
116 """Test that tests are disabled when a chart group enables all 120 """Test that tests are disabled when a chart group enables all
117 tests and the deprecated, boolean `test` key is disabled. 121 tests and the deprecated, boolean `test` key is disabled.
118 """ 122 """
119 test_values = {'enabled': False}
120
121 test_handler = test.Test( 123 test_handler = test.Test(
124 chart={'test': {
125 'enabled': False
126 }},
122 release_name='release', 127 release_name='release',
123 tiller=mock.Mock(), 128 tiller=mock.Mock(),
124 cg_test_charts=True, 129 cg_test_charts=True)
125 test_values=test_values)
126 130
127 assert test_handler.test_enabled is False 131 assert test_handler.test_enabled is False
128 132
@@ -131,6 +135,7 @@ class TestHandlerTestCase(base.ArmadaTestCase):
131 True and the chart group `test_enabled` key is disabled. 135 True and the chart group `test_enabled` key is disabled.
132 """ 136 """
133 test_handler = test.Test( 137 test_handler = test.Test(
138 chart={},
134 release_name='release', 139 release_name='release',
135 tiller=mock.Mock(), 140 tiller=mock.Mock(),
136 cg_test_charts=False, 141 cg_test_charts=False,
@@ -143,10 +148,10 @@ class TestHandlerTestCase(base.ArmadaTestCase):
143 True and the deprecated, boolean `test` key is disabled. 148 True and the deprecated, boolean `test` key is disabled.
144 """ 149 """
145 test_handler = test.Test( 150 test_handler = test.Test(
151 chart={'test': True},
146 release_name='release', 152 release_name='release',
147 tiller=mock.Mock(), 153 tiller=mock.Mock(),
148 enable_all=True, 154 enable_all=True)
149 test_values=False)
150 155
151 assert test_handler.test_enabled is True 156 assert test_handler.test_enabled is True
152 157
@@ -154,13 +159,13 @@ class TestHandlerTestCase(base.ArmadaTestCase):
154 """Test that tests are enabled when the `enable_all` parameter is 159 """Test that tests are enabled when the `enable_all` parameter is
155 True and the `test.enabled` key is False. 160 True and the `test.enabled` key is False.
156 """ 161 """
157 test_values = {'enabled': False}
158
159 test_handler = test.Test( 162 test_handler = test.Test(
163 chart={'test': {
164 'enabled': False
165 }},
160 release_name='release', 166 release_name='release',
161 tiller=mock.Mock(), 167 tiller=mock.Mock(),
162 enable_all=True, 168 enable_all=True)
163 test_values=test_values)
164 169
165 assert test_handler.test_enabled is True 170 assert test_handler.test_enabled is True
166 171
@@ -169,16 +174,16 @@ class TestHandlerTestCase(base.ArmadaTestCase):
169 for a chart's test key. 174 for a chart's test key.
170 """ 175 """
171 test_handler = test.Test( 176 test_handler = test.Test(
172 release_name='release', tiller=mock.Mock(), test_values=True) 177 chart={'test': False}, release_name='release', tiller=mock.Mock())
173 178
174 assert test_handler.test_enabled 179 assert not test_handler.test_enabled
175 180
176 def test_deprecated_test_key_true(self): 181 def test_deprecated_test_key_true(self):
177 """Test that cleanup is enabled by default when tests are enabled using 182 """Test that cleanup is enabled by default when tests are enabled using
178 the deprecated, boolean value for a chart's `test` key. 183 the deprecated, boolean value for a chart's `test` key.
179 """ 184 """
180 test_handler = test.Test( 185 test_handler = test.Test(
181 release_name='release', tiller=mock.Mock(), test_values=True) 186 chart={'test': True}, release_name='release', tiller=mock.Mock())
182 187
183 assert test_handler.test_enabled is True 188 assert test_handler.test_enabled is True
184 assert test_handler.cleanup is True 189 assert test_handler.cleanup is True
@@ -187,11 +192,12 @@ class TestHandlerTestCase(base.ArmadaTestCase):
187 """Test that tests are disabled by a chart's values using the 192 """Test that tests are disabled by a chart's values using the
188 `test.enabled` path. 193 `test.enabled` path.
189 """ 194 """
190 test_values = {'enabled': False}
191 test_handler = test.Test( 195 test_handler = test.Test(
196 chart={'test': {
197 'enabled': False
198 }},
192 release_name='release', 199 release_name='release',
193 tiller=mock.Mock(), 200 tiller=mock.Mock())
194 test_values=test_values)
195 201
196 assert test_handler.test_enabled is False 202 assert test_handler.test_enabled is False
197 203
@@ -199,11 +205,12 @@ class TestHandlerTestCase(base.ArmadaTestCase):
199 """Test that cleanup is disabled (by default) when tests are enabled by 205 """Test that cleanup is disabled (by default) when tests are enabled by
200 a chart's values using the `test.enabled` path. 206 a chart's values using the `test.enabled` path.
201 """ 207 """
202 test_values = {'enabled': True}
203 test_handler = test.Test( 208 test_handler = test.Test(
209 chart={'test': {
210 'enabled': True
211 }},
204 release_name='release', 212 release_name='release',
205 tiller=mock.Mock(), 213 tiller=mock.Mock())
206 test_values=test_values)
207 214
208 assert test_handler.test_enabled is True 215 assert test_handler.test_enabled is True
209 assert test_handler.cleanup is False 216 assert test_handler.cleanup is False
@@ -212,12 +219,15 @@ class TestHandlerTestCase(base.ArmadaTestCase):
212 """Test that the test handler uses the values provided by a chart's 219 """Test that the test handler uses the values provided by a chart's
213 `test` key. 220 `test` key.
214 """ 221 """
215 test_values = {'enabled': True, 'options': {'cleanup': True}}
216
217 test_handler = test.Test( 222 test_handler = test.Test(
223 chart={'test': {
224 'enabled': True,
225 'options': {
226 'cleanup': True
227 }
228 }},
218 release_name='release', 229 release_name='release',
219 tiller=mock.Mock(), 230 tiller=mock.Mock())
220 test_values=test_values)
221 231
222 assert test_handler.test_enabled is True 232 assert test_handler.test_enabled is True
223 assert test_handler.cleanup is True 233 assert test_handler.cleanup is True
@@ -226,12 +236,15 @@ class TestHandlerTestCase(base.ArmadaTestCase):
226 """Test that the test handler uses the values provided by a chart's 236 """Test that the test handler uses the values provided by a chart's
227 `test` key. 237 `test` key.
228 """ 238 """
229 test_values = {'enabled': True, 'options': {'cleanup': False}}
230
231 test_handler = test.Test( 239 test_handler = test.Test(
240 chart={'test': {
241 'enabled': True,
242 'options': {
243 'cleanup': False
244 }
245 }},
232 release_name='release', 246 release_name='release',
233 tiller=mock.Mock(), 247 tiller=mock.Mock())
234 test_values=test_values)
235 248
236 assert test_handler.test_enabled is True 249 assert test_handler.test_enabled is True
237 assert test_handler.cleanup is False 250 assert test_handler.cleanup is False
@@ -240,7 +253,8 @@ class TestHandlerTestCase(base.ArmadaTestCase):
240 """Test that the default values are enforced when no chart `test` 253 """Test that the default values are enforced when no chart `test`
241 values are provided (i.e. tests are enabled and cleanup is disabled). 254 values are provided (i.e. tests are enabled and cleanup is disabled).
242 """ 255 """
243 test_handler = test.Test(release_name='release', tiller=mock.Mock()) 256 test_handler = test.Test(
257 chart={}, release_name='release', tiller=mock.Mock())
244 258
245 assert test_handler.test_enabled is True 259 assert test_handler.test_enabled is True
246 assert test_handler.cleanup is False 260 assert test_handler.cleanup is False
@@ -249,13 +263,16 @@ class TestHandlerTestCase(base.ArmadaTestCase):
249 """Test that a cleanup value passed to the Test handler (i.e. from the 263 """Test that a cleanup value passed to the Test handler (i.e. from the
250 API/CLI) takes precedence over a chart's `test.cleanup` value. 264 API/CLI) takes precedence over a chart's `test.cleanup` value.
251 """ 265 """
252 test_values = {'enabled': True, 'options': {'cleanup': False}}
253
254 test_handler = test.Test( 266 test_handler = test.Test(
267 chart={'test': {
268 'enabled': True,
269 'options': {
270 'cleanup': False
271 }
272 }},
255 release_name='release', 273 release_name='release',
256 tiller=mock.Mock(), 274 tiller=mock.Mock(),
257 cleanup=True, 275 cleanup=True)
258 test_values=test_values)
259 276
260 assert test_handler.test_enabled is True 277 assert test_handler.test_enabled is True
261 assert test_handler.cleanup is True 278 assert test_handler.cleanup is True
diff --git a/armada/tests/unit/handlers/test_tiller.py b/armada/tests/unit/handlers/test_tiller.py
index c54cef5..d17e559 100644
--- a/armada/tests/unit/handlers/test_tiller.py
+++ b/armada/tests/unit/handlers/test_tiller.py
@@ -17,7 +17,7 @@ from mock import MagicMock
17 17
18from armada.exceptions import tiller_exceptions as ex 18from armada.exceptions import tiller_exceptions as ex
19from armada.handlers import tiller 19from armada.handlers import tiller
20from armada.handlers import test 20from armada.utils import helm
21from armada.tests.unit import base 21from armada.tests.unit import base
22from armada.tests.test_utils import AttrDict 22from armada.tests.test_utils import AttrDict
23 23
@@ -552,7 +552,7 @@ class TillerTestCase(base.ArmadaTestCase):
552 self._test_test_release([ 552 self._test_test_release([
553 AttrDict(**{ 553 AttrDict(**{
554 'msg': 'No Tests Found', 554 'msg': 'No Tests Found',
555 'status': test.TESTRUN_STATUS_UNKNOWN 555 'status': helm.TESTRUN_STATUS_UNKNOWN
556 }) 556 })
557 ]) 557 ])
558 558
@@ -560,11 +560,11 @@ class TillerTestCase(base.ArmadaTestCase):
560 self._test_test_release([ 560 self._test_test_release([
561 AttrDict(**{ 561 AttrDict(**{
562 'msg': 'RUNNING: ...', 562 'msg': 'RUNNING: ...',
563 'status': test.TESTRUN_STATUS_RUNNING 563 'status': helm.TESTRUN_STATUS_RUNNING
564 }), 564 }),
565 AttrDict(**{ 565 AttrDict(**{
566 'msg': 'SUCCESS: ...', 566 'msg': 'SUCCESS: ...',
567 'status': test.TESTRUN_STATUS_SUCCESS 567 'status': helm.TESTRUN_STATUS_SUCCESS
568 }) 568 })
569 ]) 569 ])
570 570
@@ -572,11 +572,11 @@ class TillerTestCase(base.ArmadaTestCase):
572 self._test_test_release([ 572 self._test_test_release([
573 AttrDict(**{ 573 AttrDict(**{
574 'msg': 'RUNNING: ...', 574 'msg': 'RUNNING: ...',
575 'status': test.TESTRUN_STATUS_RUNNING 575 'status': helm.TESTRUN_STATUS_RUNNING
576 }), 576 }),
577 AttrDict(**{ 577 AttrDict(**{
578 'msg': 'FAILURE: ...', 578 'msg': 'FAILURE: ...',
579 'status': test.TESTRUN_STATUS_FAILURE 579 'status': helm.TESTRUN_STATUS_FAILURE
580 }) 580 })
581 ]) 581 ])
582 582
diff --git a/armada/utils/helm.py b/armada/utils/helm.py
new file mode 100644
index 0000000..d3f5cb8
--- /dev/null
+++ b/armada/utils/helm.py
@@ -0,0 +1,40 @@
1# Copyright 2019 The Armada Authors.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15TESTRUN_STATUS_UNKNOWN = 0
16TESTRUN_STATUS_SUCCESS = 1
17TESTRUN_STATUS_FAILURE = 2
18TESTRUN_STATUS_RUNNING = 3
19
20HELM_HOOK_ANNOTATION = 'helm.sh/hook'
21HELM_TEST_HOOKS = ['test-success', 'test-failure']
22
23
24def is_test_pod(pod):
25 annotations = pod.metadata.annotations
26
27 # Retrieve pod's Helm test hooks
28 test_hooks = None
29 if annotations:
30 hook_string = annotations.get(HELM_HOOK_ANNOTATION)
31 if hook_string:
32 hooks = hook_string.split(',')
33 test_hooks = [h for h in hooks if h in HELM_TEST_HOOKS]
34
35 return bool(test_hooks)
36
37
38def get_test_suite_run_success(test_suite_run):
39 return all(
40 r.status == TESTRUN_STATUS_SUCCESS for r in test_suite_run.results)
diff --git a/armada/utils/release.py b/armada/utils/release.py
index 55fdda7..9267661 100644
--- a/armada/utils/release.py
+++ b/armada/utils/release.py
@@ -12,7 +12,7 @@
12# See the License for the specific language governing permissions and 12# See the License for the specific language governing permissions and
13# limitations under the License. 13# limitations under the License.
14 14
15from armada.handlers.test import Test 15from armada.utils.helm import get_test_suite_run_success
16 16
17import time 17import time
18 18
@@ -54,7 +54,7 @@ def get_last_test_result(release):
54 status = release.info.status 54 status = release.info.status
55 if not status.HasField('last_test_suite_run'): 55 if not status.HasField('last_test_suite_run'):
56 return None 56 return None
57 return Test.get_test_suite_run_success(status.last_test_suite_run) 57 return get_test_suite_run_success(status.last_test_suite_run)
58 58
59 59
60def get_last_deployment_age(release): 60def get_last_deployment_age(release):
diff --git a/doc/source/operations/guide-build-armada-yaml.rst b/doc/source/operations/guide-build-armada-yaml.rst
index 65f6ac6..5898a97 100644
--- a/doc/source/operations/guide-build-armada-yaml.rst
+++ b/doc/source/operations/guide-build-armada-yaml.rst
@@ -203,10 +203,15 @@ Test options to pass through directly to helm.
203 203
204.. note:: 204.. note::
205 205
206 The preferred way to achieve test cleanup is to add a pre-upgrade delete 206 If cleanup is ``true`` this prevents being able to debug a test in the event of failure.
207 action on the test pod, which allows for debugging the test pod up until the
208 next upgrade.
209 207
208 Historically, the preferred way to achieve test cleanup has been to add a pre-upgrade delete
209 action on the test pod.
210
211 This still works, however it is usually no longer necessary as Armada now automatically
212 cleans up any test pods which match the ``wait.labels`` of the chart, immediately before
213 running tests. Similar suggestions have been made for how ``helm test --cleanup`` itself
214 ought to work (https://github.com/helm/helm/issues/3279).
210 215
211Upgrade - Pre 216Upgrade - Pre
212^^^^^^^^^^^^^ 217^^^^^^^^^^^^^