summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorDrew Walters <drewwalters96@gmail.com>2018-11-14 17:05:28 +0000
committerDrew Walters <drewwalters96@gmail.com>2018-11-29 17:30:57 +0000
commitadfe3ae5053c0935b1bb55de9ba92ead5b2109c1 (patch)
treeb0ae15a3b64b75f9dff3e0b62087dec28a5bf32c
parenta64d435de8210cda9c5a5fb25885209c33801cd9 (diff)
test: Refactor test handler
While authoring [0], it was discovered that Armada has duplicate logic for deciding if Helm test cleanup should be enabled as well as the tests themselves. Because of this, changes to test logic (e.g. adding pre-test actions), requires changing all traces of the repeated logic, which can lead to inconsistent behavior if not properly addressed. This change moves all test decision logic to a singular Test handler, implemented by the `Test` class. This change does NOT change the expected behavior of testing during upgrades; however, tests initiated from the API and CLI will not execute when testing a manifest if they are disabled in a chart, unless using the `--enable-all` flag. [0] https://review.openstack.org/617834 Change-Id: I1530d7637b0eb6a83f048895053a5db80d033046
Notes
Notes (review): Code-Review+2: Sean Eagan <sean.eagan@att.com> Code-Review+2: Aaron Sheffield <ajs@sheffieldfamily.net> Workflow+1: Aaron Sheffield <ajs@sheffieldfamily.net> Verified+2: Zuul Submitted-by: Zuul Submitted-at: Tue, 04 Dec 2018 20:40:54 +0000 Reviewed-on: https://review.openstack.org/618355 Project: openstack/airship-armada Branch: refs/heads/master
-rw-r--r--armada/api/controller/test.py55
-rw-r--r--armada/cli/test.py59
-rw-r--r--armada/handlers/armada.py8
-rw-r--r--armada/handlers/chart_deploy.py41
-rw-r--r--armada/handlers/test.py109
-rw-r--r--armada/tests/unit/api/test_test_controller.py16
-rw-r--r--armada/tests/unit/handlers/test_armada.py54
-rw-r--r--armada/tests/unit/handlers/test_test.py201
-rw-r--r--armada/utils/release.py4
-rw-r--r--doc/source/commands/test.rst2
-rw-r--r--swagger/swaggerV3-api.yaml8
11 files changed, 400 insertions, 157 deletions
diff --git a/armada/api/controller/test.py b/armada/api/controller/test.py
index 2201fe2..d1804c5 100644
--- a/armada/api/controller/test.py
+++ b/armada/api/controller/test.py
@@ -21,8 +21,8 @@ from oslo_config import cfg
21from armada import api 21from armada import api
22from armada.common import policy 22from armada.common import policy
23from armada import const 23from armada import const
24from armada.handlers.test import test_release_for_success
25from armada.handlers.manifest import Manifest 24from armada.handlers.manifest import Manifest
25from armada.handlers.test import Test
26from armada.utils.release import release_prefixer 26from armada.utils.release import release_prefixer
27from armada.utils import validate 27from armada.utils import validate
28 28
@@ -36,14 +36,11 @@ class TestReleasesReleaseNameController(api.BaseResource):
36 36
37 @policy.enforce('armada:test_release') 37 @policy.enforce('armada:test_release')
38 def on_get(self, req, resp, release): 38 def on_get(self, req, resp, release):
39 self.logger.info('RUNNING: %s', release)
40 with self.get_tiller(req, resp) as tiller: 39 with self.get_tiller(req, resp) as tiller:
41
42 cleanup = req.get_param_as_bool('cleanup') 40 cleanup = req.get_param_as_bool('cleanup')
43 if cleanup is None: 41
44 cleanup = False 42 test_handler = Test(release, tiller, cleanup=cleanup)
45 success = test_release_for_success( 43 success = test_handler.test_release_for_success()
46 tiller, release, cleanup=cleanup)
47 44
48 if success: 45 if success:
49 msg = { 46 msg = {
@@ -56,8 +53,6 @@ class TestReleasesReleaseNameController(api.BaseResource):
56 'message': 'MESSAGE: Test Fail' 53 'message': 'MESSAGE: Test Fail'
57 } 54 }
58 55
59 self.logger.info(msg)
60
61 resp.body = json.dumps(msg) 56 resp.body = json.dumps(msg)
62 resp.status = falcon.HTTP_200 57 resp.status = falcon.HTTP_200
63 resp.content_type = 'application/json' 58 resp.content_type = 'application/json'
@@ -135,29 +130,29 @@ class TestReleasesManifestController(api.BaseResource):
135 const.KEYWORD_GROUPS): 130 const.KEYWORD_GROUPS):
136 for ch in group.get(const.KEYWORD_CHARTS): 131 for ch in group.get(const.KEYWORD_CHARTS):
137 chart = ch['chart'] 132 chart = ch['chart']
133
138 release_name = release_prefixer(prefix, chart.get('release')) 134 release_name = release_prefixer(prefix, chart.get('release'))
139 cleanup = req.get_param_as_bool('cleanup')
140 if cleanup is None:
141 test_chart_override = chart.get('test', {})
142 if isinstance(test_chart_override, bool):
143 self.logger.warn(
144 'Boolean value for chart `test` key is deprecated '
145 'and will be removed. Use `test.enabled` instead.')
146 # Use old default value.
147 cleanup = True
148 else:
149 cleanup = test_chart_override.get('options', {}).get(
150 'cleanup', False)
151 if release_name in known_releases: 135 if release_name in known_releases:
152 self.logger.info('RUNNING: %s tests', release_name) 136 cleanup = req.get_param_as_bool('cleanup')
153 success = test_release_for_success( 137 enable_all = req.get_param_as_bool('enable_all')
154 tiller, release_name, cleanup=cleanup) 138 cg_test_charts = group.get('test_charts')
155 if success: 139 test_values = chart.get('test', {})
156 self.logger.info("PASSED: %s", release_name) 140
157 message['test']['passed'].append(release_name) 141 test_handler = Test(
158 else: 142 release_name,
159 self.logger.info("FAILED: %s", release_name) 143 tiller,
160 message['test']['failed'].append(release_name) 144 cg_test_charts=cg_test_charts,
145 cleanup=cleanup,
146 enable_all=enable_all,
147 test_values=test_values)
148
149 if test_handler.test_enabled:
150 success = test_handler.test_release_for_success()
151
152 if success:
153 message['test']['passed'].append(release_name)
154 else:
155 message['test']['failed'].append(release_name)
161 else: 156 else:
162 self.logger.info('Release %s not found - SKIPPING', 157 self.logger.info('Release %s not found - SKIPPING',
163 release_name) 158 release_name)
diff --git a/armada/cli/test.py b/armada/cli/test.py
index f3636f7..998426a 100644
--- a/armada/cli/test.py
+++ b/armada/cli/test.py
@@ -19,8 +19,8 @@ from oslo_config import cfg
19 19
20from armada.cli import CliAction 20from armada.cli import CliAction
21from armada import const 21from armada import const
22from armada.handlers.test import test_release_for_success
23from armada.handlers.manifest import Manifest 22from armada.handlers.manifest import Manifest
23from armada.handlers.test import Test
24from armada.handlers.tiller import Tiller 24from armada.handlers.tiller import Tiller
25from armada.utils.release import release_prefixer 25from armada.utils.release import release_prefixer
26 26
@@ -79,20 +79,26 @@ SHORT_DESC = "Command tests releases."
79 help=("Delete test pods upon completion."), 79 help=("Delete test pods upon completion."),
80 is_flag=True, 80 is_flag=True,
81 default=None) 81 default=None)
82@click.option(
83 '--enable-all',
84 help=("Run all tests for all releases regardless of any disabled chart "
85 "tests."),
86 is_flag=True,
87 default=False)
82@click.option('--debug', help="Enable debug logging.", is_flag=True) 88@click.option('--debug', help="Enable debug logging.", is_flag=True)
83@click.pass_context 89@click.pass_context
84def test_charts(ctx, file, release, tiller_host, tiller_port, tiller_namespace, 90def test_charts(ctx, file, release, tiller_host, tiller_port, tiller_namespace,
85 target_manifest, cleanup, debug): 91 target_manifest, cleanup, enable_all, debug):
86 CONF.debug = debug 92 CONF.debug = debug
87 TestChartManifest(ctx, file, release, tiller_host, tiller_port, 93 TestChartManifest(ctx, file, release, tiller_host, tiller_port,
88 tiller_namespace, target_manifest, 94 tiller_namespace, target_manifest, cleanup,
89 cleanup).safe_invoke() 95 enable_all).safe_invoke()
90 96
91 97
92class TestChartManifest(CliAction): 98class TestChartManifest(CliAction):
93 99
94 def __init__(self, ctx, file, release, tiller_host, tiller_port, 100 def __init__(self, ctx, file, release, tiller_host, tiller_port,
95 tiller_namespace, target_manifest, cleanup): 101 tiller_namespace, target_manifest, cleanup, enable_all):
96 102
97 super(TestChartManifest, self).__init__() 103 super(TestChartManifest, self).__init__()
98 self.ctx = ctx 104 self.ctx = ctx
@@ -103,6 +109,7 @@ class TestChartManifest(CliAction):
103 self.tiller_namespace = tiller_namespace 109 self.tiller_namespace = tiller_namespace
104 self.target_manifest = target_manifest 110 self.target_manifest = target_manifest
105 self.cleanup = cleanup 111 self.cleanup = cleanup
112 self.enable_all = enable_all
106 113
107 def invoke(self): 114 def invoke(self):
108 with Tiller( 115 with Tiller(
@@ -117,13 +124,8 @@ class TestChartManifest(CliAction):
117 124
118 if self.release: 125 if self.release:
119 if not self.ctx.obj.get('api', False): 126 if not self.ctx.obj.get('api', False):
120 self.logger.info("RUNNING: %s tests", self.release) 127 test_handler = Test(self.release, tiller, cleanup=self.cleanup)
121 success = test_release_for_success( 128 test_handler.test_release_for_success()
122 tiller, self.release, cleanup=self.cleanup)
123 if success:
124 self.logger.info("PASSED: %s", self.release)
125 else:
126 self.logger.info("FAILED: %s", self.release)
127 else: 129 else:
128 client = self.ctx.obj.get('CLIENT') 130 client = self.ctx.obj.get('CLIENT')
129 query = { 131 query = {
@@ -150,32 +152,21 @@ class TestChartManifest(CliAction):
150 const.KEYWORD_GROUPS): 152 const.KEYWORD_GROUPS):
151 for ch in group.get(const.KEYWORD_CHARTS): 153 for ch in group.get(const.KEYWORD_CHARTS):
152 chart = ch['chart'] 154 chart = ch['chart']
155
153 release_name = release_prefixer( 156 release_name = release_prefixer(
154 prefix, chart.get('release')) 157 prefix, chart.get('release'))
155
156 if release_name in known_release_names: 158 if release_name in known_release_names:
157 cleanup = self.cleanup 159 test_values = chart.get('test', {})
158 if cleanup is None: 160
159 test_chart_override = chart.get('test', {}) 161 test_handler = Test(
160 if isinstance(test_chart_override, bool): 162 release_name,
161 self.logger.warn( 163 tiller,
162 'Boolean value for chart `test` key is' 164 cleanup=self.cleanup,
163 ' deprecated and support for this will' 165 enable_all=self.enable_all,
164 ' be removed. Use `test.enabled` ' 166 test_values=test_values)
165 'instead.')
166 # Use old default value.
167 cleanup = True
168 else:
169 cleanup = test_chart_override.get(
170 'options', {}).get('cleanup', False)
171 self.logger.info('RUNNING: %s tests', release_name)
172 success = test_release_for_success(
173 tiller, release_name, cleanup=cleanup)
174 if success:
175 self.logger.info("PASSED: %s", release_name)
176 else:
177 self.logger.info("FAILED: %s", release_name)
178 167
168 if test_handler.test_enabled:
169 test_handler.test_release_for_success()
179 else: 170 else:
180 self.logger.info('Release %s not found - SKIPPING', 171 self.logger.info('Release %s not found - SKIPPING',
181 release_name) 172 release_name)
diff --git a/armada/handlers/armada.py b/armada/handlers/armada.py
index 0fb5063..0bf502c 100644
--- a/armada/handlers/armada.py
+++ b/armada/handlers/armada.py
@@ -200,14 +200,6 @@ class Armada(object):
200 200
201 # TODO(MarshM): Deprecate the `test_charts` key 201 # TODO(MarshM): Deprecate the `test_charts` key
202 cg_test_all_charts = chartgroup.get('test_charts') 202 cg_test_all_charts = chartgroup.get('test_charts')
203 if isinstance(cg_test_all_charts, bool):
204 LOG.warn('The ChartGroup `test_charts` key is deprecated, '
205 'and support for this will be removed. See the '
206 'Chart `test` key for more information.')
207 else:
208 # This key defaults to True. Individual charts must
209 # explicitly disable helm tests if they choose
210 cg_test_all_charts = True
211 203
212 cg_charts = chartgroup.get(const.KEYWORD_CHARTS, []) 204 cg_charts = chartgroup.get(const.KEYWORD_CHARTS, [])
213 charts = map(lambda x: x.get('chart', {}), cg_charts) 205 charts = map(lambda x: x.get('chart', {}), cg_charts)
diff --git a/armada/handlers/chart_deploy.py b/armada/handlers/chart_deploy.py
index ab6965d..30671fd 100644
--- a/armada/handlers/chart_deploy.py
+++ b/armada/handlers/chart_deploy.py
@@ -19,8 +19,8 @@ import yaml
19from armada import const 19from armada import const
20from armada.exceptions import armada_exceptions 20from armada.exceptions import armada_exceptions
21from armada.handlers.chartbuilder import ChartBuilder 21from armada.handlers.chartbuilder import ChartBuilder
22from armada.handlers.test import test_release_for_success
23from armada.handlers.release_diff import ReleaseDiff 22from armada.handlers.release_diff import ReleaseDiff
23from armada.handlers.test import Test
24from armada.handlers.wait import ChartWait 24from armada.handlers.wait import ChartWait
25from armada.exceptions import tiller_exceptions 25from armada.exceptions import tiller_exceptions
26import armada.utils.release as r 26import armada.utils.release as r
@@ -202,34 +202,25 @@ class ChartDeploy(object):
202 chart_wait.wait(timer) 202 chart_wait.wait(timer)
203 203
204 # Test 204 # Test
205 test_chart_override = chart.get('test')
206 # Use old default value when not using newer `test` key
207 test_cleanup = True
208 if test_chart_override is None:
209 test_enabled = cg_test_all_charts
210 elif isinstance(test_chart_override, bool):
211 LOG.warn('Boolean value for chart `test` key is'
212 ' deprecated and support for this will'
213 ' be removed. Use `test.enabled` '
214 'instead.')
215 test_enabled = test_chart_override
216 else:
217 # NOTE: helm tests are enabled by default
218 test_enabled = test_chart_override.get('enabled', True)
219 test_cleanup = test_chart_override.get('options', {}).get(
220 'cleanup', False)
221
222 just_deployed = ('install' in result) or ('upgrade' in result) 205 just_deployed = ('install' in result) or ('upgrade' in result)
223 last_test_passed = old_release and r.get_last_test_result(old_release) 206 last_test_passed = old_release and r.get_last_test_result(old_release)
224 run_test = test_enabled and (just_deployed or not last_test_passed)
225 207
208 test_values = chart.get('test')
209 test_handler = Test(
210 release_name,
211 self.tiller,
212 cg_test_charts=cg_test_all_charts,
213 test_values=test_values)
214
215 run_test = test_handler.test_enabled and (just_deployed or
216 not last_test_passed)
226 if run_test: 217 if run_test:
227 timer = int(round(deadline - time.time())) 218 timer = int(round(deadline - time.time()))
228 self._test_chart(release_name, timer, test_cleanup) 219 self._test_chart(release_name, timer, test_handler)
229 220
230 return result 221 return result
231 222
232 def _test_chart(self, release_name, timeout, cleanup): 223 def _test_chart(self, release_name, timeout, test_handler):
233 if self.dry_run: 224 if self.dry_run:
234 LOG.info( 225 LOG.info(
235 'Skipping test during `dry-run`, would have tested ' 226 'Skipping test during `dry-run`, would have tested '
@@ -242,12 +233,8 @@ class ChartDeploy(object):
242 LOG.error(reason) 233 LOG.error(reason)
243 raise armada_exceptions.ArmadaTimeoutException(reason) 234 raise armada_exceptions.ArmadaTimeoutException(reason)
244 235
245 success = test_release_for_success( 236 success = test_handler.test_release_for_success(timeout=timeout)
246 self.tiller, release_name, timeout=timeout, cleanup=cleanup) 237 if not success:
247 if success:
248 LOG.info("Test passed for release: %s", release_name)
249 else:
250 LOG.info("Test failed for release: %s", release_name)
251 raise tiller_exceptions.TestFailedException(release_name) 238 raise tiller_exceptions.TestFailedException(release_name)
252 239
253 def get_diff(self, old_chart, old_values, new_chart, values): 240 def get_diff(self, old_chart, old_values, new_chart, values):
diff --git a/armada/handlers/test.py b/armada/handlers/test.py
index a463682..aa7fe03 100644
--- a/armada/handlers/test.py
+++ b/armada/handlers/test.py
@@ -12,6 +12,8 @@
12# See the License for the specific language governing permissions and 12# See the License for the specific language governing permissions and
13# limitations under the License. 13# limitations under the License.
14 14
15from oslo_log import log as logging
16
15from armada import const 17from armada import const
16 18
17TESTRUN_STATUS_UNKNOWN = 0 19TESTRUN_STATUS_UNKNOWN = 0
@@ -19,16 +21,103 @@ TESTRUN_STATUS_SUCCESS = 1
19TESTRUN_STATUS_FAILURE = 2 21TESTRUN_STATUS_FAILURE = 2
20TESTRUN_STATUS_RUNNING = 3 22TESTRUN_STATUS_RUNNING = 3
21 23
24LOG = logging.getLogger(__name__)
25
26
27class Test(object):
28
29 def __init__(self,
30 release_name,
31 tiller,
32 cg_test_charts=None,
33 cleanup=None,
34 enable_all=False,
35 test_values=None):
36 """Initialize a test handler to run Helm tests corresponding to a
37 release.
38
39 :param release_name: Name of a Helm release
40 :param tiller: Tiller object
41 :param cg_test_charts: Chart group `test_charts` key
42 :param cleanup: Triggers cleanup; overrides `test.options.cleanup`
43 :param enable_all: Run tests regardless of the value of `test.enabled`
44 :param test_values: Test values retrieved from a chart's `test` key
45
46 :type release_name: str
47 :type tiller: Tiller object
48 :type cg_test_charts: bool
49 :type cleanup: bool
50 :type enable_all: bool
51 :type test_values: dict or bool (deprecated)
52 """
53
54 self.release_name = release_name
55 self.tiller = tiller
56 self.cleanup = cleanup
57
58 # NOTE(drewwalters96): Support the chart_group `test_charts` key until
59 # its deprecation period ends. The `test.enabled`, `enable_all` flag,
60 # and deprecated, boolean `test` key override this value if provided.
61 if cg_test_charts is not None:
62 LOG.warn('Chart group key `test_charts` is deprecated and will be '
63 'removed. Use `test.enabled` instead.')
64 self.test_enabled = cg_test_charts
65 else:
66 self.test_enabled = True
67
68 # NOTE: Support old, boolean `test` key until deprecation period ends.
69 if (type(test_values) == bool):
70 LOG.warn('Boolean value for chart `test` key is deprecated and '
71 'will be removed. Use `test.enabled` instead.')
72
73 self.test_enabled = test_values
74
75 # NOTE: Use old, default cleanup value (i.e. True) if none is
76 # provided.
77 if self.cleanup is None:
78 self.cleanup = True
79 elif test_values:
80 test_enabled_opt = test_values.get('enabled')
81 if test_enabled_opt is not None:
82 self.test_enabled = test_enabled_opt
83
84 # NOTE(drewwalters96): `self.cleanup`, the cleanup value provided
85 # by the API/CLI, takes precedence over the chart value
86 # `test.cleanup`.
87 if self.cleanup is None:
88 test_options = test_values.get('options', {})
89 self.cleanup = test_options.get('cleanup', False)
90 else:
91 # Default cleanup value
92 if self.cleanup is None:
93 self.cleanup = False
94
95 if enable_all:
96 self.test_enabled = True
97
98 def test_release_for_success(self, timeout=const.DEFAULT_TILLER_TIMEOUT):
99 """Run the Helm tests corresponding to a release for success (i.e. exit
100 code 0).
101
102 :param timeout: Timeout value for a release's tests completion
103 :type timeout: int
104
105 :rtype: Helm test suite run result
106 """
107 LOG.info('RUNNING: %s tests', self.release_name)
108
109 test_suite_run = self.tiller.test_release(
110 self.release_name, timeout=timeout, cleanup=self.cleanup)
22 111
23def test_release_for_success(tiller, 112 success = self.get_test_suite_run_success(test_suite_run)
24 release, 113 if success:
25 timeout=const.DEFAULT_TILLER_TIMEOUT, 114 LOG.info('PASSED: %s', self.release_name)
26 cleanup=False): 115 else:
27 test_suite_run = tiller.test_release( 116 LOG.info('FAILED: %s', self.release_name)
28 release, timeout=timeout, cleanup=cleanup)
29 return get_test_suite_run_success(test_suite_run)
30 117
118 return success
31 119
32def get_test_suite_run_success(test_suite_run): 120 @classmethod
33 return all( 121 def get_test_suite_run_success(self, test_suite_run):
34 r.status == TESTRUN_STATUS_SUCCESS for r in test_suite_run.results) 122 return all(
123 r.status == TESTRUN_STATUS_SUCCESS for r in test_suite_run.results)
diff --git a/armada/tests/unit/api/test_test_controller.py b/armada/tests/unit/api/test_test_controller.py
index 9c8d797..d5f7fa4 100644
--- a/armada/tests/unit/api/test_test_controller.py
+++ b/armada/tests/unit/api/test_test_controller.py
@@ -59,7 +59,7 @@ class TestReleasesManifestControllerTest(base.BaseControllerTest):
59 59
60class TestReleasesReleaseNameControllerTest(base.BaseControllerTest): 60class TestReleasesReleaseNameControllerTest(base.BaseControllerTest):
61 61
62 @mock.patch.object(test, 'test_release_for_success') 62 @mock.patch.object(test.Test, 'test_release_for_success')
63 @mock.patch.object(api, 'Tiller') 63 @mock.patch.object(api, 'Tiller')
64 def test_test_controller_test_pass(self, mock_tiller, 64 def test_test_controller_test_pass(self, mock_tiller,
65 mock_test_release_for_success): 65 mock_test_release_for_success):
@@ -73,14 +73,13 @@ class TestReleasesReleaseNameControllerTest(base.BaseControllerTest):
73 73
74 release = 'fake-release' 74 release = 'fake-release'
75 resp = self.app.simulate_get('/api/v1.0/test/{}'.format(release)) 75 resp = self.app.simulate_get('/api/v1.0/test/{}'.format(release))
76 mock_test_release_for_success.assert_has_calls( 76 mock_test_release_for_success.assert_called_once()
77 [mock.call(mock_tiller.return_value, release, cleanup=False)])
78 self.assertEqual(200, resp.status_code) 77 self.assertEqual(200, resp.status_code)
79 self.assertEqual('MESSAGE: Test Pass', 78 self.assertEqual('MESSAGE: Test Pass',
80 json.loads(resp.text)['message']) 79 json.loads(resp.text)['message'])
81 m_tiller.__exit__.assert_called() 80 m_tiller.__exit__.assert_called()
82 81
83 @mock.patch.object(test, 'test_release_for_success') 82 @mock.patch.object(test.Test, 'test_release_for_success')
84 @mock.patch.object(api, 'Tiller') 83 @mock.patch.object(api, 'Tiller')
85 def test_test_controller_test_fail(self, mock_tiller, 84 def test_test_controller_test_fail(self, mock_tiller,
86 mock_test_release_for_success): 85 mock_test_release_for_success):
@@ -98,7 +97,7 @@ class TestReleasesReleaseNameControllerTest(base.BaseControllerTest):
98 json.loads(resp.text)['message']) 97 json.loads(resp.text)['message'])
99 m_tiller.__exit__.assert_called() 98 m_tiller.__exit__.assert_called()
100 99
101 @mock.patch.object(test, 'test_release_for_success') 100 @mock.patch.object(test.Test, 'test_release_for_success')
102 @mock.patch.object(api, 'Tiller') 101 @mock.patch.object(api, 'Tiller')
103 def test_test_controller_cleanup(self, mock_tiller, 102 def test_test_controller_cleanup(self, mock_tiller,
104 mock_test_release_for_success): 103 mock_test_release_for_success):
@@ -112,8 +111,7 @@ class TestReleasesReleaseNameControllerTest(base.BaseControllerTest):
112 release = 'fake-release' 111 release = 'fake-release'
113 resp = self.app.simulate_get( 112 resp = self.app.simulate_get(
114 '/api/v1.0/test/{}'.format(release), query_string='cleanup=true') 113 '/api/v1.0/test/{}'.format(release), query_string='cleanup=true')
115 mock_test_release_for_success.assert_has_calls( 114 mock_test_release_for_success.assert_called_once()
116 [mock.call(m_tiller, release, cleanup=True)])
117 self.assertEqual(200, resp.status_code) 115 self.assertEqual(200, resp.status_code)
118 self.assertEqual('MESSAGE: Test Pass', 116 self.assertEqual('MESSAGE: Test Pass',
119 json.loads(resp.text)['message']) 117 json.loads(resp.text)['message'])
@@ -125,7 +123,7 @@ class TestReleasesManifestControllerNegativeTest(base.BaseControllerTest):
125 123
126 @mock.patch.object(test, 'Manifest') 124 @mock.patch.object(test, 'Manifest')
127 @mock.patch.object(api, 'Tiller') 125 @mock.patch.object(api, 'Tiller')
128 @mock.patch.object(test, 'test_release_for_success') 126 @mock.patch.object(test.Test, 'test_release_for_success')
129 def test_test_controller_tiller_exc_returns_500( 127 def test_test_controller_tiller_exc_returns_500(
130 self, mock_test_release_for_success, mock_tiller, _): 128 self, mock_test_release_for_success, mock_tiller, _):
131 rules = {'armada:test_manifest': '@'} 129 rules = {'armada:test_manifest': '@'}
@@ -227,7 +225,7 @@ class TestReleasesManifestControllerNegativeTest(base.BaseControllerTest):
227class TestReleasesReleaseNameControllerNegativeTest(base.BaseControllerTest): 225class TestReleasesReleaseNameControllerNegativeTest(base.BaseControllerTest):
228 226
229 @mock.patch.object(api, 'Tiller') 227 @mock.patch.object(api, 'Tiller')
230 @mock.patch.object(test, 'test_release_for_success') 228 @mock.patch.object(test.Test, 'test_release_for_success')
231 def test_test_controller_tiller_exc_returns_500( 229 def test_test_controller_tiller_exc_returns_500(
232 self, mock_test_release_for_success, mock_tiller): 230 self, mock_test_release_for_success, mock_tiller):
233 rules = {'armada:test_release': '@'} 231 rules = {'armada:test_release': '@'}
diff --git a/armada/tests/unit/handlers/test_armada.py b/armada/tests/unit/handlers/test_armada.py
index ab1876c..06321da 100644
--- a/armada/tests/unit/handlers/test_armada.py
+++ b/armada/tests/unit/handlers/test_armada.py
@@ -17,7 +17,6 @@ import yaml
17 17
18from armada import const 18from armada import const
19from armada.handlers import armada 19from armada.handlers import armada
20from armada.handlers import chart_deploy
21from armada.handlers.test import TESTRUN_STATUS_SUCCESS, TESTRUN_STATUS_FAILURE 20from armada.handlers.test import TESTRUN_STATUS_SUCCESS, TESTRUN_STATUS_FAILURE
22from armada.tests.unit import base 21from armada.tests.unit import base
23from armada.tests.test_utils import AttrDict, makeMockThreadSafe 22from armada.tests.test_utils import AttrDict, makeMockThreadSafe
@@ -337,9 +336,9 @@ class ArmadaHandlerTestCase(base.ArmadaTestCase):
337 @mock.patch.object(armada.Armada, 'post_flight_ops') 336 @mock.patch.object(armada.Armada, 'post_flight_ops')
338 @mock.patch.object(armada.Armada, 'pre_flight_ops') 337 @mock.patch.object(armada.Armada, 'pre_flight_ops')
339 @mock.patch('armada.handlers.chart_deploy.ChartBuilder') 338 @mock.patch('armada.handlers.chart_deploy.ChartBuilder')
340 @mock.patch.object(chart_deploy, 'test_release_for_success') 339 @mock.patch('armada.handlers.chart_deploy.Test')
341 def _do_test(mock_test_release_for_success, mock_chartbuilder, 340 def _do_test(mock_test, mock_chartbuilder, mock_pre_flight,
342 mock_pre_flight, mock_post_flight): 341 mock_post_flight):
343 # Instantiate Armada object. 342 # Instantiate Armada object.
344 yaml_documents = list(yaml.safe_load_all(TEST_YAML)) 343 yaml_documents = list(yaml.safe_load_all(TEST_YAML))
345 344
@@ -351,8 +350,9 @@ class ArmadaHandlerTestCase(base.ArmadaTestCase):
351 350
352 chart_group = armada_obj.manifest['armada']['chart_groups'][0] 351 chart_group = armada_obj.manifest['armada']['chart_groups'][0]
353 charts = chart_group['chart_group'] 352 charts = chart_group['chart_group']
354 cg_test_all_charts = chart_group.get('test_charts', True) 353 cg_test_all_charts = chart_group.get('test_charts')
355 354
355 mock_test_release = mock_test.return_value.test_release_for_success
356 if test_failure_to_run: 356 if test_failure_to_run:
357 357
358 def fail(tiller, release, timeout=None, cleanup=False): 358 def fail(tiller, release, timeout=None, cleanup=False):
@@ -361,9 +361,9 @@ class ArmadaHandlerTestCase(base.ArmadaTestCase):
361 raise tiller_exceptions.ReleaseException( 361 raise tiller_exceptions.ReleaseException(
362 release, status, 'Test') 362 release, status, 'Test')
363 363
364 mock_test_release_for_success.side_effect = fail 364 mock_test_release.side_effect = fail
365 else: 365 else:
366 mock_test_release_for_success.return_value = test_success 366 mock_test_release.return_value = test_success
367 367
368 # Stub out irrelevant methods called by `armada.sync()`. 368 # Stub out irrelevant methods called by `armada.sync()`.
369 mock_chartbuilder.get_source_path.return_value = None 369 mock_chartbuilder.get_source_path.return_value = None
@@ -377,7 +377,7 @@ class ArmadaHandlerTestCase(base.ArmadaTestCase):
377 expected_install_release_calls = [] 377 expected_install_release_calls = []
378 expected_update_release_calls = [] 378 expected_update_release_calls = []
379 expected_uninstall_release_calls = [] 379 expected_uninstall_release_calls = []
380 expected_test_release_for_success_calls = [] 380 expected_test_constructor_calls = []
381 381
382 for c in charts: 382 for c in charts:
383 chart = c['chart'] 383 chart = c['chart']
@@ -389,7 +389,6 @@ class ArmadaHandlerTestCase(base.ArmadaTestCase):
389 native_wait_enabled = (chart['wait'].get('native', {}).get( 389 native_wait_enabled = (chart['wait'].get('native', {}).get(
390 'enabled', True)) 390 'enabled', True))
391 391
392 expected_apply = True
393 if release_name not in [x.name for x in known_releases]: 392 if release_name not in [x.name for x in known_releases]:
394 expected_install_release_calls.append( 393 expected_install_release_calls.append(
395 mock.call( 394 mock.call(
@@ -435,9 +434,7 @@ class ArmadaHandlerTestCase(base.ArmadaTestCase):
435 break 434 break
436 435
437 if status == const.STATUS_DEPLOYED: 436 if status == const.STATUS_DEPLOYED:
438 if not diff: 437 if diff:
439 expected_apply = False
440 else:
441 upgrade = chart.get('upgrade', {}) 438 upgrade = chart.get('upgrade', {})
442 disable_hooks = upgrade.get('no_hooks', False) 439 disable_hooks = upgrade.get('no_hooks', False)
443 force = upgrade.get('force', False) 440 force = upgrade.get('force', False)
@@ -462,25 +459,12 @@ class ArmadaHandlerTestCase(base.ArmadaTestCase):
462 timeout=mock.ANY)) 459 timeout=mock.ANY))
463 460
464 test_chart_override = chart.get('test') 461 test_chart_override = chart.get('test')
465 # Use old default value when not using newer `test` key 462 expected_test_constructor_calls.append(
466 test_cleanup = True 463 mock.call(
467 if test_chart_override is None: 464 release_name,
468 test_this_chart = cg_test_all_charts 465 m_tiller,
469 elif isinstance(test_chart_override, bool): 466 cg_test_charts=cg_test_all_charts,
470 test_this_chart = test_chart_override 467 test_values=test_chart_override))
471 else:
472 test_this_chart = test_chart_override.get('enabled', True)
473 test_cleanup = test_chart_override.get('options', {}).get(
474 'cleanup', False)
475
476 if test_this_chart and (expected_apply or
477 not expected_last_test_result):
478 expected_test_release_for_success_calls.append(
479 mock.call(
480 m_tiller,
481 release_name,
482 timeout=mock.ANY,
483 cleanup=test_cleanup))
484 468
485 any_order = not chart_group['sequenced'] 469 any_order = not chart_group['sequenced']
486 # Verify that at least 1 release is either installed or updated. 470 # Verify that at least 1 release is either installed or updated.
@@ -511,10 +495,10 @@ class ArmadaHandlerTestCase(base.ArmadaTestCase):
511 # Verify that the expected number of deployed releases are 495 # Verify that the expected number of deployed releases are
512 # tested with expected arguments. 496 # tested with expected arguments.
513 self.assertEqual( 497 self.assertEqual(
514 len(expected_test_release_for_success_calls), 498 len(expected_test_constructor_calls), mock_test.call_count)
515 mock_test_release_for_success.call_count) 499
516 mock_test_release_for_success.assert_has_calls( 500 mock_test.assert_has_calls(
517 expected_test_release_for_success_calls, any_order=any_order) 501 expected_test_constructor_calls, any_order=True)
518 502
519 _do_test() 503 _do_test()
520 504
diff --git a/armada/tests/unit/handlers/test_test.py b/armada/tests/unit/handlers/test_test.py
index 93afda5..6459ed3 100644
--- a/armada/tests/unit/handlers/test_test.py
+++ b/armada/tests/unit/handlers/test_test.py
@@ -14,8 +14,8 @@
14 14
15import mock 15import mock
16 16
17from armada.handlers import tiller
18from armada.handlers import test 17from armada.handlers import test
18from armada.handlers import tiller
19from armada.tests.unit import base 19from armada.tests.unit import base
20from armada.tests.test_utils import AttrDict 20from armada.tests.test_utils import AttrDict
21 21
@@ -32,7 +32,9 @@ class TestHandlerTestCase(base.ArmadaTestCase):
32 tiller_obj.test_release = mock.Mock() 32 tiller_obj.test_release = mock.Mock()
33 tiller_obj.test_release.return_value = AttrDict( 33 tiller_obj.test_release.return_value = AttrDict(
34 **{'results': results}) 34 **{'results': results})
35 success = test.test_release_for_success(tiller_obj, release) 35
36 test_handler = test.Test(release, tiller_obj)
37 success = test_handler.test_release_for_success()
36 38
37 self.assertEqual(expected_success, success) 39 self.assertEqual(expected_success, success)
38 40
@@ -62,3 +64,198 @@ class TestHandlerTestCase(base.ArmadaTestCase):
62 AttrDict(**{'status': test.TESTRUN_STATUS_SUCCESS}), 64 AttrDict(**{'status': test.TESTRUN_STATUS_SUCCESS}),
63 AttrDict(**{'status': test.TESTRUN_STATUS_RUNNING}) 65 AttrDict(**{'status': test.TESTRUN_STATUS_RUNNING})
64 ]) 66 ])
67
68 def test_cg_disabled(self):
69 """Test that tests are disabled when a chart group disables all
70 tests.
71 """
72 test_handler = test.Test(
73 release_name='release', tiller=mock.Mock(), cg_test_charts=False)
74
75 assert test_handler.test_enabled is False
76
77 def test_cg_disabled_test_key_enabled(self):
78 """Test that tests are enabled when a chart group disables all
79 tests and the deprecated, boolean `test` key is enabled.
80 """
81 test_handler = test.Test(
82 release_name='release',
83 tiller=mock.Mock(),
84 cg_test_charts=False,
85 test_values=True)
86
87 assert test_handler.test_enabled is True
88
89 def test_cg_disabled_test_values_enabled(self):
90 """Test that tests are enabled when a chart group disables all
91 tests and the `test.enabled` key is False.
92 """
93 test_values = {'enabled': True}
94
95 test_handler = test.Test(
96 release_name='release',
97 tiller=mock.Mock(),
98 cg_test_charts=False,
99 test_values=test_values)
100
101 assert test_handler.test_enabled is True
102
103 def test_cg_enabled_test_key_disabled(self):
104 """Test that tests are disabled when a chart group enables all
105 tests and the deprecated, boolean `test` key is disabled.
106 """
107 test_handler = test.Test(
108 release_name='release',
109 tiller=mock.Mock(),
110 cg_test_charts=True,
111 test_values=False)
112
113 assert test_handler.test_enabled is False
114
115 def test_cg_enabled_test_values_disabled(self):
116 """Test that tests are disabled when a chart group enables all
117 tests and the deprecated, boolean `test` key is disabled.
118 """
119 test_values = {'enabled': False}
120
121 test_handler = test.Test(
122 release_name='release',
123 tiller=mock.Mock(),
124 cg_test_charts=True,
125 test_values=test_values)
126
127 assert test_handler.test_enabled is False
128
129 def test_enable_all_cg_disabled(self):
130 """Test that tests are enabled when the `enable_all` parameter is
131 True and the chart group `test_enabled` key is disabled.
132 """
133 test_handler = test.Test(
134 release_name='release',
135 tiller=mock.Mock(),
136 cg_test_charts=False,
137 enable_all=True)
138
139 assert test_handler.test_enabled is True
140
141 def test_enable_all_test_key_disabled(self):
142 """Test that tests are enabled when the `enable_all` parameter is
143 True and the deprecated, boolean `test` key is disabled.
144 """
145 test_handler = test.Test(
146 release_name='release',
147 tiller=mock.Mock(),
148 enable_all=True,
149 test_values=False)
150
151 assert test_handler.test_enabled is True
152
153 def test_enable_all_test_values_disabled(self):
154 """Test that tests are enabled when the `enable_all` parameter is
155 True and the `test.enabled` key is False.
156 """
157 test_values = {'enabled': False}
158
159 test_handler = test.Test(
160 release_name='release',
161 tiller=mock.Mock(),
162 enable_all=True,
163 test_values=test_values)
164
165 assert test_handler.test_enabled is True
166
167 def test_deprecated_test_key_false(self):
168 """Test that tests can be disabled using the deprecated, boolean value
169 for a chart's test key.
170 """
171 test_handler = test.Test(
172 release_name='release', tiller=mock.Mock(), test_values=True)
173
174 assert test_handler.test_enabled
175
176 def test_deprecated_test_key_true(self):
177 """Test that cleanup is enabled by default when tests are enabled using
178 the deprecated, boolean value for a chart's `test` key.
179 """
180 test_handler = test.Test(
181 release_name='release', tiller=mock.Mock(), test_values=True)
182
183 assert test_handler.test_enabled is True
184 assert test_handler.cleanup is True
185
186 def test_tests_disabled(self):
187 """Test that tests are disabled by a chart's values using the
188 `test.enabled` path.
189 """
190 test_values = {'enabled': False}
191 test_handler = test.Test(
192 release_name='release',
193 tiller=mock.Mock(),
194 test_values=test_values)
195
196 assert test_handler.test_enabled is False
197
198 def test_tests_enabled(self):
199 """Test that cleanup is disabled (by default) when tests are enabled by
200 a chart's values using the `test.enabled` path.
201 """
202 test_values = {'enabled': True}
203 test_handler = test.Test(
204 release_name='release',
205 tiller=mock.Mock(),
206 test_values=test_values)
207
208 assert test_handler.test_enabled is True
209 assert test_handler.cleanup is False
210
211 def test_tests_enabled_cleanup_enabled(self):
212 """Test that the test handler uses the values provided by a chart's
213 `test` key.
214 """
215 test_values = {'enabled': True, 'options': {'cleanup': True}}
216
217 test_handler = test.Test(
218 release_name='release',
219 tiller=mock.Mock(),
220 test_values=test_values)
221
222 assert test_handler.test_enabled is True
223 assert test_handler.cleanup is True
224
225 def test_tests_enabled_cleanup_disabled(self):
226 """Test that the test handler uses the values provided by a chart's
227 `test` key.
228 """
229 test_values = {'enabled': True, 'options': {'cleanup': False}}
230
231 test_handler = test.Test(
232 release_name='release',
233 tiller=mock.Mock(),
234 test_values=test_values)
235
236 assert test_handler.test_enabled is True
237 assert test_handler.cleanup is False
238
239 def test_no_test_values(self):
240 """Test that the default values are enforced when no chart `test`
241 values are provided (i.e. tests are enabled and cleanup is disabled).
242 """
243 test_handler = test.Test(release_name='release', tiller=mock.Mock())
244
245 assert test_handler.test_enabled is True
246 assert test_handler.cleanup is False
247
248 def test_override_cleanup(self):
249 """Test that a cleanup value passed to the Test handler (i.e. from the
250 API/CLI) takes precedence over a chart's `test.cleanup` value.
251 """
252 test_values = {'enabled': True, 'options': {'cleanup': False}}
253
254 test_handler = test.Test(
255 release_name='release',
256 tiller=mock.Mock(),
257 cleanup=True,
258 test_values=test_values)
259
260 assert test_handler.test_enabled is True
261 assert test_handler.cleanup is True
diff --git a/armada/utils/release.py b/armada/utils/release.py
index dfd02df..dbd51d9 100644
--- a/armada/utils/release.py
+++ b/armada/utils/release.py
@@ -12,7 +12,7 @@
12# See the License for the specific language governing permissions and 12# See the License for the specific language governing permissions and
13# limitations under the License. 13# limitations under the License.
14 14
15from armada.handlers.test import get_test_suite_run_success 15from armada.handlers.test import Test
16 16
17 17
18def release_prefixer(prefix, release): 18def release_prefixer(prefix, release):
@@ -52,4 +52,4 @@ def get_last_test_result(release):
52 status = release.info.status 52 status = release.info.status
53 if not status.HasField('last_test_suite_run'): 53 if not status.HasField('last_test_suite_run'):
54 return None 54 return None
55 return get_test_suite_run_success(status.last_test_suite_run) 55 return Test.get_test_suite_run_success(status.last_test_suite_run)
diff --git a/doc/source/commands/test.rst b/doc/source/commands/test.rst
index 4892dc9..d305d98 100644
--- a/doc/source/commands/test.rst
+++ b/doc/source/commands/test.rst
@@ -24,6 +24,8 @@ Commands
24 $ armada test --release blog-1 24 $ armada test --release blog-1
25 25
26 Options: 26 Options:
27 --cleanup Delete test pods after test completion
28 --enable-all Run disabled chart tests
27 --file TEXT armada manifest 29 --file TEXT armada manifest
28 --release TEXT helm release 30 --release TEXT helm release
29 --tiller-host TEXT Tiller Host IP 31 --tiller-host TEXT Tiller Host IP
diff --git a/swagger/swaggerV3-api.yaml b/swagger/swaggerV3-api.yaml
index 8308b87..131ac6a 100644
--- a/swagger/swaggerV3-api.yaml
+++ b/swagger/swaggerV3-api.yaml
@@ -155,6 +155,7 @@ paths:
155 - $ref: "#/components/parameters/tiller-port" 155 - $ref: "#/components/parameters/tiller-port"
156 - $ref: "#/components/parameters/tiller-namespace" 156 - $ref: "#/components/parameters/tiller-namespace"
157 - $ref: "#/components/parameters/target-manifest" 157 - $ref: "#/components/parameters/target-manifest"
158 - $ref: "#/components/parameters/enable-all"
158 requestBody: 159 requestBody:
159 $ref: "#/components/requestBodies/manifest-body" 160 $ref: "#/components/requestBodies/manifest-body"
160 responses: 161 responses:
@@ -337,6 +338,13 @@ components:
337 description: Flag to allow for chart cleanup 338 description: Flag to allow for chart cleanup
338 schema: 339 schema:
339 type: boolean 340 type: boolean
341 enable-all:
342 in: query
343 name: enable_all
344 required: false
345 description: Flag to test disabled tests
346 schema:
347 type: boolean
340 dry-run: 348 dry-run:
341 in: query 349 in: query
342 name: dry_run 350 name: dry_run