summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorDrew Walters <drewwalters96@gmail.com>2018-11-16 16:21:00 +0000
committerDrew Walters <drewwalters96@gmail.com>2019-02-20 17:03:01 -0600
commit9a7c1f4006558226997c30af64b0f3bd0c666718 (patch)
tree30b8b879d4cb3b19b82cafcb0eeb0de4b1b9c05b
parent7f26bbcd59f97ab090516fe732912ebe7851eb30 (diff)
test: Add test-specific timeout option
Currently, tests executed during chart deployment use the wait timeout value, `wait.timeout`. This value can be too large of a timeout value for Helm tests. This change introduces a timeout for tests, `test.timeout` that is only used as a timeout for running Helm tests for a release. Story: 2003899 Depends-On: https://review.openstack.org/618355 Change-Id: Iee746444d5aede0b84b1805eb19f59f0f03c8f9e
Notes
Notes (review): Code-Review+2: Sean Eagan <sean.eagan@att.com> Code-Review+1: Nishant Kumar <nishant.e.kumar@ericsson.com> Code-Review+2: Scott Hussey <sthussey@att.com> Workflow+1: Scott Hussey <sthussey@att.com> Verified+2: Zuul Submitted-by: Zuul Submitted-at: Thu, 21 Feb 2019 17:30:41 +0000 Reviewed-on: https://review.openstack.org/618585 Project: openstack/airship-armada Branch: refs/heads/master
-rw-r--r--armada/const.py1
-rw-r--r--armada/handlers/chart_deploy.py15
-rw-r--r--armada/handlers/test.py17
-rw-r--r--armada/schemas/armada-chart-schema.yaml2
-rw-r--r--armada/tests/unit/handlers/test_test.py40
-rw-r--r--doc/source/operations/guide-build-armada-yaml.rst2
6 files changed, 59 insertions, 18 deletions
diff --git a/armada/const.py b/armada/const.py
index 03dd16b..4a95203 100644
--- a/armada/const.py
+++ b/armada/const.py
@@ -24,6 +24,7 @@ KEYWORD_RELEASE = 'release'
24 24
25# Armada 25# Armada
26DEFAULT_CHART_TIMEOUT = 900 26DEFAULT_CHART_TIMEOUT = 900
27DEFAULT_TEST_TIMEOUT = 300
27 28
28# Tiller 29# Tiller
29DEFAULT_TILLER_TIMEOUT = 300 30DEFAULT_TILLER_TIMEOUT = 300
diff --git a/armada/handlers/chart_deploy.py b/armada/handlers/chart_deploy.py
index ba31b60..139070d 100644
--- a/armada/handlers/chart_deploy.py
+++ b/armada/handlers/chart_deploy.py
@@ -244,25 +244,18 @@ class ChartDeploy(object):
244 run_test = test_handler.test_enabled and (just_deployed or 244 run_test = test_handler.test_enabled and (just_deployed or
245 not last_test_passed) 245 not last_test_passed)
246 if run_test: 246 if run_test:
247 timer = int(round(deadline - time.time())) 247 self._test_chart(release_name, test_handler)
248 self._test_chart(release_name, timer, test_handler)
249 248
250 return result 249 return result
251 250
252 def _test_chart(self, release_name, timeout, test_handler): 251 def _test_chart(self, release_name, test_handler):
253 if self.dry_run: 252 if self.dry_run:
254 LOG.info( 253 LOG.info(
255 'Skipping test during `dry-run`, would have tested ' 254 'Skipping test during `dry-run`, would have tested '
256 'release=%s with timeout %ss.', release_name, timeout) 255 'release=%s', release_name)
257 return True 256 return True
258 257
259 if timeout <= 0: 258 success = test_handler.test_release_for_success()
260 reason = ('Timeout expired before testing '
261 'release %s' % release_name)
262 LOG.error(reason)
263 raise armada_exceptions.ArmadaTimeoutException(reason)
264
265 success = test_handler.test_release_for_success(timeout=timeout)
266 if not success: 259 if not success:
267 raise tiller_exceptions.TestFailedException(release_name) 260 raise tiller_exceptions.TestFailedException(release_name)
268 261
diff --git a/armada/handlers/test.py b/armada/handlers/test.py
index 94c9479..2beab06 100644
--- a/armada/handlers/test.py
+++ b/armada/handlers/test.py
@@ -58,6 +58,8 @@ class Test(object):
58 58
59 test_values = self.chart.get('test', None) 59 test_values = self.chart.get('test', None)
60 60
61 self.timeout = const.DEFAULT_TEST_TIMEOUT
62
61 # NOTE(drewwalters96): Support the chart_group `test_charts` key until 63 # NOTE(drewwalters96): Support the chart_group `test_charts` key until
62 # its deprecation period ends. The `test.enabled`, `enable_all` flag, 64 # its deprecation period ends. The `test.enabled`, `enable_all` flag,
63 # and deprecated, boolean `test` key override this value if provided. 65 # and deprecated, boolean `test` key override this value if provided.
@@ -79,6 +81,7 @@ class Test(object):
79 # provided. 81 # provided.
80 if self.cleanup is None: 82 if self.cleanup is None:
81 self.cleanup = True 83 self.cleanup = True
84
82 elif test_values: 85 elif test_values:
83 test_enabled_opt = test_values.get('enabled') 86 test_enabled_opt = test_values.get('enabled')
84 if test_enabled_opt is not None: 87 if test_enabled_opt is not None:
@@ -90,6 +93,8 @@ class Test(object):
90 if self.cleanup is None: 93 if self.cleanup is None:
91 test_options = test_values.get('options', {}) 94 test_options = test_values.get('options', {})
92 self.cleanup = test_options.get('cleanup', False) 95 self.cleanup = test_options.get('cleanup', False)
96
97 self.timeout = test_values.get('timeout', self.timeout)
93 else: 98 else:
94 # Default cleanup value 99 # Default cleanup value
95 if self.cleanup is None: 100 if self.cleanup is None:
@@ -98,16 +103,14 @@ class Test(object):
98 if enable_all: 103 if enable_all:
99 self.test_enabled = True 104 self.test_enabled = True
100 105
101 def test_release_for_success(self, timeout=const.DEFAULT_TILLER_TIMEOUT): 106 def test_release_for_success(self):
102 """Run the Helm tests corresponding to a release for success (i.e. exit 107 """Run the Helm tests corresponding to a release for success (i.e. exit
103 code 0). 108 code 0).
104 109
105 :param timeout: Timeout value for a release's tests completion 110 :return: Helm test suite run result
106 :type timeout: int
107
108 :rtype: Helm test suite run result
109 """ 111 """
110 LOG.info('RUNNING: %s tests', self.release_name) 112 LOG.info('RUNNING: %s tests with timeout=%ds', self.release_name,
113 self.timeout)
111 114
112 try: 115 try:
113 self.delete_test_pods() 116 self.delete_test_pods()
@@ -116,7 +119,7 @@ class Test(object):
116 self.release_name) 119 self.release_name)
117 120
118 test_suite_run = self.tiller.test_release( 121 test_suite_run = self.tiller.test_release(
119 self.release_name, timeout=timeout, cleanup=self.cleanup) 122 self.release_name, timeout=self.timeout, cleanup=self.cleanup)
120 123
121 success = get_test_suite_run_success(test_suite_run) 124 success = get_test_suite_run_success(test_suite_run)
122 if success: 125 if success:
diff --git a/armada/schemas/armada-chart-schema.yaml b/armada/schemas/armada-chart-schema.yaml
index 98018f4..353eedc 100644
--- a/armada/schemas/armada-chart-schema.yaml
+++ b/armada/schemas/armada-chart-schema.yaml
@@ -66,6 +66,8 @@ data:
66 properties: 66 properties:
67 enabled: 67 enabled:
68 type: boolean 68 type: boolean
69 timeout:
70 type: integer
69 options: 71 options:
70 type: object 72 type: object
71 properties: 73 properties:
diff --git a/armada/tests/unit/handlers/test_test.py b/armada/tests/unit/handlers/test_test.py
index 9bf4544..647366d 100644
--- a/armada/tests/unit/handlers/test_test.py
+++ b/armada/tests/unit/handlers/test_test.py
@@ -14,6 +14,8 @@
14 14
15import mock 15import mock
16 16
17from armada import const
18
17from armada.handlers import test 19from armada.handlers import test
18from armada.handlers import tiller 20from armada.handlers import tiller
19from armada.tests.unit import base 21from armada.tests.unit import base
@@ -188,6 +190,16 @@ class TestHandlerTestCase(base.ArmadaTestCase):
188 assert test_handler.test_enabled is True 190 assert test_handler.test_enabled is True
189 assert test_handler.cleanup is True 191 assert test_handler.cleanup is True
190 192
193 def test_deprecated_test_key_timeout(self):
194 """Test that the default Tiller timeout is used when tests are enabled
195 using the deprecated, boolean value for a chart's `test` key.
196 """
197 mock_tiller = mock.Mock()
198 test_handler = test.Test(
199 chart={'test': True}, release_name='release', tiller=mock_tiller)
200
201 assert test_handler.timeout == const.DEFAULT_TEST_TIMEOUT
202
191 def test_tests_disabled(self): 203 def test_tests_disabled(self):
192 """Test that tests are disabled by a chart's values using the 204 """Test that tests are disabled by a chart's values using the
193 `test.enabled` path. 205 `test.enabled` path.
@@ -276,3 +288,31 @@ class TestHandlerTestCase(base.ArmadaTestCase):
276 288
277 assert test_handler.test_enabled is True 289 assert test_handler.test_enabled is True
278 assert test_handler.cleanup is True 290 assert test_handler.cleanup is True
291
292 def test_default_timeout_value(self):
293 """Test that the default timeout value is used if a test timeout value,
294 `test.timeout` is not provided.
295 """
296 test_handler = test.Test(
297 chart={'test': {
298 'enabled': True
299 }},
300 release_name='release',
301 tiller=mock.Mock(),
302 cleanup=True)
303
304 assert test_handler.timeout == const.DEFAULT_TILLER_TIMEOUT
305
306 def test_timeout_value(self):
307 """Test that a chart's test timeout value, `test.timeout` overrides the
308 default test timeout.
309 """
310 chart = {'test': {'enabled': True, 'timeout': 800}}
311
312 test_handler = test.Test(
313 chart=chart,
314 release_name='release',
315 tiller=mock.Mock(),
316 cleanup=True)
317
318 assert test_handler.timeout is chart['test']['timeout']
diff --git a/doc/source/operations/guide-build-armada-yaml.rst b/doc/source/operations/guide-build-armada-yaml.rst
index 2743cb5..ae5c7ce 100644
--- a/doc/source/operations/guide-build-armada-yaml.rst
+++ b/doc/source/operations/guide-build-armada-yaml.rst
@@ -174,6 +174,8 @@ Run helm tests on the chart after install/upgrade.
174+=============+==========+====================================================================+ 174+=============+==========+====================================================================+
175| enabled | bool | whether to enable/disable helm tests for this chart (default True) | 175| enabled | bool | whether to enable/disable helm tests for this chart (default True) |
176+-------------+----------+--------------------------------------------------------------------+ 176+-------------+----------+--------------------------------------------------------------------+
177| timeout | int | time (in sec) to wait for completion of Helm tests |
178+-------------+----------+--------------------------------------------------------------------+
177| options | object | See `Test Options`_. | 179| options | object | See `Test Options`_. |
178+-------------+----------+--------------------------------------------------------------------+ 180+-------------+----------+--------------------------------------------------------------------+
179 181