test: Add test-specific timeout option

Currently, tests executed during chart deployment use the wait timeout
value, `wait.timeout`. This value can be too large of a timeout value
for Helm tests. This change introduces a timeout for tests,
`test.timeout` that is only used as a timeout for running Helm tests for
a release.

Story: 2003899

Depends-On: https://review.openstack.org/618355
Change-Id: Iee746444d5aede0b84b1805eb19f59f0f03c8f9e
This commit is contained in:
Drew Walters 2018-11-16 16:21:00 +00:00
parent 7f26bbcd59
commit 9a7c1f4006
6 changed files with 59 additions and 18 deletions

View File

@ -24,6 +24,7 @@ KEYWORD_RELEASE = 'release'
# Armada
DEFAULT_CHART_TIMEOUT = 900
DEFAULT_TEST_TIMEOUT = 300
# Tiller
DEFAULT_TILLER_TIMEOUT = 300

View File

@ -244,25 +244,18 @@ class ChartDeploy(object):
run_test = test_handler.test_enabled and (just_deployed or
not last_test_passed)
if run_test:
timer = int(round(deadline - time.time()))
self._test_chart(release_name, timer, test_handler)
self._test_chart(release_name, test_handler)
return result
def _test_chart(self, release_name, timeout, test_handler):
def _test_chart(self, release_name, test_handler):
if self.dry_run:
LOG.info(
'Skipping test during `dry-run`, would have tested '
'release=%s with timeout %ss.', release_name, timeout)
'release=%s', release_name)
return True
if timeout <= 0:
reason = ('Timeout expired before testing '
'release %s' % release_name)
LOG.error(reason)
raise armada_exceptions.ArmadaTimeoutException(reason)
success = test_handler.test_release_for_success(timeout=timeout)
success = test_handler.test_release_for_success()
if not success:
raise tiller_exceptions.TestFailedException(release_name)

View File

@ -58,6 +58,8 @@ class Test(object):
test_values = self.chart.get('test', None)
self.timeout = const.DEFAULT_TEST_TIMEOUT
# NOTE(drewwalters96): Support the chart_group `test_charts` key until
# its deprecation period ends. The `test.enabled`, `enable_all` flag,
# and deprecated, boolean `test` key override this value if provided.
@ -79,6 +81,7 @@ class Test(object):
# provided.
if self.cleanup is None:
self.cleanup = True
elif test_values:
test_enabled_opt = test_values.get('enabled')
if test_enabled_opt is not None:
@ -90,6 +93,8 @@ class Test(object):
if self.cleanup is None:
test_options = test_values.get('options', {})
self.cleanup = test_options.get('cleanup', False)
self.timeout = test_values.get('timeout', self.timeout)
else:
# Default cleanup value
if self.cleanup is None:
@ -98,16 +103,14 @@ class Test(object):
if enable_all:
self.test_enabled = True
def test_release_for_success(self, timeout=const.DEFAULT_TILLER_TIMEOUT):
def test_release_for_success(self):
"""Run the Helm tests corresponding to a release for success (i.e. exit
code 0).
:param timeout: Timeout value for a release's tests completion
:type timeout: int
:rtype: Helm test suite run result
:return: Helm test suite run result
"""
LOG.info('RUNNING: %s tests', self.release_name)
LOG.info('RUNNING: %s tests with timeout=%ds', self.release_name,
self.timeout)
try:
self.delete_test_pods()
@ -116,7 +119,7 @@ class Test(object):
self.release_name)
test_suite_run = self.tiller.test_release(
self.release_name, timeout=timeout, cleanup=self.cleanup)
self.release_name, timeout=self.timeout, cleanup=self.cleanup)
success = get_test_suite_run_success(test_suite_run)
if success:

View File

@ -66,6 +66,8 @@ data:
properties:
enabled:
type: boolean
timeout:
type: integer
options:
type: object
properties:

View File

@ -14,6 +14,8 @@
import mock
from armada import const
from armada.handlers import test
from armada.handlers import tiller
from armada.tests.unit import base
@ -188,6 +190,16 @@ class TestHandlerTestCase(base.ArmadaTestCase):
assert test_handler.test_enabled is True
assert test_handler.cleanup is True
def test_deprecated_test_key_timeout(self):
"""Test that the default Tiller timeout is used when tests are enabled
using the deprecated, boolean value for a chart's `test` key.
"""
mock_tiller = mock.Mock()
test_handler = test.Test(
chart={'test': True}, release_name='release', tiller=mock_tiller)
assert test_handler.timeout == const.DEFAULT_TEST_TIMEOUT
def test_tests_disabled(self):
"""Test that tests are disabled by a chart's values using the
`test.enabled` path.
@ -276,3 +288,31 @@ class TestHandlerTestCase(base.ArmadaTestCase):
assert test_handler.test_enabled is True
assert test_handler.cleanup is True
def test_default_timeout_value(self):
"""Test that the default timeout value is used if a test timeout value,
`test.timeout` is not provided.
"""
test_handler = test.Test(
chart={'test': {
'enabled': True
}},
release_name='release',
tiller=mock.Mock(),
cleanup=True)
assert test_handler.timeout == const.DEFAULT_TILLER_TIMEOUT
def test_timeout_value(self):
"""Test that a chart's test timeout value, `test.timeout` overrides the
default test timeout.
"""
chart = {'test': {'enabled': True, 'timeout': 800}}
test_handler = test.Test(
chart=chart,
release_name='release',
tiller=mock.Mock(),
cleanup=True)
assert test_handler.timeout is chart['test']['timeout']

View File

@ -174,6 +174,8 @@ Run helm tests on the chart after install/upgrade.
+=============+==========+====================================================================+
| enabled | bool | whether to enable/disable helm tests for this chart (default True) |
+-------------+----------+--------------------------------------------------------------------+
| timeout | int | time (in sec) to wait for completion of Helm tests |
+-------------+----------+--------------------------------------------------------------------+
| options | object | See `Test Options`_. |
+-------------+----------+--------------------------------------------------------------------+