Fix and overhaul helm test integration

The helm test integration was severely broken, this fixes it by:

* correctly handle tiller test call response
* removes unnecessary call to tiller to get release content
* removes unnecessary call to k8s to check for test pod completion
* moves common logic into a test handler
* adds test coverage for the above
* adds logging for test results streamed from tiller

Change-Id: I09062387a1abc2fc3f6960f987c97248d9e1cb69

armada/api/controller/test.py

@@ -21,6 +21,7 @@ from oslo_config import cfg
 from armada import api
 from armada.common import policy
 from armada import const
+from armada.handlers.test import test_release_for_success
 from armada.handlers.tiller import Tiller
 from armada.handlers.manifest import Manifest
 from armada.utils.release import release_prefixer
@@ -44,7 +45,7 @@ class TestReleasesReleaseNameController(api.BaseResource):
                     'tiller_port') or CONF.tiller_port,
                 tiller_namespace=req.get_param(
                     'tiller_namespace', default=CONF.tiller_namespace))
-            tiller_resp = tiller.testing_release(release)
+            success = test_release_for_success(tiller, release)
         # TODO(fmontei): Provide more sensible exception(s) here.
         except Exception as e:
             err_message = 'Failed to test {}: {}'.format(release, e)
@@ -52,26 +53,18 @@ class TestReleasesReleaseNameController(api.BaseResource):
             return self.return_error(
                 resp, falcon.HTTP_500, message=err_message)
 
-        msg = {
+        if success:
-            'result': '',
+            msg = {
-            'message': ''
+                'result': 'PASSED: {}'.format(release),
-        }
+                'message': 'MESSAGE: Test Pass'
-
+            }
-        if tiller_resp:
-            test_status = getattr(
-                tiller_resp.info.status, 'last_test_suite_run', 'FAILED')
-
-            if test_status.result[0].status == 'PASSED':
-                msg['result'] = 'PASSED: {}'.format(release)
-                msg['message'] = 'MESSAGE: Test Pass'
-                self.logger.info(msg)
-            else:
-                msg['result'] = 'FAILED: {}'.format(release)
-                msg['message'] = 'MESSAGE: Test Fail'
-                self.logger.info(msg)
         else:
-            msg['result'] = 'FAILED: {}'.format(release)
+            msg = {
-            msg['message'] = 'MESSAGE: No test found'
+                'result': 'FAILED: {}'.format(release),
+                'message': 'MESSAGE: Test Fail'
+            }
+
+        self.logger.info(msg)
 
         resp.body = json.dumps(msg)
         resp.status = falcon.HTTP_200
@@ -174,15 +167,8 @@ class TestReleasesManifestController(api.BaseResource):
 
                 if release_name in known_releases:
                     self.logger.info('RUNNING: %s tests', release_name)
-                    resp = tiller.testing_release(release_name)
+                    success = test_release_for_success(tiller, release_name)
-
+                    if success:
-                    if not resp:
-                        continue
-
-                    test_status = getattr(
-                        resp.info.status, 'last_test_suite_run',
-                        'FAILED')
-                    if test_status.results[0].status:
                         self.logger.info("PASSED: %s", release_name)
                         message['test']['passed'].append(release_name)
                     else:

armada/cli/test.py

@@ -19,6 +19,7 @@ from oslo_config import cfg
 
 from armada.cli import CliAction
 from armada import const
+from armada.handlers.test import test_release_for_success
 from armada.handlers.manifest import Manifest
 from armada.handlers.tiller import Tiller
 from armada.utils.release import release_prefixer
@@ -112,15 +113,8 @@ class TestChartManifest(CliAction):
         if self.release:
             if not self.ctx.obj.get('api', False):
                 self.logger.info("RUNNING: %s tests", self.release)
-                resp = tiller.testing_release(self.release)
+                success = test_release_for_success(tiller, self.release)
-
+                if success:
-                if not resp:
-                    self.logger.info("FAILED: %s", self.release)
-                    return
-
-                test_status = getattr(resp.info.status, 'last_test_suite_run',
-                                      'FAILED')
-                if test_status.results[0].status:
                     self.logger.info("PASSED: %s", self.release)
                 else:
                     self.logger.info("FAILED: %s", self.release)
@@ -154,15 +148,9 @@ class TestChartManifest(CliAction):
 
                         if release_name in known_release_names:
                             self.logger.info('RUNNING: %s tests', release_name)
-                            resp = tiller.testing_release(release_name)
+                            success = test_release_for_success(
-
+                                tiller, release_name)
-                            if not resp:
+                            if success:
-                                continue
-
-                            test_status = getattr(
-                                resp.info.status, 'last_test_suite_run',
-                                'FAILED')
-                            if test_status.results[0].status:
                                 self.logger.info("PASSED: %s", release_name)
                             else:
                                 self.logger.info("FAILED: %s", release_name)

armada/exceptions/tiller_exceptions.py

@@ -118,6 +118,20 @@ class ReleaseException(TillerException):
         super(ReleaseException, self).__init__(message)
 
 
+class TestFailedException(TillerException):
+    '''
+    Exception that occurs when a release test fails.
+
+    **Troubleshoot:**
+    *Coming Soon*
+    '''
+
+    def __init__(self, release):
+        message = 'Test failed for release: {}'.format(release)
+
+        super(TestFailedException, self).__init__(message)
+
+
 class ChannelException(TillerException):
     '''
     Exception that occurs during a failed gRPC channel creation

armada/handlers/armada.py

@@ -27,6 +27,7 @@ from armada.exceptions import validate_exceptions
 from armada.handlers.chartbuilder import ChartBuilder
 from armada.handlers.manifest import Manifest
 from armada.handlers.override import Override
+from armada.handlers.test import test_release_for_success
 from armada.handlers.tiller import Tiller
 from armada.utils.release import release_prefixer
 from armada.utils import source
@@ -435,21 +436,22 @@ class Armada(object):
 
                 # Sequenced ChartGroup should run tests after each Chart
                 timer = int(round(deadline - time.time()))
-                if test_this_chart and cg_sequenced:
+                if test_this_chart:
-                    LOG.info('Running sequenced test, timeout remaining: %ss.',
+                    if cg_sequenced:
-                             timer)
+                        LOG.info('Running sequenced test, timeout remaining: '
-                    if timer <= 0:
+                                 '%ss.', timer)
-                        reason = ('Timeout expired before testing sequenced '
+                        if timer <= 0:
-                                  'release %s' % release_name)
+                            reason = ('Timeout expired before testing '
-                        LOG.error(reason)
+                                      'sequenced release %s' % release_name)
-                        raise armada_exceptions.ArmadaTimeoutException(reason)
+                            LOG.error(reason)
-                    self._test_chart(release_name, timer)
+                            raise armada_exceptions.ArmadaTimeoutException(
-                    # TODO(MarshM): handle test failure or timeout
+                                reason)
+                        self._test_chart(release_name, timer)
 
-                # Un-sequenced ChartGroup should run tests at the end
+                    # Un-sequenced ChartGroup should run tests at the end
-                elif test_this_chart:
+                    else:
-                    # Keeping track of time remaining
+                        # Keeping track of time remaining
-                    tests_to_run.append((release_name, timer))
+                        tests_to_run.append((release_name, timer))
 
             # End of Charts in ChartGroup
             LOG.info('All Charts applied in ChartGroup %s.', cg_name)
@@ -481,7 +483,6 @@ class Armada(object):
             # After entire ChartGroup is healthy, run any pending tests
             for (test, test_timer) in tests_to_run:
                 self._test_chart(test, test_timer)
-                # TODO(MarshM): handle test failure or timeout
 
         self.post_flight_ops()
 
@@ -532,16 +533,13 @@ class Armada(object):
                      'release=%s with timeout %ss.', release_name, timeout)
             return True
 
-        # TODO(MarshM): Fix testing, it's broken, and track timeout
+        success = test_release_for_success(
-        resp = self.tiller.testing_release(release_name, timeout=timeout)
+            self.tiller, release_name, timeout=timeout)
-        status = getattr(resp.info.status, 'last_test_suite_run', 'FAILED')
+        if success:
-        LOG.info("Test info.status: %s", status)
-        if resp:
             LOG.info("Test passed for release: %s", release_name)
-            return True
         else:
             LOG.info("Test failed for release: %s", release_name)
-            return False
+            raise tiller_exceptions.TestFailedException(release_name)
 
     def show_diff(self, chart, installed_chart, installed_values, target_chart,
                   target_values, msg):

armada/handlers/test.py

@@ -0,0 +1,29 @@
+# Copyright 2018 The Armada Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from armada import const
+
+TESTRUN_STATUS_UNKNOWN = 0
+TESTRUN_STATUS_SUCCESS = 1
+TESTRUN_STATUS_FAILURE = 2
+TESTRUN_STATUS_RUNNING = 3
+
+
+def test_release_for_success(tiller, release,
+                             timeout=const.DEFAULT_TILLER_TIMEOUT):
+    test_suite_run = tiller.test_release(release, timeout)
+    results = getattr(test_suite_run, 'results', [])
+    failed_results = [r for r in results if
+                      r.status != TESTRUN_STATUS_SUCCESS]
+    return len(failed_results) == 0

armada/handlers/tiller.py

@@ -32,12 +32,12 @@ from oslo_log import log as logging
 from armada import const
 from armada.exceptions import tiller_exceptions as ex
 from armada.handlers.k8s import K8s
+from armada.handlers import test
 from armada.utils.release import label_selectors
 
 TILLER_VERSION = b'2.7.2'
 GRPC_EPSILON = 60
 RELEASE_LIMIT = 128  # TODO(mark-burnett): There may be a better page size.
-RELEASE_RUNTEST_SUCCESS = 9
 
 # the standard gRPC max message size is 4MB
 # this expansion comes at a performance penalty
@@ -434,41 +434,43 @@ class Tiller(object):
             status = self.get_release_status(release)
             raise ex.ReleaseException(release, status, 'Install')
 
-    def testing_release(self, release, timeout=const.DEFAULT_TILLER_TIMEOUT,
+    def test_release(self, release, timeout=const.DEFAULT_TILLER_TIMEOUT,
-                        cleanup=True):
+                     cleanup=True):
         '''
         :param release - name of release to test
         :param timeout - runtime before exiting
         :param cleanup - removes testing pod created
 
-        :returns - results of test pod
+        :returns - test suite run object
         '''
 
         LOG.info("Running Helm test: release=%s, timeout=%s", release, timeout)
 
         try:
-
             stub = ReleaseServiceStub(self.channel)
 
+            # TODO: This timeout is redundant since we already have the grpc
+            # timeout below, and it's actually used by tiller for individual
+            # k8s operations not the overall request, should we:
+            #     1. Remove this timeout
+            #     2. Add `k8s_timeout=const.DEFAULT_K8S_TIMEOUT` arg and use
             release_request = TestReleaseRequest(
-                name=release, timeout=timeout, cleanup=cleanup)
+                name=release, timeout=timeout,
+                cleanup=cleanup)
 
-            content = self.get_release_content(release)
+            test_message_stream = stub.RunReleaseTest(
+                release_request, timeout, metadata=self.metadata)
 
-            if not len(content.release.hooks):
+            failed = 0
-                LOG.info('No test found')
+            for test_message in test_message_stream:
-                return False
+                if test_message.status == test.TESTRUN_STATUS_FAILURE:
+                    failed += 1
+                LOG.info(test_message.msg)
+            if failed:
+                LOG.info('{} test(s) failed'.format(failed))
 
-            if content.release.hooks[0].events[0] == RELEASE_RUNTEST_SUCCESS:
+            status = self.get_release_status(release)
-                test = stub.RunReleaseTest(
+            return status.info.status.last_test_suite_run
-                    release_request, timeout, metadata=self.metadata)
-
-                if test.running():
-                    self.k8s.wait_get_completed_podphase(release, timeout)
-
-                test.cancel()
-
-                return self.get_release_status(release)
 
         except Exception:
             LOG.exception('Error while testing release %s', release)

armada/tests/test_utils.py

@@ -107,3 +107,12 @@ def attr(**kwargs):
         return f
 
     return decorator
+
+
+class AttrDict(dict):
+    """Allows defining objects with attributes without defining a class
+
+    """
+    def __init__(self, *args, **kwargs):
+        super(AttrDict, self).__init__(*args, **kwargs)
+        self.__dict__ = self

armada/tests/unit/api/test_test_controller.py

@@ -55,59 +55,48 @@ class TestReleasesManifestControllerTest(base.BaseControllerTest):
 
 class TestReleasesReleaseNameControllerTest(base.BaseControllerTest):
 
+    @mock.patch.object(test, 'test_release_for_success')
     @mock.patch.object(test, 'Tiller')
-    def test_test_controller_test_pass(self, mock_tiller):
+    def test_test_controller_test_pass(
+            self, mock_tiller, mock_test_release_for_success):
         rules = {'armada:test_release': '@'}
         self.policy.set_rules(rules)
 
-        testing_release = mock_tiller.return_value.testing_release
+        mock_test_release_for_success.return_value = True
-        testing_release.return_value = mock.Mock(
-            **{'info.status.last_test_suite_run.result': [
-                mock.Mock(status='PASSED')]})
 
         resp = self.app.simulate_get('/api/v1.0/test/fake-release')
         self.assertEqual(200, resp.status_code)
         self.assertEqual('MESSAGE: Test Pass',
                          json.loads(resp.text)['message'])
 
+    @mock.patch.object(test, 'test_release_for_success')
     @mock.patch.object(test, 'Tiller')
-    def test_test_controller_test_fail(self, mock_tiller):
+    def test_test_controller_test_fail(
+            self, mock_tiller, mock_test_release_for_success):
         rules = {'armada:test_release': '@'}
         self.policy.set_rules(rules)
 
-        testing_release = mock_tiller.return_value.testing_release
+        mock_test_release_for_success.return_value = False
-        testing_release.return_value = mock.Mock(
-            **{'info.status.last_test_suite_run.result': [
-                mock.Mock(status='FAILED')]})
 
         resp = self.app.simulate_get('/api/v1.0/test/fake-release')
         self.assertEqual(200, resp.status_code)
         self.assertEqual('MESSAGE: Test Fail',
                          json.loads(resp.text)['message'])
 
-    @mock.patch.object(test, 'Tiller')
-    def test_test_controller_no_test_found(self, mock_tiller):
-        rules = {'armada:test_release': '@'}
-        self.policy.set_rules(rules)
-
-        mock_tiller.return_value.testing_release.return_value = None
-
-        resp = self.app.simulate_get('/api/v1.0/test/fake-release')
-        self.assertEqual(200, resp.status_code)
-        self.assertEqual('MESSAGE: No test found',
-                         json.loads(resp.text)['message'])
-
 
 @test_utils.attr(type=['negative'])
 class TestReleasesManifestControllerNegativeTest(base.BaseControllerTest):
 
     @mock.patch.object(test, 'Manifest')
     @mock.patch.object(test, 'Tiller')
-    def test_test_controller_tiller_exc_returns_500(self, mock_tiller, _):
+    @mock.patch.object(test, 'test_release_for_success')
+    def test_test_controller_tiller_exc_returns_500(
+            self, mock_test_release_for_success, mock_tiller, _):
         rules = {'armada:tests_manifest': '@'}
         self.policy.set_rules(rules)
 
         mock_tiller.side_effect = Exception
+        mock_test_release_for_success.side_effect = Exception
 
         resp = self.app.simulate_post('/api/v1.0/tests')
         self.assertEqual(500, resp.status_code)
@@ -187,11 +176,14 @@ class TestReleasesManifestControllerNegativeTest(base.BaseControllerTest):
 class TestReleasesReleaseNameControllerNegativeTest(base.BaseControllerTest):
 
     @mock.patch.object(test, 'Tiller')
-    def test_test_controller_tiller_exc_returns_500(self, mock_tiller):
+    @mock.patch.object(test, 'test_release_for_success')
+    def test_test_controller_tiller_exc_returns_500(
+            self, mock_test_release_for_success, mock_tiller):
         rules = {'armada:test_release': '@'}
         self.policy.set_rules(rules)
 
         mock_tiller.side_effect = Exception
+        mock_test_release_for_success.side_effect = Exception
 
         resp = self.app.simulate_get('/api/v1.0/test/fake-release')
         self.assertEqual(500, resp.status_code)

armada/tests/unit/handlers/test_armada.py

@@ -18,7 +18,9 @@ import yaml
 from armada import const
 from armada.handlers import armada
 from armada.tests.unit import base
+from armada.tests.test_utils import AttrDict
 from armada.utils.release import release_prefixer
+from armada.exceptions import tiller_exceptions
 from armada.exceptions.armada_exceptions import ProtectedReleaseException
 
 TEST_YAML = """
@@ -43,6 +45,27 @@ data:
     - example-chart-1
     - example-chart-2
     - example-chart-3
+    - example-chart-4
+---
+schema: armada/Chart/v1
+metadata:
+  schema: metadata/Document/v1
+  name: example-chart-4
+data:
+    chart_name: test_chart_4
+    release: test_chart_4
+    namespace: test
+    values: {}
+    source:
+      type: local
+      location: /tmp/dummy/armada
+      subpath: chart_4
+    dependencies: []
+    test: true
+    wait:
+      timeout: 10
+    upgrade:
+      no_hooks: false
 ---
 schema: armada/Chart/v1
 metadata:
@@ -64,9 +87,6 @@ data:
       timeout: 10
     upgrade:
       no_hooks: false
-      options:
-        force: true
-        recreate_pods: true
 ---
 schema: armada/Chart/v1
 metadata:
@@ -113,7 +133,8 @@ data:
 
 CHART_SOURCES = [('git://github.com/dummy/armada', 'chart_1'),
                  ('/tmp/dummy/armada', 'chart_2'),
-                 ('/tmp/dummy/armada', 'chart_3')]
+                 ('/tmp/dummy/armada', 'chart_3'),
+                 ('/tmp/dummy/armada', 'chart_4')]
 
 
 class ArmadaHandlerTestCase(base.ArmadaTestCase):
@@ -195,13 +216,31 @@ class ArmadaHandlerTestCase(base.ArmadaTestCase):
                                         'timeout': 10
                                     },
                                     'upgrade': {
-                                        'no_hooks': False,
+                                        'no_hooks': False
-                                        'options': {
-                                            'force': True,
-                                            'recreate_pods': True
-                                        }
                                     }
                                 }
+                            },
+                            {
+                                'chart': {
+                                    'dependencies': [],
+                                    'chart_name': 'test_chart_4',
+                                    'namespace': 'test',
+                                    'release': 'test_chart_4',
+                                    'source': {
+                                        'location': '/tmp/dummy/armada',
+                                        'subpath': 'chart_4',
+                                        'type': 'local'
+                                    },
+                                    'source_dir': CHART_SOURCES[3],
+                                    'values': {},
+                                    'wait': {
+                                        'timeout': 10
+                                    },
+                                    'upgrade': {
+                                        'no_hooks': False
+                                    },
+                                    'test': True
+                                }
                             }
                         ],
                         'description': 'this is a test',
@@ -261,26 +300,41 @@ class ArmadaHandlerTestCase(base.ArmadaTestCase):
                     mock_source.source_cleanup.assert_called_with(
                         CHART_SOURCES[counter][0])
 
-    def _test_sync(self, known_releases):
+    def _test_sync(self, known_releases, test_success=True,
+                   test_failure_to_run=False):
         """Test install functionality from the sync() method."""
 
         @mock.patch.object(armada.Armada, 'post_flight_ops')
         @mock.patch.object(armada.Armada, 'pre_flight_ops')
         @mock.patch('armada.handlers.armada.ChartBuilder')
         @mock.patch('armada.handlers.armada.Tiller')
-        def _do_test(mock_tiller, mock_chartbuilder, mock_pre_flight,
+        @mock.patch.object(armada, 'test_release_for_success')
-                     mock_post_flight):
+        def _do_test(mock_test_release_for_success, mock_tiller,
+                     mock_chartbuilder, mock_pre_flight, mock_post_flight):
             # Instantiate Armada object.
             yaml_documents = list(yaml.safe_load_all(TEST_YAML))
             armada_obj = armada.Armada(yaml_documents)
             armada_obj.show_diff = mock.Mock()
 
-            charts = armada_obj.manifest['armada']['chart_groups'][0][
+            chart_group = armada_obj.manifest['armada']['chart_groups'][0]
-                'chart_group']
+            charts = chart_group['chart_group']
 
             m_tiller = mock_tiller.return_value
             m_tiller.list_charts.return_value = known_releases
 
+            if test_failure_to_run:
+                def fail(tiller, release, timeout=None):
+                    status = AttrDict(**{
+                        'info': AttrDict(**{
+                            'Description': 'Failed'
+                        })
+                    })
+                    raise tiller_exceptions.ReleaseException(
+                        release, status, 'Test')
+                mock_test_release_for_success.side_effect = fail
+            else:
+                mock_test_release_for_success.return_value = test_success
+
             # Stub out irrelevant methods called by `armada.sync()`.
             mock_chartbuilder.get_source_path.return_value = None
             mock_chartbuilder.get_helm_chart.return_value = None
@@ -290,6 +344,7 @@ class ArmadaHandlerTestCase(base.ArmadaTestCase):
             expected_install_release_calls = []
             expected_update_release_calls = []
             expected_uninstall_release_calls = []
+            expected_test_release_for_success_calls = []
 
             for c in charts:
                 chart = c['chart']
@@ -371,6 +426,18 @@ class ArmadaHandlerTestCase(base.ArmadaTestCase):
                                     timeout=chart['wait']['timeout']
                                 )
                             )
+                test_this_chart = chart.get(
+                    'test',
+                    chart_group.get('test_charts', False))
+
+                if test_this_chart:
+                    expected_test_release_for_success_calls.append(
+                        mock.call(
+                            m_tiller,
+                            release_name,
+                            timeout=mock.ANY
+                        )
+                    )
 
             # Verify that at least 1 release is either installed or updated.
             self.assertTrue(
@@ -388,6 +455,18 @@ class ArmadaHandlerTestCase(base.ArmadaTestCase):
                              m_tiller.update_release.call_count)
             m_tiller.update_release.assert_has_calls(
                 expected_update_release_calls)
+            # Verify that the expected number of deployed releases are
+            # uninstalled with expected arguments.
+            self.assertEqual(len(expected_uninstall_release_calls),
+                             m_tiller.uninstall_release.call_count)
+            m_tiller.uninstall_release.assert_has_calls(
+                expected_uninstall_release_calls)
+            # Verify that the expected number of deployed releases are
+            # tested with expected arguments.
+            self.assertEqual(len(expected_test_release_for_success_calls),
+                             mock_test_release_for_success.call_count)
+            mock_test_release_for_success.assert_has_calls(
+                expected_test_release_for_success_calls)
 
         _do_test()
 
@@ -458,6 +537,22 @@ class ArmadaHandlerTestCase(base.ArmadaTestCase):
             ProtectedReleaseException,
             _test_method)
 
+    def test_armada_sync_test_failure(self):
+        def _test_method():
+            self._test_sync([], test_success=False)
+
+        self.assertRaises(
+            tiller_exceptions.TestFailedException,
+            _test_method)
+
+    def test_armada_sync_test_failure_to_run(self):
+        def _test_method():
+            self._test_sync([], test_failure_to_run=True)
+
+        self.assertRaises(
+            tiller_exceptions.ReleaseException,
+            _test_method)
+
     @mock.patch.object(armada.Armada, 'post_flight_ops')
     @mock.patch.object(armada.Armada, 'pre_flight_ops')
     @mock.patch('armada.handlers.armada.ChartBuilder')

armada/tests/unit/handlers/test_test.py

@@ -0,0 +1,80 @@
+# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import mock
+
+from armada.handlers import tiller
+from armada.handlers import test
+from armada.tests.unit import base
+from armada.tests.test_utils import AttrDict
+
+
+class TestHandlerTestCase(base.ArmadaTestCase):
+
+    def _test_test_release_for_success(self, expected_success, results):
+
+        @mock.patch('armada.handlers.tiller.K8s')
+        def do_test(_):
+            tiller_obj = tiller.Tiller('host', '8080', None)
+            release = 'release'
+
+            tiller_obj.test_release = mock.Mock()
+            tiller_obj.test_release.return_value = AttrDict(**{
+                'results': results
+            })
+            success = test.test_release_for_success(tiller_obj, release)
+
+            self.assertEqual(expected_success, success)
+
+        do_test()
+
+    def test_no_results(self):
+        self._test_test_release_for_success(True, [])
+
+    def test_unknown(self):
+        self._test_test_release_for_success(False, [
+            AttrDict(**{
+                'status': test.TESTRUN_STATUS_SUCCESS
+            }),
+            AttrDict(**{
+                'status': test.TESTRUN_STATUS_UNKNOWN
+            })
+        ])
+
+    def test_success(self):
+        self._test_test_release_for_success(True, [
+            AttrDict(**{
+                'status': test.TESTRUN_STATUS_SUCCESS
+            })
+        ])
+
+    def test_failure(self):
+        self._test_test_release_for_success(False, [
+            AttrDict(**{
+                'status': test.TESTRUN_STATUS_SUCCESS
+            }),
+            AttrDict(**{
+                'status': test.TESTRUN_STATUS_FAILURE
+            })
+        ])
+
+    def test_running(self):
+        self._test_test_release_for_success(False, [
+            AttrDict(**{
+                'status': test.TESTRUN_STATUS_SUCCESS
+            }),
+            AttrDict(**{
+                'status': test.TESTRUN_STATUS_RUNNING
+            })
+        ])

armada/tests/unit/handlers/test_tiller.py

@@ -17,7 +17,9 @@ from mock import MagicMock
 
 from armada.exceptions import tiller_exceptions as ex
 from armada.handlers import tiller
+from armada.handlers import test
 from armada.tests.unit import base
+from armada.tests.test_utils import AttrDict
 
 
 class TillerTestCase(base.ArmadaTestCase):
@@ -421,8 +423,80 @@ class TillerTestCase(base.ArmadaTestCase):
 
         self.assertEqual(expected_result, result)
 
+    def _test_test_release(self, grpc_response_mock):
 
-class AttrDict(dict):
+        @mock.patch('armada.handlers.tiller.K8s')
-    def __init__(self, *args, **kwargs):
+        @mock.patch('armada.handlers.tiller.grpc')
-        super(AttrDict, self).__init__(*args, **kwargs)
+        @mock.patch('armada.handlers.tiller.Config')
-        self.__dict__ = self
+        @mock.patch.object(tiller, 'TestReleaseRequest')
+        @mock.patch.object(tiller, 'ReleaseServiceStub')
+        def do_test(self, mock_release_service_stub,
+                    mock_test_release_request, mock_config,
+                    _, __):
+            tiller_obj = tiller.Tiller('host', '8080', None)
+            release = 'release'
+            test_suite_run = {}
+
+            mock_release_service_stub.return_value.RunReleaseTest\
+                .return_value = grpc_response_mock
+
+            tiller_obj.get_release_status = mock.Mock()
+            tiller_obj.get_release_status.return_value = AttrDict(**{
+                'info': AttrDict(**{
+                    'status': AttrDict(**{
+                        'last_test_suite_run': test_suite_run
+                    }),
+                    'Description': 'Failed'
+                })
+            })
+
+            result = tiller_obj.test_release(release)
+
+            self.assertEqual(test_suite_run, result)
+
+        do_test(self)
+
+    def test_test_release_no_tests(self):
+        self._test_test_release([
+            AttrDict(**{
+                'msg': 'No Tests Found',
+                'status': test.TESTRUN_STATUS_UNKNOWN
+            })
+        ])
+
+    def test_test_release_success(self):
+        self._test_test_release([
+            AttrDict(**{
+                'msg': 'RUNNING: ...',
+                'status': test.TESTRUN_STATUS_RUNNING
+            }),
+            AttrDict(**{
+                'msg': 'SUCCESS: ...',
+                'status': test.TESTRUN_STATUS_SUCCESS
+            })
+        ])
+
+    def test_test_release_failure(self):
+        self._test_test_release([
+            AttrDict(**{
+                'msg': 'RUNNING: ...',
+                'status': test.TESTRUN_STATUS_RUNNING
+            }),
+            AttrDict(**{
+                'msg': 'FAILURE: ...',
+                'status': test.TESTRUN_STATUS_FAILURE
+            })
+        ])
+
+    def test_test_release_failure_to_run(self):
+        class Iterator:
+            def __iter__(self):
+                return self
+
+            def __next__(self):
+                raise Exception
+
+        def test():
+            self._test_test_release(Iterator())
+
+        self.assertRaises(ex.ReleaseException, test)