Run helm tests by default

- Armada will now run helm tests by default, and the charts must
  disable tests if they choose. A helm test that does not exist
  is still a happy-path resolution.
- Documentation and schema updates to signify new deault behavior.
- Preparing to deprecate `test_charts` in ChartGroup processing.

Change-Id: I8e51e33a5d9559b11c2b75e122ecfc97af084ca4

armada/handlers/armada.py

@@ -254,11 +254,19 @@ class Armada(object):
             cg_name = chartgroup.get('name', '<missing name>')
             cg_desc = chartgroup.get('description', '<missing description>')
             cg_sequenced = chartgroup.get('sequenced', False)
-            cg_test_all_charts = chartgroup.get('test_charts', False)
+            LOG.info('Processing ChartGroup: %s (%s), sequenced=%s', cg_name,
-            LOG.info(
+                     cg_desc, cg_sequenced)
-                'Processing ChartGroup: %s (%s), sequenced=%s, '
+
-                'test_charts=%s', cg_name, cg_desc, cg_sequenced,
+            # TODO(MarshM): Deprecate the `test_charts` key
-                cg_test_all_charts)
+            cg_test_all_charts = chartgroup.get('test_charts')
+            if isinstance(cg_test_all_charts, bool):
+                LOG.warn('The ChartGroup `test_charts` key is deprecated, '
+                         'and support for this will be removed. See the '
+                         'Chart `test` key for more information.')
+            else:
+                # This key defaults to True. Individual charts must
+                # explicitly disable helm tests if they choose
+                cg_test_all_charts = True
 
             ns_label_set = set()
             tests_to_run = []
@@ -363,7 +371,8 @@ class Armada(object):
                              'instead.')
                     test_this_chart = test_chart_override
                 else:
-                    test_this_chart = test_chart_override['enabled']
+                    # NOTE: helm tests are enabled by default
+                    test_this_chart = test_chart_override.get('enabled', True)
                     test_cleanup = test_chart_override.get('options', {}).get(
                         'cleanup', False)
 
@@ -474,25 +483,19 @@ class Armada(object):
                              tiller_result.__dict__)
                     msg['install'].append(release_name)
 
-                # Sequenced ChartGroup should run tests after each Chart
+                # Keeping track of time remaining
                 timer = int(round(deadline - time.time()))
                 test_chart_args = (release_name, timer, test_cleanup)
                 if test_this_chart:
+                    # Sequenced ChartGroup should run tests after each Chart
                     if cg_sequenced:
                         LOG.info(
                             'Running sequenced test, timeout remaining: '
                             '%ss.', timer)
-                        if timer <= 0:
-                            reason = ('Timeout expired before testing '
-                                      'sequenced release %s' % release_name)
-                            LOG.error(reason)
-                            raise armada_exceptions.ArmadaTimeoutException(
-                                reason)
                         self._test_chart(*test_chart_args)
 
                     # Un-sequenced ChartGroup should run tests at the end
                     else:
-                        # Keeping track of time remaining
                         tests_to_run.append(
                             functools.partial(self._test_chart,
                                               *test_chart_args))
@@ -574,6 +577,12 @@ class Armada(object):
                 'release=%s with timeout %ss.', release_name, timeout)
             return True
 
+        if timeout <= 0:
+            reason = ('Timeout expired before testing '
+                      'release %s' % release_name)
+            LOG.error(reason)
+            raise armada_exceptions.ArmadaTimeoutException(reason)
+
         success = test_release_for_success(
             self.tiller, release_name, timeout=timeout, cleanup=cleanup)
         if success:

armada/schemas/armada-chart-schema.yaml

@@ -73,9 +73,7 @@ data:
                   type: boolean
               additionalProperties: false
           additionalProperties: false
-          required:
+    # TODO(MarshM): Deprecate this `timeout` in favor of `wait.timeout`
-            - enabled
-
     timeout:
       type: integer
     wait:

armada/schemas/armada-chartgroup-schema.yaml

@@ -27,6 +27,7 @@ data:
       type: string
     sequenced:
       type: boolean
+    # TODO(MarshM): Deprecate `test_charts`, it is no longer useful
     test_charts:
       type: boolean
     chart_group:

armada/tests/unit/handlers/test_armada.py

@@ -334,7 +334,7 @@ class ArmadaHandlerTestCase(base.ArmadaTestCase):
 
             chart_group = armada_obj.manifest['armada']['chart_groups'][0]
             charts = chart_group['chart_group']
-            cg_test_all_charts = chart_group.get('test_charts', False)
+            cg_test_all_charts = chart_group.get('test_charts', True)
 
             m_tiller = mock_tiller.return_value
             m_tiller.list_charts.return_value = known_releases

armada/tests/unit/resources/keystone-manifest.yaml

@@ -120,7 +120,6 @@ metadata:
 data:
   description: "Deploying OpenStack Keystone"
   sequenced: True
-  test_charts: False
   chart_group:
     - keystone
 ---

armada/tests/unit/resources/valid_armada_document.yaml

@@ -124,7 +124,6 @@ metadata:
 data:
   description: "Deploying OpenStack Keystone"
   sequenced: True
-  test_charts: False
   chart_group:
     - keystone
 ---

doc/source/operations/guide-build-armada-yaml.rst

@@ -52,9 +52,15 @@ armada/ChartGroup/v1
 +-----------------+----------+------------------------------------------------------------------------+
 | sequenced       | bool     | enables sequenced chart deployment in a group                          |
 +-----------------+----------+------------------------------------------------------------------------+
-| test_charts     | bool     | run pre-defined helm tests helm in a ChartGroup                        |
+| test_charts     | bool     | run pre-defined helm tests in a ChartGroup (DEPRECATED)                |
 +-----------------+----------+------------------------------------------------------------------------+
 
+.. DANGER::
+
+    DEPRECATION: The ``test_charts`` key will be removed, as Armada will run
+    helm tests for all charts by default.
+
+
 Chart Group Example
 ^^^^^^^^^^^^^^^^^^^
 
@@ -98,7 +104,7 @@ Chart
 | protected       | object   | do not delete FAILED releases when encountered from previous run (provide the         |
 |                 |          | 'continue_processing' bool to continue or halt execution (default: halt))             |
 +-----------------+----------+---------------------------------------------------------------------------------------+
-| test            | object   | Run helm tests on the chart after install/upgrade                                     |
+| test            | object   | Run helm tests on the chart after install/upgrade (default enabled)                   |
 +-----------------+----------+---------------------------------------------------------------------------------------+
 | install         | object   | install the chart into your Kubernetes cluster                                        |
 +-----------------+----------+---------------------------------------------------------------------------------------+
@@ -116,20 +122,25 @@ Chart
 Test
 ^^^^
 
-+-------------+----------+---------------------------------------------------------------+
++-------------+----------+--------------------------------------------------------------------+
-| keyword     | type     | action                                                        |
+| keyword     | type     | action                                                             |
-+=============+==========+===============================================================+
++=============+==========+====================================================================+
-| enabled     | bool     | whether to enable helm tests for this chart                   |
+| enabled     | bool     | whether to enable/disable helm tests for this chart (default True) |
-+-------------+----------+---------------------------------------------------------------+
++-------------+----------+--------------------------------------------------------------------+
-| options     | object   | options to pass through to helm                               |
+| options     | object   | options to pass through to helm                                    |
-+-------------+----------+---------------------------------------------------------------+
++-------------+----------+--------------------------------------------------------------------+
+
+.. note::
+
+    Armada will attempt to run helm tests by default. They may be disabled by
+    setting the ``enabled`` key to ``False``.
 
 .. DANGER::
 
     DEPRECATION: In addition to an object with the above fields, the ``test``
     key currently also supports ``bool``, which maps to ``enabled``, but this is
     deprecated and will be removed.  The ``cleanup`` option below is set to true
-    in this case for backward compatability.
+    in this case for backward compatibility.
 
 Test - Options
 ^^^^^^^^^^^^^^

examples/keystone-manifest.yaml

@@ -125,7 +125,6 @@ metadata:
 data:
   description: "Deploying OpenStack Keystone"
   sequenced: True
-  test_charts: False
   chart_group:
     - keystone
 ---