Fix Deckhand render throwing exception on missing sub source

This PS resolves a recent issue with Deckhand in which missing
substitution sources cause Promenade to fail during genesis,
while using Deckhand to render documents. The fix involves
introducing a new flag called fail_on_missing_sub_src which
if False logs a warning rather than raises an exception
in the event that a substitution source document is missing.

Also adds better exception handling and logging around
Deckhand.

Example error:

Traceback (most recent call last):
  File "/usr/local/bin/promenade", line 10, in <module>
    sys.exit(promenade())
  File "/usr/local/lib/python3.6/site-packages/click/core.py", line 722, in __call__
    return self.main(*args, **kwargs)
  File "/usr/local/lib/python3.6/site-packages/click/core.py", line 697, in main
    rv = self.invoke(ctx)
  File "/usr/local/lib/python3.6/site-packages/click/core.py", line 1066, in invoke
    return _process_result(sub_ctx.command.invoke(sub_ctx))
  File "/usr/local/lib/python3.6/site-packages/click/core.py", line 895, in invoke
    return ctx.invoke(self.callback, **ctx.params)
  File "/usr/local/lib/python3.6/site-packages/click/core.py", line 535, in invoke
    return callback(*args, **kwargs)
  File "/opt/promenade/promenade/cli.py", line 55, in genereate_certs
    debug=debug, streams=config_files, substitute=True, validate=False)
  File "/opt/promenade/promenade/config.py", line 49, in from_streams
    return cls(documents=documents, **kwargs)
  File "/opt/promenade/promenade/config.py", line 29, in __init__
    documents = [dict(d) for d in deckhand_eng.render()]
  File "/usr/local/lib/python3.6/site-packages/deckhand/engine/layering.py", line 485, in render
    self.secrets_substitution.substitute_all(doc))
  File "/usr/local/lib/python3.6/site-packages/deckhand/engine/secrets_manager.py", line 182, in substitute_all
    document_name=document.name)

Depends-On: https://review.gerrithub.io/#/c/400880/
Change-Id: I4486535d4555ece54eb4d47bfb56472250f97ab4

promenade/config.py

@@ -5,6 +5,7 @@ import jsonpath_ng
 import yaml
 
 from deckhand.engine import layering
+from deckhand import errors as dh_errors
 
 __all__ = ['Configuration']
 
@@ -24,9 +25,18 @@ class Configuration:
         LOG.info("Building config from %d documents." % len(documents))
         if substitute:
             LOG.info("Rendering documents via Deckhand engine.")
-            deckhand_eng = layering.DocumentLayering(
-                documents, substitution_sources=documents)
-            documents = [dict(d) for d in deckhand_eng.render()]
+            try:
+                deckhand_eng = layering.DocumentLayering(
+                    documents,
+                    substitution_sources=documents,
+                    fail_on_missing_sub_src=False)
+                documents = [dict(d) for d in deckhand_eng.render()]
+            except dh_errors.DeckhandException as e:
+                LOG.exception(str(e))
+                LOG.error('An unknown Deckhand exception occurred while trying'
+                          ' to render documents.')
+                raise exceptions.DeckhandException(str(e))
+
             LOG.info("Deckhand engine returned %d documents." % len(documents))
         if validate:
             validation.check_schemas(documents, schemas=schema_set)

promenade/control/join_scripts.py

@@ -12,15 +12,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from promenade.control.base import BaseResource
-from promenade.builder import Builder
-from promenade.config import Configuration
-from promenade import logging
-from promenade import policy
 import falcon
 import kubernetes
 import random
 
+from promenade.control.base import BaseResource
+from promenade.builder import Builder
+from promenade.config import Configuration
+from promenade import exceptions
+from promenade import logging
+from promenade import policy
+
 LOG = logging.getLogger(__name__)
 
 
@@ -40,7 +42,11 @@ class JoinScriptsResource(BaseResource):
 
         join_ip = _get_join_ip()
 
-        config = Configuration.from_design_ref(design_ref)
+        try:
+            config = Configuration.from_design_ref(design_ref)
+        except exceptions.DeckhandException as e:
+            raise falcon.HTTPInternalServerError(description=str(e))
+
         node_document = {
             'schema': 'promenade/KubernetesNode/v1',
             'metadata': {

promenade/exceptions.py

@@ -241,6 +241,11 @@ class ValidationException(PromenadeException):
     status = falcon.HTTP_400
 
 
+class DeckhandException(PromenadeException):
+    title = 'Deckhand Engine Error'
+    status = falcon.HTTP_500
+
+
 def massage_error_list(error_list, placeholder_description):
     """
     Returns a best-effort attempt to make a nice error list