DECKHAND-66: Document substitution implementation

This PS implements documentation substitution and
the rendered-documents endpoint. Each time the
rendered-documents is queried, the documents for
the reqeust revision_id dynamically undergo
secret substitution.

All functional tests related to secret substitution
have been unskipped.

Deckhand currently does not real testing for
verifying that secret substitution works
for encrypted documents. This will only happen
when integration testing is added to Deckhand to
test its interaction with Keystone and Barbican.

Included in this PS:
  - basic implementation for secret substitution
  - introduction of jsonpath_ng for searching for and
    updating jsonpaths in documents
  - rendered-documents endpoint
  - unit tests
  - all relevant functional tests unskipped
  - additional bucket controller tests include RBAC
    tests and framework testing RBAC via unit tests

Change-Id: I86f269a5b616b518e5f742a4005891412226fe2a

AUTHORS

@@ -1,7 +1,9 @@
 Alan Meadows <alan.meadows@gmail.com>
 Anthony Lin <anthony.jclin@gmail.com>
+Bryan Strassner <bryan.strassner@gmail.com>
 Felipe Monteiro <felipe.monteiro@att.com>
 Felipe Monteiro <fmontei@users.noreply.github.com>
 Mark Burnett <mark.m.burnett@gmail.com>
 Pete Birley <pete@port.direct>
 Scott Hussey <sh8121@att.com>
+Tin Lam <tin@irrational.io>

deckhand/context.py

@@ -14,9 +14,6 @@
 
 from oslo_config import cfg
 from oslo_context import context
-from oslo_policy import policy as common_policy
-
-from deckhand import policy
 
 CONF = cfg.CONF
 
@@ -28,12 +25,10 @@ class RequestContext(context.RequestContext):
     accesses the system, as well as additional request information.
     """
 
-    def __init__(self, policy_enforcer=None, project=None, **kwargs):
+    def __init__(self, project=None, **kwargs):
         if project:
             kwargs['tenant'] = project
         self.project = project
-        self.policy_enforcer = policy_enforcer or common_policy.Enforcer(CONF)
-        policy.register_rules(self.policy_enforcer)
         super(RequestContext, self).__init__(**kwargs)
 
     def to_dict(self):

deckhand/control/api.py

@@ -67,6 +67,8 @@ def start_api():
             revision_diffing.RevisionDiffingResource()),
         ('revisions/{revision_id}/documents',
             revision_documents.RevisionDocumentsResource()),
+        ('revisions/{revision_id}/rendered-documents',
+            revision_documents.RenderedDocumentsResource()),
         ('revisions/{revision_id}/tags', revision_tags.RevisionTagsResource()),
         ('revisions/{revision_id}/tags/{tag}',
             revision_tags.RevisionTagsResource()),

deckhand/control/base.py

@@ -51,10 +51,9 @@ class BaseResource(object):
 
 class DeckhandRequest(falcon.Request):
 
-    def __init__(self, env, options=None, policy_enforcer=None):
+    def __init__(self, env, options=None):
         super(DeckhandRequest, self).__init__(env, options)
-        self.context = context.RequestContext.from_environ(
+        self.context = context.RequestContext.from_environ(self.env)
-            self.env, policy_enforcer=policy_enforcer)
 
     @property
     def project_id(self):

deckhand/control/buckets.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import itertools
 import yaml
 
 import falcon
@@ -37,6 +36,7 @@ class BucketsResource(api_base.BaseResource):
     view_builder = document_view.ViewBuilder()
     secrets_mgr = secrets_manager.SecretsManager()
 
+    @policy.authorize('deckhand:create_cleartext_documents')
     def on_put(self, req, resp, bucket_name=None):
         document_data = req.stream.read(req.content_length or 0)
         try:
@@ -47,10 +47,34 @@ class BucketsResource(api_base.BaseResource):
             LOG.error(error_msg)
             raise falcon.HTTPBadRequest(description=six.text_type(e))
 
+        # NOTE: Must validate documents before doing policy enforcement,
+        # because we expect certain formatting of the documents while doing
+        # policy enforcement.
+        validation_policies = self._create_validation_policies(documents)
+
+        for document in documents:
+            if document['metadata'].get('storagePolicy') == 'encrypted':
+                policy.conditional_authorize(
+                    'deckhand:create_encrypted_documents', req.context)
+                break
+
+        self._prepare_secret_documents(documents)
+
+        # Save all the documents, including validation policies.
+        documents_to_create = documents + validation_policies
+        created_documents = self._create_revision_documents(
+            bucket_name, list(documents_to_create))
+
+        if created_documents:
+            resp.body = self.to_yaml_body(
+                self.view_builder.list(created_documents))
+        resp.status = falcon.HTTP_200
+        resp.append_header('Content-Type', 'application/x-yaml')
+
+    def _create_validation_policies(self, documents):
+        # All concrete documents in the payload must successfully pass their
+        # JSON schema validations. Otherwise raise an error.
         try:
-            # NOTE: Must validate documents before doing policy enforcement,
-            # because we expect certain formatting of the documents while doing
-            # policy enforcement.
             validation_policies = document_validation.DocumentValidation(
                 documents).validate_all()
         except deckhand_errors.InvalidDocumentFormat as e:
@@ -58,42 +82,25 @@ class BucketsResource(api_base.BaseResource):
             # validation policy in the DB for future debugging, and only
             # afterward raise an exception.
             raise falcon.HTTPBadRequest(description=e.format_message())
+        return validation_policies
 
-        cleartext_documents = []
+    def _prepare_secret_documents(self, secret_documents):
-        secret_documents = []
+        # Encrypt data for secret documents, if any.
-
-        for document in documents:
-            if any([document['schema'].startswith(t)
-                    for t in types.DOCUMENT_SECRET_TYPES]):
-                secret_documents.append(document)
-            else:
-                cleartext_documents.append(document)
-
-        if secret_documents and any(
-                [d['metadata'].get('storagePolicy') == 'encrypted'
-                for d in secret_documents]):
-            policy.conditional_authorize('deckhand:create_encrypted_documents',
-                                         req.context)
-        if cleartext_documents:
-            policy.conditional_authorize('deckhand:create_cleartext_documents',
-                                         req.context)
-
         for document in secret_documents:
-            secret_data = self.secrets_mgr.create(document)
+            # TODO(fmontei): Move all of this to document validation directly.
-            document['data'] = secret_data
+            if document['metadata'].get('storagePolicy') == 'encrypted':
+                secret_data = self.secrets_mgr.create(document)
+                document['data'] = secret_data
+            elif any([document['schema'].startswith(t)
+                      for t in types.DOCUMENT_SECRET_TYPES]):
+                document['data'] = {'secret': document['data']}
 
+    def _create_revision_documents(self, bucket_name, documents):
         try:
-            documents_to_create = itertools.chain(
+            created_documents = db_api.documents_create(bucket_name, documents)
-                cleartext_documents, secret_documents, validation_policies)
-            created_documents = db_api.documents_create(
-                bucket_name, list(documents_to_create))
         except deckhand_errors.DocumentExists as e:
             raise falcon.HTTPConflict(description=e.format_message())
         except Exception as e:
             raise falcon.HTTPInternalServerError(description=six.text_type(e))
 
-        if created_documents:
+        return created_documents
-            resp.body = self.to_yaml_body(
-                self.view_builder.list(created_documents))
-        resp.status = falcon.HTTP_200
-        resp.append_header('Content-Type', 'application/x-yaml')

deckhand/control/revision_documents.py

@@ -19,6 +19,7 @@ from deckhand.control import base as api_base
 from deckhand.control import common
 from deckhand.control.views import document as document_view
 from deckhand.db.sqlalchemy import api as db_api
+from deckhand.engine import secrets_manager
 from deckhand import errors
 from deckhand import policy
 
@@ -26,10 +27,11 @@ LOG = logging.getLogger(__name__)
 
 
 class RevisionDocumentsResource(api_base.BaseResource):
-    """API resource for realizing CRUD endpoints for revision documents."""
+    """API resource for realizing revision documents endpoint."""
 
     view_builder = document_view.ViewBuilder()
 
+    @policy.authorize('deckhand:list_cleartext_documents')
     @common.sanitize_params([
         'schema', 'metadata.name', 'metadata.layeringDefinition.abstract',
         'metadata.layeringDefinition.layer', 'metadata.label',
@@ -42,18 +44,13 @@ class RevisionDocumentsResource(api_base.BaseResource):
         documents will be as originally posted with no substitutions or
         layering applied.
         """
-        include_cleartext = policy.conditional_authorize(
-            'deckhand:list_cleartext_documents', req.context, do_raise=False)
         include_encrypted = policy.conditional_authorize(
             'deckhand:list_encrypted_documents', req.context, do_raise=False)
 
         filters = sanitized_params.copy()
-        filters['metadata.storagePolicy'] = []
+        filters['metadata.storagePolicy'] = ['cleartext']
-        if include_cleartext:
-            filters['metadata.storagePolicy'].append('cleartext')
         if include_encrypted:
             filters['metadata.storagePolicy'].append('encrypted')
-
         # Never return deleted documents to user.
         filters['deleted'] = False
 
@@ -66,3 +63,51 @@ class RevisionDocumentsResource(api_base.BaseResource):
         resp.status = falcon.HTTP_200
         resp.append_header('Content-Type', 'application/x-yaml')
         resp.body = self.to_yaml_body(self.view_builder.list(documents))
+
+
+class RenderedDocumentsResource(api_base.BaseResource):
+    """API resource for realizing rendered documents endpoint.
+
+    Rendered documents are also revision documents, but unlike revision
+    documents, they are finalized documents, having undergone secret
+    substitution and document layering.
+
+    Returns a multi-document YAML response containing all the documents
+    matching the filters specified via query string parameters. Returned
+    documents will have secrets substituted into them and be layered with
+    other documents in the revision, in accordance with the ``LayeringPolicy``
+    that currently exists in the system.
+    """
+
+    view_builder = document_view.ViewBuilder()
+
+    @policy.authorize('deckhand:list_cleartext_documents')
+    @common.sanitize_params([
+        'schema', 'metadata.name', 'metadata.label'])
+    def on_get(self, req, resp, sanitized_params, revision_id):
+        include_encrypted = policy.conditional_authorize(
+            'deckhand:list_encrypted_documents', req.context, do_raise=False)
+
+        filters = sanitized_params.copy()
+        filters['metadata.storagePolicy'] = ['cleartext']
+        if include_encrypted:
+            filters['metadata.storagePolicy'].append('encrypted')
+
+        try:
+            documents = db_api.revision_get_documents(
+                revision_id, **filters)
+        except (errors.RevisionNotFound) as e:
+            raise falcon.HTTPNotFound(description=e.format_message())
+
+        # TODO(fmontei): Currently the only phase of rendering that is
+        # performed is secret substitution, which can be done in any randomized
+        # order. However, secret substitution logic will have to be moved into
+        # a separate module that handles layering alongside substitution once
+        # layering has been fully integrated into this endpoint.
+        secrets_substitution = secrets_manager.SecretsSubstitution(documents)
+        rendered_documents = secrets_substitution.substitute_all()
+
+        resp.status = falcon.HTTP_200
+        resp.append_header('Content-Type', 'application/x-yaml')
+        resp.body = self.to_yaml_body(
+            self.view_builder.list(rendered_documents))

deckhand/control/rollback.py

@@ -34,12 +34,10 @@ class RollbackResource(api_base.BaseResource):
             raise falcon.HTTPNotFound(description=e.format_message())
 
         for document in latest_revision['documents']:
-            if document['metadata'].get('storagePolicy') == 'cleartext':
+            if document['metadata'].get('storagePolicy') == 'encrypted':
-                policy.conditional_authorize(
-                    'deckhand:create_cleartext_documents', req.context)
-            elif document['metadata'].get('storagePolicy') == 'encrypted':
                 policy.conditional_authorize(
                     'deckhand:create_encrypted_documents', req.context)
+                break
 
         try:
             rollback_revision = db_api.revision_rollback(

deckhand/control/views/document.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 from deckhand.control import common
+from deckhand import types
 
 
 class ViewBuilder(common.ViewBuilder):
@@ -30,34 +31,30 @@ class ViewBuilder(common.ViewBuilder):
     _collection_name = 'documents'
 
     def list(self, documents):
-        # Edge case for when all documents are deleted from a bucket. Still
-        # need to return bucket_id and revision_id.
-        if len(documents) == 1 and documents[0]['deleted']:
-            resp_obj = {'status': {}}
-            resp_obj['status']['bucket'] = documents[0]['bucket_name']
-            resp_obj['status']['revision'] = documents[0]['revision_id']
-            return [resp_obj]
-
         resp_list = []
         attrs = ['id', 'metadata', 'data', 'schema']
 
         for document in documents:
+            if document['deleted']:
+                continue
+            if document['schema'].startswith(types.VALIDATION_POLICY_SCHEMA):
+                continue
             resp_obj = {x: document[x] for x in attrs}
             resp_obj.setdefault('status', {})
             resp_obj['status']['bucket'] = document['bucket_name']
             resp_obj['status']['revision'] = document['revision_id']
             resp_list.append(resp_obj)
 
-        # In the case where no documents are passed to PUT
+        # Edge case for when all documents are deleted from a bucket. To detect
-        # buckets/{{bucket_name}}/documents, we need to mangle the response
+        # the edge case, check whether ``resp_list`` is empty and whether there
-        # body a bit. The revision_id and buckete_id should be returned, as
+        # are still documents to be returned. This means that all the documents
-        # at the very least the revision_id will be needed by the user.
+        # are either deleted or validation policies. Either way, we still need
+        # to return bucket_id and revision_id, which should be the same
+        # across all the documents in ``documents``.
         if not resp_list and documents:
-            resp_obj = {}
+            resp_obj = {'status': {}}
-            resp_obj.setdefault('status', {})
+            resp_obj['status']['bucket'] = documents[0]['bucket_name']
-            resp_obj['status']['bucket'] = documents[0]['bucket_id']
             resp_obj['status']['revision'] = documents[0]['revision_id']
-
+            return [resp_obj]
-            resp_list.append(resp_obj)
 
         return resp_list

deckhand/control/views/revision.py

@@ -59,7 +59,7 @@ class ViewBuilder(common.ViewBuilder):
         success_status = 'success'
 
         for vp in [d for d in revision['documents']
-                   if d['schema'] == types.VALIDATION_POLICY_SCHEMA]:
+                   if d['schema'].startswith(types.VALIDATION_POLICY_SCHEMA)]:
             validation_policy = {}
             validation_policy['name'] = vp.get('name')
             validation_policy['url'] = self._gen_url(vp)

deckhand/db/sqlalchemy/api.py

@@ -261,18 +261,29 @@ def document_get(session=None, raw_dict=False, **filters):
     """
     session = session or get_session()
 
-    # Retrieve the most recently created version of a document. Documents with
+    # TODO(fmontei): Currently Deckhand doesn't support filtering by nested
-    # the same metadata.name and schema can exist across different revisions,
+    # JSON fields via sqlalchemy. For now, filter the documents using all
-    # so it is necessary to use `first` instead of `one` to avoid errors.
+    # "regular" filters via sqlalchemy and all nested filters via Python.
-    document = session.query(models.Document)\
+    nested_filters = {}
+    for f in filters.copy():
+        if '.' in f:
+            nested_filters.setdefault(f, filters.pop(f))
+
+    # Documents with the the same metadata.name and schema can exist across
+    # different revisions, so it is necessary to order documents by creation
+    # date, then return the first document that matches all desired filters.
+    documents = session.query(models.Document)\
         .filter_by(**filters)\
         .order_by(models.Document.created_at.desc())\
-        .first()
+        .all()
 
-    if not document:
+    for doc in documents:
-        raise errors.DocumentNotFound(document=filters)
+        d = doc.to_dict(raw_dict=raw_dict)
+        if _apply_filters(d, **nested_filters):
+            return d
 
-    return document.to_dict(raw_dict=raw_dict)
+    filters.update(nested_filters)
+    raise errors.DocumentNotFound(document=filters)
 
 
 ####################

deckhand/engine/document.py

@@ -72,6 +72,9 @@ class Document(object):
     def get_labels(self):
         return self._inner['metadata']['labels']
 
+    def get_substitutions(self):
+        return self._inner['metadata'].get('substitutions', None)
+
     def get_actions(self):
         try:
             return self._inner['metadata']['layeringDefinition']['actions']
@@ -118,4 +121,7 @@ class Document(object):
         return not self.__contains__(k)
 
     def __repr__(self):
-        return repr(self._inner)
+        return '(%s, %s)' % (self.get_schema(), self.get_name())
+
+    def __str__(self):
+        return str(self._inner)

deckhand/engine/schema/v1_0/certificate_key_schema.py

@@ -27,6 +27,13 @@ schema = {
                     'pattern': '^(metadata/Document/v[1]{1}(\.[0]{1}){0,1})$',
                 },
                 'name': {'type': 'string'},
+                # Not strictly needed for secrets.
+                'layeringDefinition': {
+                    'type': 'object',
+                    'properties': {
+                        'layer': {'type': 'string'}
+                    }
+                },
                 'storagePolicy': {
                     'type': 'string',
                     'enum': ['encrypted', 'cleartext']

deckhand/engine/schema/v1_0/certificate_schema.py

@@ -27,6 +27,13 @@ schema = {
                     'pattern': '^(metadata/Document/v[1]{1}(\.[0]{1}){0,1})$',
                 },
                 'name': {'type': 'string'},
+                # Not strictly needed for secrets.
+                'layeringDefinition': {
+                    'type': 'object',
+                    'properties': {
+                        'layer': {'type': 'string'}
+                    }
+                },
                 'storagePolicy': {
                     'type': 'string',
                     'enum': ['encrypted', 'cleartext']

deckhand/engine/schema/v1_0/data_schema.py

@@ -33,6 +33,10 @@ schema = {
                 # Labels are optional.
                 'labels': {
                     'type': 'object'
+                },
+                'storagePolicy': {
+                    'type': 'string',
+                    'enum': ['encrypted', 'cleartext']
                 }
             },
             'additionalProperties': False,

deckhand/engine/schema/v1_0/document_schema.py

@@ -84,6 +84,10 @@ schema = {
                 'substitutions': {
                     'type': 'array',
                     'items': substitution_schema
+                },
+                'storagePolicy': {
+                    'type': 'string',
+                    'enum': ['encrypted', 'cleartext']
                 }
             },
             'additionalProperties': False,

deckhand/engine/schema/v1_0/layering_schema.py

@@ -26,7 +26,11 @@ schema = {
                     'type': 'string',
                     'pattern': '^(metadata/Control/v[1]{1}(\.[0]{1}){0,1})$'
                 },
-                'name': {'type': 'string'}
+                'name': {'type': 'string'},
+                'storagePolicy': {
+                    'type': 'string',
+                    'enum': ['encrypted', 'cleartext']
+                }
             },
             'additionalProperties': False,
             'required': ['schema', 'name']

deckhand/engine/schema/v1_0/passphrase_schema.py

@@ -27,6 +27,13 @@ schema = {
                     'pattern': '^(metadata/Document/v[1]{1}(\.[0]{1}){0,1})$',
                 },
                 'name': {'type': 'string'},
+                # Not strictly needed for secrets.
+                'layeringDefinition': {
+                    'type': 'object',
+                    'properties': {
+                        'layer': {'type': 'string'}
+                    }
+                },
                 'storagePolicy': {
                     'type': 'string',
                     'enum': ['encrypted', 'cleartext']

deckhand/engine/schema/v1_0/validation_schema.py

@@ -26,7 +26,11 @@ schema = {
                     'type': 'string',
                     'pattern': '^(metadata/Control/v[1]{1}(\.[0]{1}){0,1})$'
                 },
-                'name': {'type': 'string'}
+                'name': {'type': 'string'},
+                'storagePolicy': {
+                    'type': 'string',
+                    'enum': ['encrypted', 'cleartext']
+                }
             },
             'additionalProperties': False,
             'required': ['schema', 'name']

deckhand/engine/secrets_manager.py

@@ -12,7 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from oslo_log import log as logging
+
 from deckhand.barbican import driver
+from deckhand.db.sqlalchemy import api as db_api
+from deckhand.engine import document as document_wrapper
+from deckhand import utils
+
+LOG = logging.getLogger(__name__)
 
 CLEARTEXT = 'cleartext'
 ENCRYPTED = 'encrypted'
@@ -34,22 +41,22 @@ class SecretsManager(object):
         documents with the schemas enumerated below) must be stored using a
         secure storage service like Barbican.
 
-        Documents with metadata.storagePolicy == "clearText" have their secrets
+        Documents with ``metadata.storagePolicy`` == "clearText" have their
-        stored directly in Deckhand.
+        secrets stored directly in Deckhand.
 
-        Documents with metadata.storagePolicy == "encrypted" are stored in
+        Documents with ``metadata.storagePolicy`` == "encrypted" are stored in
         Barbican directly. Deckhand in turn stores the reference returned
         by Barbican in Deckhand.
 
         :param secret_doc: A Deckhand document with one of the following
             schemas:
 
-                * deckhand/Certificate/v1
+                * ``deckhand/Certificate/v1``
-                * deckhand/CertificateKey/v1
+                * ``deckhand/CertificateKey/v1``
-                * deckhand/Passphrase/v1
+                * ``deckhand/Passphrase/v1``
 
         :returns: Dictionary representation of
-            `deckhand.db.sqlalchemy.models.DocumentSecret`.
+            ``deckhand.db.sqlalchemy.models.DocumentSecret``.
         """
         encryption_type = secret_doc['metadata']['storagePolicy']
         secret_type = self._get_secret_type(secret_doc['schema'])
@@ -73,9 +80,9 @@ class SecretsManager(object):
     def _get_secret_type(self, schema):
         """Get the Barbican secret type based on the following mapping:
 
-        deckhand/Certificate/v1 => certificate
+        ``deckhand/Certificate/v1`` => certificate
-        deckhand/CertificateKey/v1 => private
+        ``deckhand/CertificateKey/v1`` => private
-        deckhand/Passphrase/v1 => passphrase
+        ``deckhand/Passphrase/v1`` => passphrase
 
         :param schema: The document's schema.
         :returns: The value corresponding to the mapping above.
@@ -84,3 +91,65 @@ class SecretsManager(object):
         if _schema == 'certificatekey':
             return 'private'
         return _schema
+
+
+class SecretsSubstitution(object):
+    """Class for document substitution logic for YAML files."""
+
+    def __init__(self, documents):
+        """SecretSubstitution constructor.
+
+        :param documents: List of YAML documents in dictionary format that are
+            candidates for secret substitution. This class will automatically
+            detect documents that require substitution; documents need not be
+            filtered prior to being passed to the constructor.
+        """
+        if not isinstance(documents, (list, tuple)):
+            documents = [documents]
+        substitute_docs = [document_wrapper.Document(d) for d in documents if
+                           'substitutions' in d['metadata']]
+        self.documents = substitute_docs
+
+    def substitute_all(self):
+        """Substitute all documents that have a `metadata.substitutions` field.
+
+        Concrete (non-abstract) documents can be used as a source of
+        substitution into other documents. This substitution is
+        layer-independent, a document in the region layer could insert data
+        from a document in the site layer.
+
+        :returns: List of fully substituted documents.
+        """
+        LOG.debug('Substituting secrets for documents: %s', self.documents)
+        substituted_docs = []
+
+        for doc in self.documents:
+            LOG.debug(
+                'Checking for substitutions in schema=%s, metadata.name=%s',
+                doc.get_name(), doc.get_schema())
+            for sub in doc.get_substitutions():
+                src_schema = sub['src']['schema']
+                src_name = sub['src']['name']
+                src_path = sub['src']['path']
+                if src_path == '.':
+                    src_path = '.secret'
+
+                # TODO(fmontei): Use secrets_manager for this logic. Need to
+                # check Barbican for the secret if it has been encrypted.
+                src_doc = db_api.document_get(
+                    schema=src_schema, name=src_name, is_secret=True,
+                    **{'metadata.layeringDefinition.abstract': False})
+                src_secret = utils.jsonpath_parse(src_doc['data'], src_path)
+
+                dest_path = sub['dest']['path']
+                dest_pattern = sub['dest'].get('pattern', None)
+
+                LOG.debug('Substituting from schema=%s name=%s src_path=%s '
+                          'into dest_path=%s, dest_pattern=%s', src_schema,
+                          src_name, src_path, dest_path, dest_pattern)
+                substituted_data = utils.jsonpath_replace(
+                    doc['data'], src_secret, dest_path, dest_pattern)
+                doc['data'].update(substituted_data)
+
+            substituted_docs.append(doc.to_dict())
+        return substituted_docs

deckhand/errors.py

@@ -23,18 +23,11 @@ class DeckhandException(Exception):
     code = 500
 
     def __init__(self, message=None, **kwargs):
-        self.kwargs = kwargs
+        kwargs.setdefault('code', DeckhandException.code)
-
-        if 'code' not in self.kwargs:
-            try:
-                self.kwargs['code'] = self.code
-            except AttributeError:
-                pass
 
         if not message:
             try:
                 message = self.msg_fmt % kwargs
-
             except Exception:
                 message = self.msg_fmt
 
@@ -58,15 +51,6 @@ class InvalidDocumentFormat(DeckhandException):
         super(InvalidDocumentFormat, self).__init__(**kwargs)
 
 
-# TODO(fmontei): Remove this in a future commit.
-class ApiError(Exception):
-    pass
-
-
-class InvalidFormat(ApiError):
-    """The YAML file is incorrectly formatted and cannot be read."""
-
-
 class DocumentExists(DeckhandException):
     msg_fmt = ("Document with schema %(schema)s and metadata.name "
                "%(name)s already exists in bucket %(bucket)s.")
@@ -100,6 +84,12 @@ class MissingDocumentKey(DeckhandException):
                "Parent: %(parent)s. Child: %(child)s.")
 
 
+class MissingDocumentPattern(DeckhandException):
+    msg_fmt = ("Substitution pattern %(pattern)s could not be found for the "
+               "JSON path %(path)s in the destination document data %(data)s.")
+    code = 400
+
+
 class UnsupportedActionMethod(DeckhandException):
     msg_fmt = ("Method in %(actions)s is invalid for document %(document)s.")
     code = 400

deckhand/factories.py

@@ -61,7 +61,6 @@ class DocumentFactory(DeckhandFactory):
             "layeringDefinition": {
                 "abstract": False,
                 "layer": "",
-                "parentSelector": "",
                 "actions": []
             },
             "name": "",
@@ -92,7 +91,7 @@ class DocumentFactory(DeckhandFactory):
             ]
 
         :param num_layers: Total number of layers. Only supported values
-            include 2 or 3.
+            include 1, 2 or 3.
         :type num_layers: integer
         :param docs_per_layer: The number of documents to be included per
             layer. For example, if ``num_layers`` is 3, then ``docs_per_layer``
@@ -105,12 +104,14 @@ class DocumentFactory(DeckhandFactory):
             compatible with ``docs_per_layer``.
         """
         # Set up the layering definition's layerOrder.
-        if num_layers == 2:
+        if num_layers == 1:
+            layer_order = ["global"]
+        elif num_layers == 2:
             layer_order = ["global", "site"]
         elif num_layers == 3:
             layer_order = ["global", "region", "site"]
         else:
-            raise ValueError("'num_layers' must either be 2 or 3.")
+            raise ValueError("'num_layers' must be a value between 1 - 3.")
         self.LAYERING_DEFINITION['data']['layerOrder'] = layer_order
 
         if not isinstance(docs_per_layer, (list, tuple)):
@@ -225,14 +226,30 @@ class DocumentFactory(DeckhandFactory):
                 data_key = "_%s_DATA_%d_" % (layer_name.upper(), count + 1)
                 actions_key = "_%s_ACTIONS_%d_" % (
                     layer_name.upper(), count + 1)
+                sub_key = "_%s_SUBSTITUTIONS_%d_" % (
+                    layer_name.upper(), count + 1)
 
                 try:
                     layer_template['data'] = mapping[data_key]['data']
+                except KeyError as e:
+                    LOG.debug('Could not map %s because it was not found in '
+                              'the `mapping` dict.', e.args[0])
+                    pass
+
+                try:
                     layer_template['metadata']['layeringDefinition'][
                         'actions'] = mapping[actions_key]['actions']
                 except KeyError as e:
-                    LOG.warning('Could not map %s because it was not found in '
+                    LOG.debug('Could not map %s because it was not found in '
-                                'the `mapping` dict.', e.args[0])
+                              'the `mapping` dict.', e.args[0])
+                    pass
+
+                try:
+                    layer_template['metadata']['substitutions'] = mapping[
+                        sub_key]
+                except KeyError as e:
+                    LOG.debug('Could not map %s because it was not found in '
+                              'the `mapping` dict.', e.args[0])
                     pass
 
                 rendered_template.append(layer_template)

deckhand/policies/document.py

@@ -24,10 +24,7 @@ document_policies = [
         """Create a batch of documents specified in the request body, whereby
 a new revision is created. Also, roll back a revision to a previous one in the
 revision history, whereby the target revision's documents are re-created for
-the new revision.
+the new revision.""",
-
-Conditionally enforced for the endpoints below if the any of the documents in
-the request body have a `metadata.storagePolicy` of "cleartext".""",
         [
             {
                 'method': 'PUT',
@@ -46,8 +43,10 @@ a new revision is created. Also, roll back a revision to a previous one in the
 history, whereby the target revision's documents are re-created for the new
 revision.
 
+Only enforced after ``create_cleartext_documents`` passes.
+
 Conditionally enforced for the endpoints below if the any of the documents in
-the request body have a `metadata.storagePolicy` of "encrypted".""",
+the request body have a ``metadata.storagePolicy`` of "encrypted".""",
         [
             {
                 'method': 'PUT',
@@ -63,11 +62,7 @@ the request body have a `metadata.storagePolicy` of "encrypted".""",
         base.RULE_ADMIN_API,
         """List cleartext documents for a revision (with no layering or
 substitution applied) as well as fully layered and substituted concrete
-documents.
+documents.""",
-
-Conditionally enforced for the endpoints below if the any of the documents in
-the request body have a `metadata.storagePolicy` of "cleartext". If policy
-enforcement fails, cleartext documents are omitted.""",
         [
             {
                 'method': 'GET',
@@ -81,13 +76,15 @@ enforcement fails, cleartext documents are omitted.""",
     policy.DocumentedRuleDefault(
         base.POLICY_ROOT % 'list_encrypted_documents',
         base.RULE_ADMIN_API,
-        """List cleartext documents for a revision (with no layering or
+        """List encrypted documents for a revision (with no layering or
 substitution applied) as well as fully layered and substituted concrete
 documents.
 
-Conditionally enforced for the endpoints below if the any of the documents in
+Only enforced after ``list_cleartext_documents`` passes.
-the request body have a `metadata.storagePolicy` of "encrypted". If policy
+
-enforcement fails, encrypted documents are omitted.""",
+Conditionally enforced for the endpoints below if any of the documents in the
+request body have a ``metadata.storagePolicy`` of "encrypted". If policy
+enforcement fails, encrypted documents are exluded from the response.""",
         [
             {
                 'method': 'GET',

deckhand/policy.py

@@ -25,22 +25,54 @@ from deckhand import policies
 
 CONF = cfg.CONF
 LOG = logging.getLogger(__name__)
+_ENFORCER = None
+
+
+def reset():
+    global _ENFORCER
+    if _ENFORCER:
+        _ENFORCER.clear()
+        _ENFORCER = None
+
+
+def init(policy_file=None, rules=None, default_rule=None, use_conf=True):
+    """Init an Enforcer class.
+
+    :param policy_file: Custom policy file to use, if none is specified,
+        ``CONF.policy_file`` will be used.
+    :param rules: Default dictionary / Rules to use. It will be
+        considered just in the first instantiation.
+    :param default_rule: Default rule to use; ``CONF.default_rule`` will
+        be used if none is specified.
+    :param use_conf: Whether to load rules from config file.
+    """
+
+    global _ENFORCER
+
+    if not _ENFORCER:
+        _ENFORCER = policy.Enforcer(CONF,
+                                    policy_file=policy_file,
+                                    rules=rules,
+                                    default_rule=default_rule,
+                                    use_conf=use_conf)
+        register_rules(_ENFORCER)
 
 
 def _do_enforce_rbac(action, context, do_raise=True):
-    policy_enforcer = context.policy_enforcer
+    init()
+
     credentials = context.to_policy_values()
     target = {'project_id': context.project_id,
               'user_id': context.user_id}
     exc = errors.PolicyNotAuthorized
 
     try:
-        # oslo.policy supports both enforce and authorize. authorize is
+        # `oslo.policy` supports both enforce and authorize. authorize is
         # stricter because it'll raise an exception if the policy action is
         # not found in the list of registered rules. This means that attempting
         # to enforce anything not found in ``deckhand.policies`` will error out
         # with a 'Policy not registered' message.
-        return policy_enforcer.authorize(
+        return _ENFORCER.authorize(
             action, target, context.to_dict(), do_raise=do_raise,
             exc=exc, action=action)
     except policy.PolicyNotRegistered as e:

deckhand/tests/functional/gabbits/document-render-success-multiple-bucket-with-substitution.yaml

@@ -16,25 +16,22 @@ tests:
     desc: Begin testing from known state.
     DELETE: /api/v1.0/revisions
     status: 204
-    skip: Not implemented.
 
   - name: add_bucket_a
     desc: Create documents for bucket a
     PUT: /api/v1.0/bucket/a/documents
     status: 200
-    data: <@resources/design-doc-substition-sample-split-bucket-a.yaml
+    data: <@resources/design-doc-substitution-sample-split-bucket-a.yaml
-    skip: Not implemented.
 
   - name: add_bucket_b
     desc: Create documents for bucket b
     PUT: /api/v1.0/bucket/b/documents
     status: 200
-    data: <@resources/design-doc-substition-sample-split-bucket-b.yaml
+    data: <@resources/design-doc-substitution-sample-split-bucket-b.yaml
-    skip: Not implemented.
 
   - name: verify_substitutions
     desc: Check for expected substitutions
-    GET: /api/v1.0/revisions/$RESPONSE['$.[0].revision']/rendered-documents?schema=armada/Chart/v1
+    GET: /api/v1.0/revisions/$RESPONSE['$.[0].status.revision']/rendered-documents?schema=armada/Chart/v1
     status: 200
     response_multidoc_jsonpaths:
       $.[*].metadata.name: example-chart-01
@@ -49,4 +46,3 @@ tests:
               key: |
                 KEY DATA
             some_url: http://admin:my-secret-password@service-name:8080/v1
-    skip: Not implemented.

deckhand/tests/functional/gabbits/document-render-success-single-bucket-with-substitution.yaml

@@ -15,18 +15,16 @@ tests:
     desc: Begin testing from known state.
     DELETE: /api/v1.0/revisions
     status: 204
-    skip: Not implemented.
 
   - name: initialize
     desc: Create initial documents
     PUT: /api/v1.0/bucket/mop/documents
     status: 200
-    data: <@resources/design-doc-substition-sample.yaml
+    data: <@resources/design-doc-substitution-sample.yaml
-    skip: Not implemented.
 
   - name: verify_substitutions
     desc: Check for expected substitutions
-    GET: /api/v1.0/revisions/$RESPONSE['$.[0].revision']/rendered-documents?schema=armada/Chart/v1
+    GET: /api/v1.0/revisions/$RESPONSE['$.[0].status.revision']/rendered-documents?schema=armada/Chart/v1
     status: 200
     response_multidoc_jsonpaths:
       $.[*].metadata.name: example-chart-01
@@ -41,4 +39,3 @@ tests:
               key: |
                 KEY DATA
             some_url: http://admin:my-secret-password@service-name:8080/v1
-    skip: Not implemented.

deckhand/tests/functional/gabbits/resources/design-doc-substitution-sample-split-bucket-a.yaml

@@ -2,9 +2,10 @@
 schema: deckhand/CertificateKey/v1
 metadata:
   name: example-key
-  storagePolicy: encrypted
+  schema: metadata/Document/v1
   layeringDefinition:
     layer: site
+  storagePolicy: cleartext
 data: |
   KEY DATA
 ...

deckhand/tests/functional/gabbits/resources/design-doc-substitution-sample-split-bucket-b.yaml

@@ -2,24 +2,26 @@
 schema: deckhand/Certificate/v1
 metadata:
   name: example-cert
-  storagePolicy: cleartext
+  schema: metadata/Document/v1
   layeringDefinition:
     layer: site
+  storagePolicy: cleartext
 data: |
   CERTIFICATE DATA
 ---
 schema: deckhand/Passphrase/v1
 metadata:
   name: example-password
-  storagePolicy: encrypted
+  schema: metadata/Document/v1
   layeringDefinition:
     layer: site
+  storagePolicy: cleartext
 data: my-secret-password
 ---
 schema: armada/Chart/v1
 metadata:
   name: example-chart-01
-  storagePolicy: cleartext
+  schema: metadata/Document/v1
   layeringDefinition:
     layer: region
   substitutions:

deckhand/tests/functional/gabbits/resources/design-doc-substitution-sample.yaml

@@ -2,33 +2,36 @@
 schema: deckhand/Certificate/v1
 metadata:
   name: example-cert
-  storagePolicy: cleartext
+  schema: metadata/Document/v1
   layeringDefinition:
     layer: site
+  storagePolicy: cleartext
 data: |
   CERTIFICATE DATA
 ---
 schema: deckhand/CertificateKey/v1
 metadata:
   name: example-key
-  storagePolicy: encrypted
+  schema: metadata/Document/v1
   layeringDefinition:
     layer: site
+  storagePolicy: cleartext
 data: |
   KEY DATA
 ---
 schema: deckhand/Passphrase/v1
 metadata:
   name: example-password
-  storagePolicy: encrypted
+  schema: metadata/Document/v1
   layeringDefinition:
     layer: site
+  storagePolicy: cleartext
 data: my-secret-password
 ---
 schema: armada/Chart/v1
 metadata:
   name: example-chart-01
-  storagePolicy: cleartext
+  schema: metadata/Document/v1
   layeringDefinition:
     layer: region
   substitutions:

deckhand/tests/functional/gabbits/revision-diff-success.yaml

@@ -196,6 +196,10 @@ tests:
     PUT: /api/v1.0/bucket/bucket_mistake/documents
     status: 200
     data: ""
+    # Verification for whether a bucket_name was returned even though all the
+    # documents for this bucket were deleted.
+    response_multidoc_jsonpaths:
+      $.[*].status.bucket: bucket_mistake
 
   - name: verify_diff_between_created_and_deleted_mistake
     desc: Validates response for deletion between the last 2 revisions

deckhand/tests/unit/control/base.py

@@ -16,6 +16,7 @@ from falcon import testing as falcon_testing
 
 from deckhand.control import api
 from deckhand.tests.unit import base as test_base
+from deckhand.tests.unit import policy_fixture
 
 
 class BaseControllerTest(test_base.DeckhandWithDBTestCase,
@@ -25,3 +26,4 @@ class BaseControllerTest(test_base.DeckhandWithDBTestCase,
     def setUp(self):
         super(BaseControllerTest, self).setUp()
         self.app = falcon_testing.TestClient(api.start_api())
+        self.policy = self.useFixture(policy_fixture.RealPolicyFixture())

deckhand/tests/unit/control/test_api_initialization.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import inspect
 import mock
 
 from deckhand.control import api
@@ -24,6 +25,7 @@ from deckhand.control import revisions
 from deckhand.control import rollback
 from deckhand.control import versions
 from deckhand.tests.unit import base as test_base
+from deckhand import utils
 
 
 class TestApi(test_base.DeckhandTestCase):
@@ -32,11 +34,16 @@ class TestApi(test_base.DeckhandTestCase):
         super(TestApi, self).setUp()
         for resource in (buckets, revision_diffing, revision_documents,
                          revision_tags, revisions, rollback, versions):
-            resource_name = resource.__name__.split('.')[-1]
+            class_names = self._get_module_class_names(resource)
-            resource_obj = self.patchobject(
+            for class_name in class_names:
-                resource, '%sResource' % resource_name.title().replace(
+                resource_obj = self.patchobject(
-                    '_', ''), autospec=True)
+                    resource, class_name, autospec=True)
-            setattr(self, '%s_resource' % resource_name, resource_obj)
+                setattr(self, utils.to_snake_case(class_name), resource_obj)
+
+    def _get_module_class_names(self, module):
+        class_names = [obj.__name__ for name, obj in inspect.getmembers(module)
+                       if inspect.isclass(obj)]
+        return class_names
 
     @mock.patch.object(api, 'db_api', autospec=True)
     @mock.patch.object(api, 'logging', autospec=True)
@@ -62,6 +69,8 @@ class TestApi(test_base.DeckhandTestCase):
                       self.revision_diffing_resource()),
             mock.call('/api/v1.0/revisions/{revision_id}/documents',
                       self.revision_documents_resource()),
+            mock.call('/api/v1.0/revisions/{revision_id}/rendered-documents',
+                      self.rendered_documents_resource()),
             mock.call('/api/v1.0/revisions/{revision_id}/tags',
                       self.revision_tags_resource()),
             mock.call('/api/v1.0/revisions/{revision_id}/tags/{tag}',

deckhand/tests/unit/control/test_buckets_controller.py

@@ -12,17 +12,106 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import yaml
+
+import mock
+from oslo_config import cfg
+
+from deckhand.control import buckets
+from deckhand import factories
 from deckhand.tests.unit.control import base as test_base
 
+CONF = cfg.CONF
+
 
 class TestBucketsController(test_base.BaseControllerTest):
-    """Test suite for validating positive scenarios bucket controller."""
+    """Test suite for validating positive scenarios for bucket controller."""
+
+    def test_put_bucket(self):
+        rules = {'deckhand:create_cleartext_documents': '@'}
+        self.policy.set_rules(rules)
+
+        documents_factory = factories.DocumentFactory(2, [1, 1])
+        document_mapping = {
+            "_GLOBAL_DATA_1_": {"data": {"a": {"x": 1, "y": 2}}},
+            "_SITE_DATA_1_": {"data": {"a": {"x": 7, "z": 3}, "b": 4}},
+            "_SITE_ACTIONS_1_": {
+                "actions": [{"method": "merge", "path": "."}]}
+        }
+        payload = documents_factory.gen_test(document_mapping)
+
+        resp = self.app.simulate_put('/api/v1.0/bucket/mop/documents',
+                                     body=yaml.safe_dump_all(payload))
+        self.assertEqual(200, resp.status_code)
+        created_documents = list(yaml.safe_load_all(resp.text))
+        self.assertEqual(3, len(created_documents))
+        expected = sorted([(d['schema'], d['metadata']['name'])
+                           for d in payload])
+        actual = sorted([(d['schema'], d['metadata']['name'])
+                         for d in created_documents])
+        self.assertEqual(expected, actual)
+
+    def test_put_bucket_with_secret(self):
+        def _do_test(payload):
+            resp = self.app.simulate_put('/api/v1.0/bucket/mop/documents',
+                                         body=yaml.safe_dump_all(payload))
+            self.assertEqual(200, resp.status_code)
+            created_documents = list(yaml.safe_load_all(resp.text))
+            self.assertEqual(1, len(created_documents))
+            expected = sorted([(d['schema'], d['metadata']['name'])
+                               for d in payload])
+            actual = sorted([(d['schema'], d['metadata']['name'])
+                             for d in created_documents])
+            self.assertEqual(expected, actual)
+            self.assertEqual({'secret': payload[0]['data']},
+                             created_documents[0]['data'])
+
+        # Verify whether creating a cleartext secret works.
+        rules = {'deckhand:create_cleartext_documents': '@'}
+        self.policy.set_rules(rules)
+
+        secrets_factory = factories.DocumentSecretFactory()
+        payload = [secrets_factory.gen_test('Certificate', 'cleartext')]
+        _do_test(payload)
+
+        # Verify whether creating an encrypted secret works.
+        rules = {'deckhand:create_cleartext_documents': '@',
+                 'deckhand:create_encrypted_documents': '@'}
+        self.policy.set_rules(rules)
+
+        secrets_factory = factories.DocumentSecretFactory()
+        payload = [secrets_factory.gen_test('Certificate', 'encrypted')]
+
+        with mock.patch.object(buckets.BucketsResource, 'secrets_mgr',
+                               autospec=True) as mock_secrets_mgr:
+            mock_secrets_mgr.create.return_value = {
+                'secret': payload[0]['data']}
+            _do_test(payload)
+
+        # Verify whether any document can be encrypted if its
+        # `metadata.storagePolicy`='encrypted'. In the case below,
+        # a generic document is tested.
+        documents_factory = factories.DocumentFactory(1, [1])
+        document_mapping = {
+            "_GLOBAL_DATA_1_": {"data": {"a": {"x": 1, "y": 2}}}
+        }
+        payload = documents_factory.gen_test(document_mapping,
+                                             global_abstract=False)
+        payload[-1]['metadata']['storagePolicy'] = 'encrypted'
+        with mock.patch.object(buckets.BucketsResource, 'secrets_mgr',
+                               autospec=True) as mock_secrets_mgr:
+            mock_secrets_mgr.create.return_value = {
+                'secret': payload[-1]['data']}
+            _do_test([payload[-1]])
 
 
 class TestBucketsControllerNegative(test_base.BaseControllerTest):
-    """Test suite for validating negative scenarios bucket controller."""
+    """Test suite for validating negative scenarios for bucket controller."""
 
     def test_put_bucket_with_invalid_document_payload(self):
+        rules = {'deckhand:create_cleartext_documents': '@'}
+        self.policy.set_rules(rules)
+
         no_colon_spaces = """
 name:foo
 schema:
@@ -38,3 +127,42 @@ schema:
                                          body=payload)
             self.assertEqual(400, resp.status_code)
             self.assertRegexpMatches(resp.text, error_re[idx])
+
+
+class TestBucketsControllerNegativeRBAC(test_base.BaseControllerTest):
+    """Test suite for validating negative RBAC scenarios for bucket
+    controller.
+    """
+
+    def test_put_bucket_cleartext_documents_except_forbidden(self):
+        rules = {'deckhand:create_cleartext_documents': 'rule:admin_api'}
+        self.policy.set_rules(rules)
+
+        documents_factory = factories.DocumentFactory(2, [1, 1])
+        payload = documents_factory.gen_test({})
+
+        resp = self.app.simulate_put('/api/v1.0/bucket/mop/documents',
+                                     body=yaml.safe_dump_all(payload))
+        self.assertEqual(403, resp.status_code)
+
+    def test_put_bucket_cleartext_secret_except_forbidden(self):
+        rules = {'deckhand:create_cleartext_documents': 'rule:admin_api'}
+        self.policy.set_rules(rules)
+
+        secrets_factory = factories.DocumentSecretFactory()
+        payload = [secrets_factory.gen_test('Certificate', 'cleartext')]
+
+        resp = self.app.simulate_put('/api/v1.0/bucket/mop/documents',
+                                     body=yaml.safe_dump_all(payload))
+        self.assertEqual(403, resp.status_code)
+
+    def test_put_bucket_encrypted_secret_except_forbidden(self):
+        rules = {'deckhand:create_encrypted_documents': 'rule:admin_api'}
+        self.policy.set_rules(rules)
+
+        secrets_factory = factories.DocumentSecretFactory()
+        payload = [secrets_factory.gen_test('Certificate', 'encrypted')]
+
+        resp = self.app.simulate_put('/api/v1.0/bucket/mop/documents',
+                                     body=yaml.safe_dump_all(payload))
+        self.assertEqual(403, resp.status_code)

deckhand/tests/unit/db/base.py

@@ -40,7 +40,8 @@ class DocumentFixture(object):
                 'layeringDefinition': {
                     'abstract': test_utils.rand_bool(),
                     'layer': test_utils.rand_name('layer')
-                }
+                },
+                'storagePolicy': test_utils.rand_name('storage_policy')
             },
             'schema': test_utils.rand_name('schema')}
         fixture.update(kwargs)

deckhand/tests/unit/engine/test_secrets_manager.py

@@ -12,13 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import copy
+
 from deckhand.engine import secrets_manager
 from deckhand import factories
 from deckhand.tests import test_utils
-from deckhand.tests.unit.db import base
+from deckhand.tests.unit.db import base as test_base
 
 
-class TestSecretsManager(base.TestDbBase):
+class TestSecretsManager(test_base.TestDbBase):
 
     def setUp(self):
         super(TestSecretsManager, self).setUp()
@@ -71,3 +73,215 @@ class TestSecretsManager(base.TestDbBase):
 
     def test_create_encrypted_passphrase(self):
         self._test_create_secret('encrypted', 'Passphrase')
+
+
+class TestSecretsSubstitution(test_base.TestDbBase):
+
+    def setUp(self):
+        super(TestSecretsSubstitution, self).setUp()
+        self.document_factory = factories.DocumentFactory(1, [1])
+        self.secrets_factory = factories.DocumentSecretFactory()
+
+    def _test_secret_substitution(self, document_mapping, secret_documents,
+                                  expected_data):
+        payload = self.document_factory.gen_test(document_mapping,
+                                                 global_abstract=False)
+        bucket_name = test_utils.rand_name('bucket')
+        documents = self.create_documents(
+            bucket_name, secret_documents + [payload[-1]])
+
+        expected_documents = copy.deepcopy([documents[-1]])
+        expected_documents[0]['data'] = expected_data
+
+        secret_substitution = secrets_manager.SecretsSubstitution(documents)
+        substituted_docs = secret_substitution.substitute_all()
+
+        self.assertEqual(expected_documents, substituted_docs)
+
+    def test_secret_substitution_single_cleartext(self):
+        certificate = self.secrets_factory.gen_test(
+            'Certificate', 'cleartext', data={'secret': 'CERTIFICATE DATA'})
+        certificate['metadata']['name'] = 'example-cert'
+
+        document_mapping = {
+            "_GLOBAL_SUBSTITUTIONS_1_": [{
+                "dest": {
+                    "path": ".chart.values.tls.certificate"
+                },
+                "src": {
+                    "schema": "deckhand/Certificate/v1",
+                    "name": "example-cert",
+                    "path": "."
+                }
+
+            }]
+        }
+        expected_data = {
+            'chart': {
+                'values': {
+                    'tls': {
+                        'certificate': 'CERTIFICATE DATA'
+                    }
+                }
+            }
+        }
+        self._test_secret_substitution(
+            document_mapping, [certificate], expected_data)
+
+    def test_secret_substitution_single_cleartext_with_pattern(self):
+        passphrase = self.secrets_factory.gen_test(
+            'Passphrase', 'cleartext', data={'secret': 'my-secret-password'})
+        passphrase['metadata']['name'] = 'example-password'
+
+        document_mapping = {
+            "_GLOBAL_DATA_1_": {
+                'data': {
+                    'chart': {
+                        'values': {
+                            'some_url': (
+                                'http://admin:INSERT_PASSWORD_HERE'
+                                '@service-name:8080/v1')
+                        }
+                    }
+                }
+            },
+            "_GLOBAL_SUBSTITUTIONS_1_": [{
+                "dest": {
+                    "path": ".chart.values.some_url",
+                    "pattern": "INSERT_[A-Z]+_HERE"
+                },
+                "src": {
+                    "schema": "deckhand/Passphrase/v1",
+                    "name": "example-password",
+                    "path": "."
+                }
+            }]
+        }
+        expected_data = {
+            'chart': {
+                'values': {
+                    'some_url': (
+                        'http://admin:my-secret-password@service-name:8080/v1')
+                }
+            }
+        }
+        self._test_secret_substitution(
+            document_mapping, [passphrase], expected_data)
+
+    def test_secret_substitution_double_cleartext(self):
+        certificate = self.secrets_factory.gen_test(
+            'Certificate', 'cleartext', data={'secret': 'CERTIFICATE DATA'})
+        certificate['metadata']['name'] = 'example-cert'
+
+        certificate_key = self.secrets_factory.gen_test(
+            'CertificateKey', 'cleartext', data={'secret': 'KEY DATA'})
+        certificate_key['metadata']['name'] = 'example-key'
+
+        document_mapping = {
+            "_GLOBAL_SUBSTITUTIONS_1_": [{
+                "dest": {
+                    "path": ".chart.values.tls.certificate"
+                },
+                "src": {
+                    "schema": "deckhand/Certificate/v1",
+                    "name": "example-cert",
+                    "path": "."
+                }
+
+            }, {
+                "dest": {
+                    "path": ".chart.values.tls.key"
+                },
+                "src": {
+                    "schema": "deckhand/CertificateKey/v1",
+                    "name": "example-key",
+                    "path": "."
+                }
+
+            }]
+        }
+        expected_data = {
+            'chart': {
+                'values': {
+                    'tls': {
+                        'certificate': 'CERTIFICATE DATA',
+                        'key': 'KEY DATA'
+                    }
+                }
+            }
+        }
+        self._test_secret_substitution(
+            document_mapping, [certificate, certificate_key], expected_data)
+
+    def test_secret_substitution_multiple_cleartext(self):
+        certificate = self.secrets_factory.gen_test(
+            'Certificate', 'cleartext', data={'secret': 'CERTIFICATE DATA'})
+        certificate['metadata']['name'] = 'example-cert'
+
+        certificate_key = self.secrets_factory.gen_test(
+            'CertificateKey', 'cleartext', data={'secret': 'KEY DATA'})
+        certificate_key['metadata']['name'] = 'example-key'
+
+        passphrase = self.secrets_factory.gen_test(
+            'Passphrase', 'cleartext', data={'secret': 'my-secret-password'})
+        passphrase['metadata']['name'] = 'example-password'
+
+        document_mapping = {
+            "_GLOBAL_DATA_1_": {
+                'data': {
+                    'chart': {
+                        'values': {
+                            'some_url': (
+                                'http://admin:INSERT_PASSWORD_HERE'
+                                '@service-name:8080/v1')
+                        }
+                    }
+                }
+            },
+            "_GLOBAL_SUBSTITUTIONS_1_": [{
+                "dest": {
+                    "path": ".chart.values.tls.certificate"
+                },
+                "src": {
+                    "schema": "deckhand/Certificate/v1",
+                    "name": "example-cert",
+                    "path": "."
+                }
+
+            }, {
+                "dest": {
+                    "path": ".chart.values.tls.key"
+                },
+                "src": {
+                    "schema": "deckhand/CertificateKey/v1",
+                    "name": "example-key",
+                    "path": "."
+                }
+
+            }, {
+                "dest": {
+                    "path": ".chart.values.some_url",
+                    "pattern": "INSERT_[A-Z]+_HERE"
+                },
+                "src": {
+                    "schema": "deckhand/Passphrase/v1",
+                    "name": "example-password",
+                    "path": "."
+                }
+            }]
+        }
+        expected_data = {
+            'chart': {
+                'values': {
+                    'tls': {
+                        'certificate': 'CERTIFICATE DATA',
+                        'key': 'KEY DATA'
+                    },
+                    'some_url': (
+                        'http://admin:my-secret-password@service-name:8080/v1')
+                }
+            }
+        }
+        self._test_secret_substitution(
+            document_mapping, [certificate, certificate_key, passphrase],
+            expected_data)

deckhand/tests/unit/fake_policy.py

@@ -0,0 +1,31 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+policy_data = """
+"admin_api": "role:admin"
+"deckhand:create_cleartext_documents": "rule:admin_api"
+"deckhand:create_encrypted_documents": "rule:admin_api"
+"deckhand:list_cleartext_documents": "rule:admin_api"
+"deckhand:list_encrypted_documents": "rule:admin_api"
+"deckhand:show_revision": "rule:admin_api"
+"deckhand:list_revisions": "rule:admin_api"
+"deckhand:delete_revisions": "rule:admin_api"
+"deckhand:show_revision_diff": "rule:admin_api"
+"deckhand:create_tag": "rule:admin_api"
+"deckhand:show_tag": "rule:admin_api"
+"deckhand:list_tags": "rule:admin_api"
+"deckhand:delete_tag": "rule:admin_api"
+"deckhand:delete_tags": "rule:admin_api"
+"""

deckhand/tests/unit/policy_fixture.py

@@ -0,0 +1,74 @@
+# Copyright 2012 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import os
+import yaml
+
+import fixtures
+from oslo_config import cfg
+from oslo_policy import opts as policy_opts
+from oslo_policy import policy as oslo_policy
+
+from deckhand import policies
+import deckhand.policy
+from deckhand.tests.unit import fake_policy
+
+
+CONF = cfg.CONF
+
+
+class RealPolicyFixture(fixtures.Fixture):
+    """Load the live policy for tests.
+
+    A base policy fixture that starts with the assumption that you'd
+    like to load and enforce the shipped default policy in tests.
+
+    """
+
+    def setUp(self):
+        super(RealPolicyFixture, self).setUp()
+        self.policy_dir = self.useFixture(fixtures.TempDir())
+        self.policy_file = os.path.join(self.policy_dir.path,
+                                        'policy.yaml')
+        # Load the fake_policy data and add the missing default rules.
+        policy_rules = yaml.safe_load(fake_policy.policy_data)
+        self.add_missing_default_rules(policy_rules)
+        with open(self.policy_file, 'w') as f:
+            yaml.safe_dump(policy_rules, f)
+
+        policy_opts.set_defaults(CONF)
+        CONF.set_override('policy_dirs', [], group='oslo_policy')
+        CONF.set_override('policy_file', self.policy_file, group='oslo_policy')
+
+        deckhand.policy.reset()
+        deckhand.policy.init()
+        self.addCleanup(deckhand.policy.reset)
+
+    def add_missing_default_rules(self, rules):
+        """Adds default rules and their values to the given rules dict.
+
+        The given rulen dict may have an incomplete set of policy rules.
+        This method will add the default policy rules and their values to
+        the dict. It will not override the existing rules.
+        """
+        for rule in policies.list_rules():
+            if rule.name not in rules:
+                rules[rule.name] = rule.check_str
+
+    def set_rules(self, rules, overwrite=True):
+        if isinstance(rules, dict):
+            rules = oslo_policy.Rules.from_dict(rules)
+
+        policy = deckhand.policy._ENFORCER
+        policy.set_rules(rules, overwrite=overwrite)

deckhand/tests/unit/test_policy.py

@@ -14,12 +14,10 @@ import falcon
 import mock
 from oslo_policy import policy as common_policy
 
-from deckhand.conf import config
 from deckhand.control import base as api_base
-from deckhand import policy
+import deckhand.policy
 from deckhand.tests.unit import base as test_base
-
+from deckhand.tests.unit import policy_fixture
-CONF = config.CONF
 
 
 class PolicyBaseTestCase(test_base.DeckhandTestCase):
@@ -33,18 +31,18 @@ class PolicyBaseTestCase(test_base.DeckhandTestCase):
             "deckhand:create_cleartext_documents": [['@']],
             "deckhand:list_cleartext_documents": [['rule:admin_api']]
         }
-        self.policy_enforcer = common_policy.Enforcer(CONF)
+
+        self.policy = self.useFixture(policy_fixture.RealPolicyFixture())
         self._set_rules()
 
     def _set_rules(self):
-        rules = common_policy.Rules.from_dict(self.rules)
+        these_rules = common_policy.Rules.from_dict(self.rules)
-        self.policy_enforcer.set_rules(rules)
+        deckhand.policy._ENFORCER.set_rules(these_rules)
-        self.addCleanup(self.policy_enforcer.clear)
 
     def _enforce_policy(self, action):
         api_args = self._get_args()
 
-        @policy.authorize(action)
+        @deckhand.policy.authorize(action)
         def noop(*args, **kwargs):
             pass
 
@@ -53,8 +51,7 @@ class PolicyBaseTestCase(test_base.DeckhandTestCase):
     def _get_args(self):
         # Returns the first two arguments that would be passed to any falcon
         # on_{HTTP_VERB} method: (self (which is mocked), falcon Request obj).
-        falcon_req = api_base.DeckhandRequest(
+        falcon_req = api_base.DeckhandRequest(mock.MagicMock())
-            mock.MagicMock(), policy_enforcer=self.policy_enforcer)
         return (mock.Mock(), falcon_req)
 
 

deckhand/utils.py

@@ -17,6 +17,8 @@ import string
 
 import jsonpath_ng
 
+from deckhand import errors
+
 
 def to_camel_case(s):
     """Convert string to camel case."""
@@ -30,30 +32,114 @@ def to_snake_case(name):
     return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower()
 
 
-def jsonpath_parse(document, jsonpath):
+def jsonpath_parse(data, jsonpath):
-    """Parse value given JSON path in the document.
+    """Parse value in the data for the given ``jsonpath``.
 
-    Retrieve the value corresponding to document[jsonpath] where ``jsonpath``
+    Retrieve the nested entry corresponding to ``data[jsonpath]``. For
-    is a multi-part key. A multi-key is a series of keys and nested keys
+    example, a ``jsonpath`` of ".foo.bar.baz" means that the data section
-    concatenated together with ".". For exampple, ``jsonpath`` of
+    should conform to:
-    ".foo.bar.baz" should mean that ``document`` has the format:
 
     .. code-block:: yaml
 
-       ---
+        ---
-       foo:
+        foo:
-           bar:
+            bar:
-               baz: <data_to_be_extracted_here>
+                baz: <data_to_be_extracted_here>
 
-    :param document: Dictionary used for extracting nested entry.
+    :param data: The `data` section of a document.
-    :param jsonpath: A multi-part key that references nested data in a
+    :param jsonpath: A multi-part key that references a nested path in
-        dictionary.
+        ``data``.
-    :returns: Nested entry in ``document`` if present, else None.
+    :returns: Entry that corresponds to ``data[jsonpath]`` if present,
+        else None.
+
+    Example::
+
+        src_name = sub['src']['name']
+        src_path = sub['src']['path']
+        src_doc = db_api.document_get(schema=src_schema, name=src_name)
+        src_secret = utils.jsonpath_parse(src_doc['data'], src_path)
+        # Do something with the extracted secret from the source document.
     """
     if jsonpath.startswith('.'):
         jsonpath = '$' + jsonpath
 
     p = jsonpath_ng.parse(jsonpath)
-    matches = p.find(document)
+    matches = p.find(data)
     if matches:
         return matches[0].value
+
+
+def jsonpath_replace(data, value, jsonpath, pattern=None):
+    """Update value in ``data`` at the path specified by ``jsonpath``.
+
+    If the nested path corresponding to ``jsonpath`` isn't found in ``data``,
+    the path is created as an empty ``{}`` for each sub-path along the
+    ``jsonpath``.
+
+    :param data: The `data` section of a document.
+    :param value: The new value for ``data[jsonpath]``.
+    :param jsonpath: A multi-part key that references a nested path in
+        ``data``.
+    :param pattern: A regular expression pattern.
+    :returns: Updated value at ``data[jsonpath]``.
+    :raises: MissingDocumentPattern if ``pattern`` is not None and
+        ``data[jsonpath]`` doesn't exist.
+
+    Example::
+
+        doc = {
+            'data': {
+                'some_url': http://admin:INSERT_PASSWORD_HERE@svc-name:8080/v1
+            }
+        }
+        secret = 'super-duper-secret'
+        path = '$.some_url'
+        pattern = 'INSERT_[A-Z]+_HERE'
+        replaced_data = utils.jsonpath_replace(
+            doc['data'], secret, path, pattern)
+        # The returned URL will look like:
+        # http://admin:super-duper-secret@svc-name:8080/v1
+        doc['data'].update(replaced_data)
+    """
+    data = data.copy()
+    if jsonpath.startswith('.'):
+        jsonpath = '$' + jsonpath
+
+    def _do_replace():
+        p = jsonpath_ng.parse(jsonpath)
+        p_to_change = p.find(data)
+
+        if p_to_change:
+            _value = value
+            if pattern:
+                to_replace = p_to_change[0].value
+                # value represents the value to inject into to_replace that
+                # matches the pattern.
+                try:
+                    _value = re.sub(pattern, value, to_replace)
+                except TypeError:
+                    _value = None
+            return p.update(data, _value)
+
+    result = _do_replace()
+    if result:
+        return result
+
+    # A pattern requires us to look up the data located at data[jsonpath]
+    # and then figure out what re.match(data[jsonpath], pattern) is (in
+    # pseudocode). But raise an exception in case the path isn't present in the
+    # data and a pattern has been provided since it is impossible to do the
+    # look up.
+    if pattern:
+        raise errors.MissingDocumentPattern(
+            data=data, path=jsonpath, pattern=pattern)
+
+    # However, Deckhand should be smart enough to create the nested keys in the
+    # data if they don't exist and a pattern isn't required.
+    d = data
+    for path in jsonpath.split('.')[1:]:
+        if path not in d:
+            d.setdefault(path, {})
+        d = d.get(path)
+
+    return _do_replace()

doc/design.md

@@ -290,7 +290,7 @@ layer example above, which includes `global`, `region` and `site` layers, a
 document in the `region` layer could insert data from a document in the
 `site` layer.
 
-Here is a sample set of documents demonstrating subistution:
+Here is a sample set of documents demonstrating substitution:
 
 ```yaml
 ---

etc/deckhand/policy.yaml.sample

@@ -7,10 +7,6 @@
 # revision history, whereby the target revision's documents are re-
 # created for
 # the new revision.
-#
-# Conditionally enforced for the endpoints below if the any of the
-# documents in
-# the request body have a `metadata.storagePolicy` of "cleartext".
 # PUT  /api/v1.0/bucket/{bucket_name}/documents
 # POST  /api/v1.0/rollback/{target_revision_id}
 #"deckhand:create_cleartext_documents": "rule:admin_api"
@@ -22,9 +18,11 @@
 # the new
 # revision.
 #
+# Only enforced after ``create_cleartext_documents`` passes.
+#
 # Conditionally enforced for the endpoints below if the any of the
 # documents in
-# the request body have a `metadata.storagePolicy` of "encrypted".
+# the request body have a ``metadata.storagePolicy`` of "encrypted".
 # PUT  /api/v1.0/bucket/{bucket_name}/documents
 # POST  /api/v1.0/rollback/{target_revision_id}
 #"deckhand:create_encrypted_documents": "rule:admin_api"
@@ -33,31 +31,28 @@
 # substitution applied) as well as fully layered and substituted
 # concrete
 # documents.
-#
-# Conditionally enforced for the endpoints below if the any of the
-# documents in
-# the request body have a `metadata.storagePolicy` of "cleartext". If
-# policy
-# enforcement fails, cleartext documents are omitted.
 # GET  api/v1.0/revisions/{revision_id}/documents
 # GET  api/v1.0/revisions/{revision_id}/rendered-documents
 #"deckhand:list_cleartext_documents": "rule:admin_api"
 
-# List cleartext documents for a revision (with no layering or
+# List encrypted documents for a revision (with no layering or
 # substitution applied) as well as fully layered and substituted
 # concrete
 # documents.
 #
-# Conditionally enforced for the endpoints below if the any of the
+# Only enforced after ``list_cleartext_documents`` passes.
-# documents in
+#
-# the request body have a `metadata.storagePolicy` of "encrypted". If
+# Conditionally enforced for the endpoints below if any of the
+# documents in the
+# request body have a ``metadata.storagePolicy`` of "encrypted". If
 # policy
-# enforcement fails, encrypted documents are omitted.
+# enforcement fails, encrypted documents are exluded from the
+# response.
 # GET  api/v1.0/revisions/{revision_id}/documents
 # GET  api/v1.0/revisions/{revision_id}/rendered-documents
 #"deckhand:list_encrypted_documents": "rule:admin_api"
 
-# Show details for a revision tag.
+# Show details for a revision.
 # GET  /api/v1.0/revisions/{revision_id}
 #"deckhand:show_revision": "rule:admin_api"
 

releasenotes/notes/secret-substitution-6eff2c93bf11d82e.yaml

@@ -0,0 +1,8 @@
+---
+features:
+  - |
+    Deckhand now supports secret substitution for documents. The endpoint
+    ``GET revisions/{revision_id}/rendered-documents`` has been added to
+    Deckhand, which allows the possibility of listing fully substituted
+    documents. Only documents with ``metadata.substitutions`` field undergo
+    secret substitution dynamically.