From d2d2312af96f97409b7243dc77d1680bbb927386 Mon Sep 17 00:00:00 2001 From: Felipe Monteiro Date: Tue, 3 Oct 2017 20:18:47 +0100 Subject: [PATCH] DECKHAND-66: Document substitution implementation This PS implements documentation substitution and the rendered-documents endpoint. Each time the rendered-documents is queried, the documents for the reqeust revision_id dynamically undergo secret substitution. All functional tests related to secret substitution have been unskipped. Deckhand currently does not real testing for verifying that secret substitution works for encrypted documents. This will only happen when integration testing is added to Deckhand to test its interaction with Keystone and Barbican. Included in this PS: - basic implementation for secret substitution - introduction of jsonpath_ng for searching for and updating jsonpaths in documents - rendered-documents endpoint - unit tests - all relevant functional tests unskipped - additional bucket controller tests include RBAC tests and framework testing RBAC via unit tests Change-Id: I86f269a5b616b518e5f742a4005891412226fe2a --- AUTHORS | 2 + deckhand/context.py | 7 +- deckhand/control/api.py | 2 + deckhand/control/base.py | 5 +- deckhand/control/buckets.py | 75 +++--- deckhand/control/revision_documents.py | 59 ++++- deckhand/control/rollback.py | 6 +- deckhand/control/views/document.py | 31 ++- deckhand/control/views/revision.py | 2 +- deckhand/db/sqlalchemy/api.py | 27 ++- deckhand/engine/document.py | 8 +- .../schema/v1_0/certificate_key_schema.py | 7 + .../engine/schema/v1_0/certificate_schema.py | 7 + deckhand/engine/schema/v1_0/data_schema.py | 4 + .../engine/schema/v1_0/document_schema.py | 4 + .../engine/schema/v1_0/layering_schema.py | 6 +- .../engine/schema/v1_0/passphrase_schema.py | 7 + .../engine/schema/v1_0/validation_schema.py | 6 +- deckhand/engine/secrets_manager.py | 89 ++++++- deckhand/errors.py | 24 +- deckhand/factories.py | 29 ++- deckhand/policies/document.py | 25 +- deckhand/policy.py | 38 ++- ...ess-multiple-bucket-with-substitution.yaml | 10 +- ...ccess-single-bucket-with-substitution.yaml | 7 +- ...oc-substitution-sample-split-bucket-a.yaml | 3 +- ...oc-substitution-sample-split-bucket-b.yaml | 8 +- .../design-doc-substitution-sample.yaml | 11 +- .../gabbits/revision-diff-success.yaml | 4 + deckhand/tests/unit/control/base.py | 2 + .../unit/control/test_api_initialization.py | 19 +- .../unit/control/test_buckets_controller.py | 132 ++++++++++- deckhand/tests/unit/db/base.py | 3 +- .../tests/unit/engine/test_secrets_manager.py | 218 +++++++++++++++++- deckhand/tests/unit/fake_policy.py | 31 +++ deckhand/tests/unit/policy_fixture.py | 74 ++++++ deckhand/tests/unit/test_policy.py | 19 +- deckhand/utils.py | 116 ++++++++-- doc/design.md | 2 +- etc/deckhand/policy.yaml.sample | 29 +-- .../secret-substitution-6eff2c93bf11d82e.yaml | 8 + 41 files changed, 959 insertions(+), 207 deletions(-) create mode 100644 deckhand/tests/unit/fake_policy.py create mode 100644 deckhand/tests/unit/policy_fixture.py create mode 100644 releasenotes/notes/secret-substitution-6eff2c93bf11d82e.yaml diff --git a/AUTHORS b/AUTHORS index c07cb36e..b8399261 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,7 +1,9 @@ Alan Meadows Anthony Lin +Bryan Strassner Felipe Monteiro Felipe Monteiro Mark Burnett Pete Birley Scott Hussey +Tin Lam diff --git a/deckhand/context.py b/deckhand/context.py index b4206e15..aa0d484c 100644 --- a/deckhand/context.py +++ b/deckhand/context.py @@ -14,9 +14,6 @@ from oslo_config import cfg from oslo_context import context -from oslo_policy import policy as common_policy - -from deckhand import policy CONF = cfg.CONF @@ -28,12 +25,10 @@ class RequestContext(context.RequestContext): accesses the system, as well as additional request information. """ - def __init__(self, policy_enforcer=None, project=None, **kwargs): + def __init__(self, project=None, **kwargs): if project: kwargs['tenant'] = project self.project = project - self.policy_enforcer = policy_enforcer or common_policy.Enforcer(CONF) - policy.register_rules(self.policy_enforcer) super(RequestContext, self).__init__(**kwargs) def to_dict(self): diff --git a/deckhand/control/api.py b/deckhand/control/api.py index fe6adbbc..23b064ea 100644 --- a/deckhand/control/api.py +++ b/deckhand/control/api.py @@ -67,6 +67,8 @@ def start_api(): revision_diffing.RevisionDiffingResource()), ('revisions/{revision_id}/documents', revision_documents.RevisionDocumentsResource()), + ('revisions/{revision_id}/rendered-documents', + revision_documents.RenderedDocumentsResource()), ('revisions/{revision_id}/tags', revision_tags.RevisionTagsResource()), ('revisions/{revision_id}/tags/{tag}', revision_tags.RevisionTagsResource()), diff --git a/deckhand/control/base.py b/deckhand/control/base.py index ff7ad319..b44d7c9f 100644 --- a/deckhand/control/base.py +++ b/deckhand/control/base.py @@ -51,10 +51,9 @@ class BaseResource(object): class DeckhandRequest(falcon.Request): - def __init__(self, env, options=None, policy_enforcer=None): + def __init__(self, env, options=None): super(DeckhandRequest, self).__init__(env, options) - self.context = context.RequestContext.from_environ( - self.env, policy_enforcer=policy_enforcer) + self.context = context.RequestContext.from_environ(self.env) @property def project_id(self): diff --git a/deckhand/control/buckets.py b/deckhand/control/buckets.py index f0575d55..2e1dee2a 100644 --- a/deckhand/control/buckets.py +++ b/deckhand/control/buckets.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import itertools import yaml import falcon @@ -37,6 +36,7 @@ class BucketsResource(api_base.BaseResource): view_builder = document_view.ViewBuilder() secrets_mgr = secrets_manager.SecretsManager() + @policy.authorize('deckhand:create_cleartext_documents') def on_put(self, req, resp, bucket_name=None): document_data = req.stream.read(req.content_length or 0) try: @@ -47,10 +47,34 @@ class BucketsResource(api_base.BaseResource): LOG.error(error_msg) raise falcon.HTTPBadRequest(description=six.text_type(e)) + # NOTE: Must validate documents before doing policy enforcement, + # because we expect certain formatting of the documents while doing + # policy enforcement. + validation_policies = self._create_validation_policies(documents) + + for document in documents: + if document['metadata'].get('storagePolicy') == 'encrypted': + policy.conditional_authorize( + 'deckhand:create_encrypted_documents', req.context) + break + + self._prepare_secret_documents(documents) + + # Save all the documents, including validation policies. + documents_to_create = documents + validation_policies + created_documents = self._create_revision_documents( + bucket_name, list(documents_to_create)) + + if created_documents: + resp.body = self.to_yaml_body( + self.view_builder.list(created_documents)) + resp.status = falcon.HTTP_200 + resp.append_header('Content-Type', 'application/x-yaml') + + def _create_validation_policies(self, documents): + # All concrete documents in the payload must successfully pass their + # JSON schema validations. Otherwise raise an error. try: - # NOTE: Must validate documents before doing policy enforcement, - # because we expect certain formatting of the documents while doing - # policy enforcement. validation_policies = document_validation.DocumentValidation( documents).validate_all() except deckhand_errors.InvalidDocumentFormat as e: @@ -58,42 +82,25 @@ class BucketsResource(api_base.BaseResource): # validation policy in the DB for future debugging, and only # afterward raise an exception. raise falcon.HTTPBadRequest(description=e.format_message()) + return validation_policies - cleartext_documents = [] - secret_documents = [] - - for document in documents: - if any([document['schema'].startswith(t) - for t in types.DOCUMENT_SECRET_TYPES]): - secret_documents.append(document) - else: - cleartext_documents.append(document) - - if secret_documents and any( - [d['metadata'].get('storagePolicy') == 'encrypted' - for d in secret_documents]): - policy.conditional_authorize('deckhand:create_encrypted_documents', - req.context) - if cleartext_documents: - policy.conditional_authorize('deckhand:create_cleartext_documents', - req.context) - + def _prepare_secret_documents(self, secret_documents): + # Encrypt data for secret documents, if any. for document in secret_documents: - secret_data = self.secrets_mgr.create(document) - document['data'] = secret_data + # TODO(fmontei): Move all of this to document validation directly. + if document['metadata'].get('storagePolicy') == 'encrypted': + secret_data = self.secrets_mgr.create(document) + document['data'] = secret_data + elif any([document['schema'].startswith(t) + for t in types.DOCUMENT_SECRET_TYPES]): + document['data'] = {'secret': document['data']} + def _create_revision_documents(self, bucket_name, documents): try: - documents_to_create = itertools.chain( - cleartext_documents, secret_documents, validation_policies) - created_documents = db_api.documents_create( - bucket_name, list(documents_to_create)) + created_documents = db_api.documents_create(bucket_name, documents) except deckhand_errors.DocumentExists as e: raise falcon.HTTPConflict(description=e.format_message()) except Exception as e: raise falcon.HTTPInternalServerError(description=six.text_type(e)) - if created_documents: - resp.body = self.to_yaml_body( - self.view_builder.list(created_documents)) - resp.status = falcon.HTTP_200 - resp.append_header('Content-Type', 'application/x-yaml') + return created_documents diff --git a/deckhand/control/revision_documents.py b/deckhand/control/revision_documents.py index a0a8cb00..9262c473 100644 --- a/deckhand/control/revision_documents.py +++ b/deckhand/control/revision_documents.py @@ -19,6 +19,7 @@ from deckhand.control import base as api_base from deckhand.control import common from deckhand.control.views import document as document_view from deckhand.db.sqlalchemy import api as db_api +from deckhand.engine import secrets_manager from deckhand import errors from deckhand import policy @@ -26,10 +27,11 @@ LOG = logging.getLogger(__name__) class RevisionDocumentsResource(api_base.BaseResource): - """API resource for realizing CRUD endpoints for revision documents.""" + """API resource for realizing revision documents endpoint.""" view_builder = document_view.ViewBuilder() + @policy.authorize('deckhand:list_cleartext_documents') @common.sanitize_params([ 'schema', 'metadata.name', 'metadata.layeringDefinition.abstract', 'metadata.layeringDefinition.layer', 'metadata.label', @@ -42,18 +44,13 @@ class RevisionDocumentsResource(api_base.BaseResource): documents will be as originally posted with no substitutions or layering applied. """ - include_cleartext = policy.conditional_authorize( - 'deckhand:list_cleartext_documents', req.context, do_raise=False) include_encrypted = policy.conditional_authorize( 'deckhand:list_encrypted_documents', req.context, do_raise=False) filters = sanitized_params.copy() - filters['metadata.storagePolicy'] = [] - if include_cleartext: - filters['metadata.storagePolicy'].append('cleartext') + filters['metadata.storagePolicy'] = ['cleartext'] if include_encrypted: filters['metadata.storagePolicy'].append('encrypted') - # Never return deleted documents to user. filters['deleted'] = False @@ -66,3 +63,51 @@ class RevisionDocumentsResource(api_base.BaseResource): resp.status = falcon.HTTP_200 resp.append_header('Content-Type', 'application/x-yaml') resp.body = self.to_yaml_body(self.view_builder.list(documents)) + + +class RenderedDocumentsResource(api_base.BaseResource): + """API resource for realizing rendered documents endpoint. + + Rendered documents are also revision documents, but unlike revision + documents, they are finalized documents, having undergone secret + substitution and document layering. + + Returns a multi-document YAML response containing all the documents + matching the filters specified via query string parameters. Returned + documents will have secrets substituted into them and be layered with + other documents in the revision, in accordance with the ``LayeringPolicy`` + that currently exists in the system. + """ + + view_builder = document_view.ViewBuilder() + + @policy.authorize('deckhand:list_cleartext_documents') + @common.sanitize_params([ + 'schema', 'metadata.name', 'metadata.label']) + def on_get(self, req, resp, sanitized_params, revision_id): + include_encrypted = policy.conditional_authorize( + 'deckhand:list_encrypted_documents', req.context, do_raise=False) + + filters = sanitized_params.copy() + filters['metadata.storagePolicy'] = ['cleartext'] + if include_encrypted: + filters['metadata.storagePolicy'].append('encrypted') + + try: + documents = db_api.revision_get_documents( + revision_id, **filters) + except (errors.RevisionNotFound) as e: + raise falcon.HTTPNotFound(description=e.format_message()) + + # TODO(fmontei): Currently the only phase of rendering that is + # performed is secret substitution, which can be done in any randomized + # order. However, secret substitution logic will have to be moved into + # a separate module that handles layering alongside substitution once + # layering has been fully integrated into this endpoint. + secrets_substitution = secrets_manager.SecretsSubstitution(documents) + rendered_documents = secrets_substitution.substitute_all() + + resp.status = falcon.HTTP_200 + resp.append_header('Content-Type', 'application/x-yaml') + resp.body = self.to_yaml_body( + self.view_builder.list(rendered_documents)) diff --git a/deckhand/control/rollback.py b/deckhand/control/rollback.py index 96f1c7b6..77169fb8 100644 --- a/deckhand/control/rollback.py +++ b/deckhand/control/rollback.py @@ -34,12 +34,10 @@ class RollbackResource(api_base.BaseResource): raise falcon.HTTPNotFound(description=e.format_message()) for document in latest_revision['documents']: - if document['metadata'].get('storagePolicy') == 'cleartext': - policy.conditional_authorize( - 'deckhand:create_cleartext_documents', req.context) - elif document['metadata'].get('storagePolicy') == 'encrypted': + if document['metadata'].get('storagePolicy') == 'encrypted': policy.conditional_authorize( 'deckhand:create_encrypted_documents', req.context) + break try: rollback_revision = db_api.revision_rollback( diff --git a/deckhand/control/views/document.py b/deckhand/control/views/document.py index ec126e05..ed9d47d7 100644 --- a/deckhand/control/views/document.py +++ b/deckhand/control/views/document.py @@ -13,6 +13,7 @@ # limitations under the License. from deckhand.control import common +from deckhand import types class ViewBuilder(common.ViewBuilder): @@ -30,34 +31,30 @@ class ViewBuilder(common.ViewBuilder): _collection_name = 'documents' def list(self, documents): - # Edge case for when all documents are deleted from a bucket. Still - # need to return bucket_id and revision_id. - if len(documents) == 1 and documents[0]['deleted']: - resp_obj = {'status': {}} - resp_obj['status']['bucket'] = documents[0]['bucket_name'] - resp_obj['status']['revision'] = documents[0]['revision_id'] - return [resp_obj] - resp_list = [] attrs = ['id', 'metadata', 'data', 'schema'] for document in documents: + if document['deleted']: + continue + if document['schema'].startswith(types.VALIDATION_POLICY_SCHEMA): + continue resp_obj = {x: document[x] for x in attrs} resp_obj.setdefault('status', {}) resp_obj['status']['bucket'] = document['bucket_name'] resp_obj['status']['revision'] = document['revision_id'] resp_list.append(resp_obj) - # In the case where no documents are passed to PUT - # buckets/{{bucket_name}}/documents, we need to mangle the response - # body a bit. The revision_id and buckete_id should be returned, as - # at the very least the revision_id will be needed by the user. + # Edge case for when all documents are deleted from a bucket. To detect + # the edge case, check whether ``resp_list`` is empty and whether there + # are still documents to be returned. This means that all the documents + # are either deleted or validation policies. Either way, we still need + # to return bucket_id and revision_id, which should be the same + # across all the documents in ``documents``. if not resp_list and documents: - resp_obj = {} - resp_obj.setdefault('status', {}) - resp_obj['status']['bucket'] = documents[0]['bucket_id'] + resp_obj = {'status': {}} + resp_obj['status']['bucket'] = documents[0]['bucket_name'] resp_obj['status']['revision'] = documents[0]['revision_id'] - - resp_list.append(resp_obj) + return [resp_obj] return resp_list diff --git a/deckhand/control/views/revision.py b/deckhand/control/views/revision.py index da15f554..569309df 100644 --- a/deckhand/control/views/revision.py +++ b/deckhand/control/views/revision.py @@ -59,7 +59,7 @@ class ViewBuilder(common.ViewBuilder): success_status = 'success' for vp in [d for d in revision['documents'] - if d['schema'] == types.VALIDATION_POLICY_SCHEMA]: + if d['schema'].startswith(types.VALIDATION_POLICY_SCHEMA)]: validation_policy = {} validation_policy['name'] = vp.get('name') validation_policy['url'] = self._gen_url(vp) diff --git a/deckhand/db/sqlalchemy/api.py b/deckhand/db/sqlalchemy/api.py index c08a1dca..c346dd12 100644 --- a/deckhand/db/sqlalchemy/api.py +++ b/deckhand/db/sqlalchemy/api.py @@ -261,18 +261,29 @@ def document_get(session=None, raw_dict=False, **filters): """ session = session or get_session() - # Retrieve the most recently created version of a document. Documents with - # the same metadata.name and schema can exist across different revisions, - # so it is necessary to use `first` instead of `one` to avoid errors. - document = session.query(models.Document)\ + # TODO(fmontei): Currently Deckhand doesn't support filtering by nested + # JSON fields via sqlalchemy. For now, filter the documents using all + # "regular" filters via sqlalchemy and all nested filters via Python. + nested_filters = {} + for f in filters.copy(): + if '.' in f: + nested_filters.setdefault(f, filters.pop(f)) + + # Documents with the the same metadata.name and schema can exist across + # different revisions, so it is necessary to order documents by creation + # date, then return the first document that matches all desired filters. + documents = session.query(models.Document)\ .filter_by(**filters)\ .order_by(models.Document.created_at.desc())\ - .first() + .all() - if not document: - raise errors.DocumentNotFound(document=filters) + for doc in documents: + d = doc.to_dict(raw_dict=raw_dict) + if _apply_filters(d, **nested_filters): + return d - return document.to_dict(raw_dict=raw_dict) + filters.update(nested_filters) + raise errors.DocumentNotFound(document=filters) #################### diff --git a/deckhand/engine/document.py b/deckhand/engine/document.py index af6587bc..d04f7783 100644 --- a/deckhand/engine/document.py +++ b/deckhand/engine/document.py @@ -72,6 +72,9 @@ class Document(object): def get_labels(self): return self._inner['metadata']['labels'] + def get_substitutions(self): + return self._inner['metadata'].get('substitutions', None) + def get_actions(self): try: return self._inner['metadata']['layeringDefinition']['actions'] @@ -118,4 +121,7 @@ class Document(object): return not self.__contains__(k) def __repr__(self): - return repr(self._inner) + return '(%s, %s)' % (self.get_schema(), self.get_name()) + + def __str__(self): + return str(self._inner) diff --git a/deckhand/engine/schema/v1_0/certificate_key_schema.py b/deckhand/engine/schema/v1_0/certificate_key_schema.py index 92c08fb8..93abd0a1 100644 --- a/deckhand/engine/schema/v1_0/certificate_key_schema.py +++ b/deckhand/engine/schema/v1_0/certificate_key_schema.py @@ -27,6 +27,13 @@ schema = { 'pattern': '^(metadata/Document/v[1]{1}(\.[0]{1}){0,1})$', }, 'name': {'type': 'string'}, + # Not strictly needed for secrets. + 'layeringDefinition': { + 'type': 'object', + 'properties': { + 'layer': {'type': 'string'} + } + }, 'storagePolicy': { 'type': 'string', 'enum': ['encrypted', 'cleartext'] diff --git a/deckhand/engine/schema/v1_0/certificate_schema.py b/deckhand/engine/schema/v1_0/certificate_schema.py index 9335b6db..25f63a66 100644 --- a/deckhand/engine/schema/v1_0/certificate_schema.py +++ b/deckhand/engine/schema/v1_0/certificate_schema.py @@ -27,6 +27,13 @@ schema = { 'pattern': '^(metadata/Document/v[1]{1}(\.[0]{1}){0,1})$', }, 'name': {'type': 'string'}, + # Not strictly needed for secrets. + 'layeringDefinition': { + 'type': 'object', + 'properties': { + 'layer': {'type': 'string'} + } + }, 'storagePolicy': { 'type': 'string', 'enum': ['encrypted', 'cleartext'] diff --git a/deckhand/engine/schema/v1_0/data_schema.py b/deckhand/engine/schema/v1_0/data_schema.py index 346b2d02..8185763a 100644 --- a/deckhand/engine/schema/v1_0/data_schema.py +++ b/deckhand/engine/schema/v1_0/data_schema.py @@ -33,6 +33,10 @@ schema = { # Labels are optional. 'labels': { 'type': 'object' + }, + 'storagePolicy': { + 'type': 'string', + 'enum': ['encrypted', 'cleartext'] } }, 'additionalProperties': False, diff --git a/deckhand/engine/schema/v1_0/document_schema.py b/deckhand/engine/schema/v1_0/document_schema.py index f903e2b2..eaf16d4e 100644 --- a/deckhand/engine/schema/v1_0/document_schema.py +++ b/deckhand/engine/schema/v1_0/document_schema.py @@ -84,6 +84,10 @@ schema = { 'substitutions': { 'type': 'array', 'items': substitution_schema + }, + 'storagePolicy': { + 'type': 'string', + 'enum': ['encrypted', 'cleartext'] } }, 'additionalProperties': False, diff --git a/deckhand/engine/schema/v1_0/layering_schema.py b/deckhand/engine/schema/v1_0/layering_schema.py index f03c4443..ace8e863 100644 --- a/deckhand/engine/schema/v1_0/layering_schema.py +++ b/deckhand/engine/schema/v1_0/layering_schema.py @@ -26,7 +26,11 @@ schema = { 'type': 'string', 'pattern': '^(metadata/Control/v[1]{1}(\.[0]{1}){0,1})$' }, - 'name': {'type': 'string'} + 'name': {'type': 'string'}, + 'storagePolicy': { + 'type': 'string', + 'enum': ['encrypted', 'cleartext'] + } }, 'additionalProperties': False, 'required': ['schema', 'name'] diff --git a/deckhand/engine/schema/v1_0/passphrase_schema.py b/deckhand/engine/schema/v1_0/passphrase_schema.py index 679a79e5..e9e7b211 100644 --- a/deckhand/engine/schema/v1_0/passphrase_schema.py +++ b/deckhand/engine/schema/v1_0/passphrase_schema.py @@ -27,6 +27,13 @@ schema = { 'pattern': '^(metadata/Document/v[1]{1}(\.[0]{1}){0,1})$', }, 'name': {'type': 'string'}, + # Not strictly needed for secrets. + 'layeringDefinition': { + 'type': 'object', + 'properties': { + 'layer': {'type': 'string'} + } + }, 'storagePolicy': { 'type': 'string', 'enum': ['encrypted', 'cleartext'] diff --git a/deckhand/engine/schema/v1_0/validation_schema.py b/deckhand/engine/schema/v1_0/validation_schema.py index 9681265d..8416bc2a 100644 --- a/deckhand/engine/schema/v1_0/validation_schema.py +++ b/deckhand/engine/schema/v1_0/validation_schema.py @@ -26,7 +26,11 @@ schema = { 'type': 'string', 'pattern': '^(metadata/Control/v[1]{1}(\.[0]{1}){0,1})$' }, - 'name': {'type': 'string'} + 'name': {'type': 'string'}, + 'storagePolicy': { + 'type': 'string', + 'enum': ['encrypted', 'cleartext'] + } }, 'additionalProperties': False, 'required': ['schema', 'name'] diff --git a/deckhand/engine/secrets_manager.py b/deckhand/engine/secrets_manager.py index a4179fce..41f0f80f 100644 --- a/deckhand/engine/secrets_manager.py +++ b/deckhand/engine/secrets_manager.py @@ -12,7 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +from oslo_log import log as logging + from deckhand.barbican import driver +from deckhand.db.sqlalchemy import api as db_api +from deckhand.engine import document as document_wrapper +from deckhand import utils + +LOG = logging.getLogger(__name__) CLEARTEXT = 'cleartext' ENCRYPTED = 'encrypted' @@ -34,22 +41,22 @@ class SecretsManager(object): documents with the schemas enumerated below) must be stored using a secure storage service like Barbican. - Documents with metadata.storagePolicy == "clearText" have their secrets - stored directly in Deckhand. + Documents with ``metadata.storagePolicy`` == "clearText" have their + secrets stored directly in Deckhand. - Documents with metadata.storagePolicy == "encrypted" are stored in + Documents with ``metadata.storagePolicy`` == "encrypted" are stored in Barbican directly. Deckhand in turn stores the reference returned by Barbican in Deckhand. :param secret_doc: A Deckhand document with one of the following schemas: - * deckhand/Certificate/v1 - * deckhand/CertificateKey/v1 - * deckhand/Passphrase/v1 + * ``deckhand/Certificate/v1`` + * ``deckhand/CertificateKey/v1`` + * ``deckhand/Passphrase/v1`` :returns: Dictionary representation of - `deckhand.db.sqlalchemy.models.DocumentSecret`. + ``deckhand.db.sqlalchemy.models.DocumentSecret``. """ encryption_type = secret_doc['metadata']['storagePolicy'] secret_type = self._get_secret_type(secret_doc['schema']) @@ -73,9 +80,9 @@ class SecretsManager(object): def _get_secret_type(self, schema): """Get the Barbican secret type based on the following mapping: - deckhand/Certificate/v1 => certificate - deckhand/CertificateKey/v1 => private - deckhand/Passphrase/v1 => passphrase + ``deckhand/Certificate/v1`` => certificate + ``deckhand/CertificateKey/v1`` => private + ``deckhand/Passphrase/v1`` => passphrase :param schema: The document's schema. :returns: The value corresponding to the mapping above. @@ -84,3 +91,65 @@ class SecretsManager(object): if _schema == 'certificatekey': return 'private' return _schema + + +class SecretsSubstitution(object): + """Class for document substitution logic for YAML files.""" + + def __init__(self, documents): + """SecretSubstitution constructor. + + :param documents: List of YAML documents in dictionary format that are + candidates for secret substitution. This class will automatically + detect documents that require substitution; documents need not be + filtered prior to being passed to the constructor. + """ + if not isinstance(documents, (list, tuple)): + documents = [documents] + substitute_docs = [document_wrapper.Document(d) for d in documents if + 'substitutions' in d['metadata']] + self.documents = substitute_docs + + def substitute_all(self): + """Substitute all documents that have a `metadata.substitutions` field. + + Concrete (non-abstract) documents can be used as a source of + substitution into other documents. This substitution is + layer-independent, a document in the region layer could insert data + from a document in the site layer. + + :returns: List of fully substituted documents. + """ + LOG.debug('Substituting secrets for documents: %s', self.documents) + substituted_docs = [] + + for doc in self.documents: + LOG.debug( + 'Checking for substitutions in schema=%s, metadata.name=%s', + doc.get_name(), doc.get_schema()) + for sub in doc.get_substitutions(): + src_schema = sub['src']['schema'] + src_name = sub['src']['name'] + src_path = sub['src']['path'] + if src_path == '.': + src_path = '.secret' + + # TODO(fmontei): Use secrets_manager for this logic. Need to + # check Barbican for the secret if it has been encrypted. + src_doc = db_api.document_get( + schema=src_schema, name=src_name, is_secret=True, + **{'metadata.layeringDefinition.abstract': False}) + src_secret = utils.jsonpath_parse(src_doc['data'], src_path) + + dest_path = sub['dest']['path'] + dest_pattern = sub['dest'].get('pattern', None) + + LOG.debug('Substituting from schema=%s name=%s src_path=%s ' + 'into dest_path=%s, dest_pattern=%s', src_schema, + src_name, src_path, dest_path, dest_pattern) + substituted_data = utils.jsonpath_replace( + doc['data'], src_secret, dest_path, dest_pattern) + doc['data'].update(substituted_data) + + substituted_docs.append(doc.to_dict()) + return substituted_docs diff --git a/deckhand/errors.py b/deckhand/errors.py index 45c8a062..30d3b4a3 100644 --- a/deckhand/errors.py +++ b/deckhand/errors.py @@ -23,18 +23,11 @@ class DeckhandException(Exception): code = 500 def __init__(self, message=None, **kwargs): - self.kwargs = kwargs - - if 'code' not in self.kwargs: - try: - self.kwargs['code'] = self.code - except AttributeError: - pass + kwargs.setdefault('code', DeckhandException.code) if not message: try: message = self.msg_fmt % kwargs - except Exception: message = self.msg_fmt @@ -58,15 +51,6 @@ class InvalidDocumentFormat(DeckhandException): super(InvalidDocumentFormat, self).__init__(**kwargs) -# TODO(fmontei): Remove this in a future commit. -class ApiError(Exception): - pass - - -class InvalidFormat(ApiError): - """The YAML file is incorrectly formatted and cannot be read.""" - - class DocumentExists(DeckhandException): msg_fmt = ("Document with schema %(schema)s and metadata.name " "%(name)s already exists in bucket %(bucket)s.") @@ -100,6 +84,12 @@ class MissingDocumentKey(DeckhandException): "Parent: %(parent)s. Child: %(child)s.") +class MissingDocumentPattern(DeckhandException): + msg_fmt = ("Substitution pattern %(pattern)s could not be found for the " + "JSON path %(path)s in the destination document data %(data)s.") + code = 400 + + class UnsupportedActionMethod(DeckhandException): msg_fmt = ("Method in %(actions)s is invalid for document %(document)s.") code = 400 diff --git a/deckhand/factories.py b/deckhand/factories.py index 76a16563..26f70889 100644 --- a/deckhand/factories.py +++ b/deckhand/factories.py @@ -61,7 +61,6 @@ class DocumentFactory(DeckhandFactory): "layeringDefinition": { "abstract": False, "layer": "", - "parentSelector": "", "actions": [] }, "name": "", @@ -92,7 +91,7 @@ class DocumentFactory(DeckhandFactory): ] :param num_layers: Total number of layers. Only supported values - include 2 or 3. + include 1, 2 or 3. :type num_layers: integer :param docs_per_layer: The number of documents to be included per layer. For example, if ``num_layers`` is 3, then ``docs_per_layer`` @@ -105,12 +104,14 @@ class DocumentFactory(DeckhandFactory): compatible with ``docs_per_layer``. """ # Set up the layering definition's layerOrder. - if num_layers == 2: + if num_layers == 1: + layer_order = ["global"] + elif num_layers == 2: layer_order = ["global", "site"] elif num_layers == 3: layer_order = ["global", "region", "site"] else: - raise ValueError("'num_layers' must either be 2 or 3.") + raise ValueError("'num_layers' must be a value between 1 - 3.") self.LAYERING_DEFINITION['data']['layerOrder'] = layer_order if not isinstance(docs_per_layer, (list, tuple)): @@ -225,14 +226,30 @@ class DocumentFactory(DeckhandFactory): data_key = "_%s_DATA_%d_" % (layer_name.upper(), count + 1) actions_key = "_%s_ACTIONS_%d_" % ( layer_name.upper(), count + 1) + sub_key = "_%s_SUBSTITUTIONS_%d_" % ( + layer_name.upper(), count + 1) try: layer_template['data'] = mapping[data_key]['data'] + except KeyError as e: + LOG.debug('Could not map %s because it was not found in ' + 'the `mapping` dict.', e.args[0]) + pass + + try: layer_template['metadata']['layeringDefinition'][ 'actions'] = mapping[actions_key]['actions'] except KeyError as e: - LOG.warning('Could not map %s because it was not found in ' - 'the `mapping` dict.', e.args[0]) + LOG.debug('Could not map %s because it was not found in ' + 'the `mapping` dict.', e.args[0]) + pass + + try: + layer_template['metadata']['substitutions'] = mapping[ + sub_key] + except KeyError as e: + LOG.debug('Could not map %s because it was not found in ' + 'the `mapping` dict.', e.args[0]) pass rendered_template.append(layer_template) diff --git a/deckhand/policies/document.py b/deckhand/policies/document.py index c54d7a3a..7433e950 100644 --- a/deckhand/policies/document.py +++ b/deckhand/policies/document.py @@ -24,10 +24,7 @@ document_policies = [ """Create a batch of documents specified in the request body, whereby a new revision is created. Also, roll back a revision to a previous one in the revision history, whereby the target revision's documents are re-created for -the new revision. - -Conditionally enforced for the endpoints below if the any of the documents in -the request body have a `metadata.storagePolicy` of "cleartext".""", +the new revision.""", [ { 'method': 'PUT', @@ -46,8 +43,10 @@ a new revision is created. Also, roll back a revision to a previous one in the history, whereby the target revision's documents are re-created for the new revision. +Only enforced after ``create_cleartext_documents`` passes. + Conditionally enforced for the endpoints below if the any of the documents in -the request body have a `metadata.storagePolicy` of "encrypted".""", +the request body have a ``metadata.storagePolicy`` of "encrypted".""", [ { 'method': 'PUT', @@ -63,11 +62,7 @@ the request body have a `metadata.storagePolicy` of "encrypted".""", base.RULE_ADMIN_API, """List cleartext documents for a revision (with no layering or substitution applied) as well as fully layered and substituted concrete -documents. - -Conditionally enforced for the endpoints below if the any of the documents in -the request body have a `metadata.storagePolicy` of "cleartext". If policy -enforcement fails, cleartext documents are omitted.""", +documents.""", [ { 'method': 'GET', @@ -81,13 +76,15 @@ enforcement fails, cleartext documents are omitted.""", policy.DocumentedRuleDefault( base.POLICY_ROOT % 'list_encrypted_documents', base.RULE_ADMIN_API, - """List cleartext documents for a revision (with no layering or + """List encrypted documents for a revision (with no layering or substitution applied) as well as fully layered and substituted concrete documents. -Conditionally enforced for the endpoints below if the any of the documents in -the request body have a `metadata.storagePolicy` of "encrypted". If policy -enforcement fails, encrypted documents are omitted.""", +Only enforced after ``list_cleartext_documents`` passes. + +Conditionally enforced for the endpoints below if any of the documents in the +request body have a ``metadata.storagePolicy`` of "encrypted". If policy +enforcement fails, encrypted documents are exluded from the response.""", [ { 'method': 'GET', diff --git a/deckhand/policy.py b/deckhand/policy.py index 9a2cb8ae..bda8b27b 100644 --- a/deckhand/policy.py +++ b/deckhand/policy.py @@ -25,22 +25,54 @@ from deckhand import policies CONF = cfg.CONF LOG = logging.getLogger(__name__) +_ENFORCER = None + + +def reset(): + global _ENFORCER + if _ENFORCER: + _ENFORCER.clear() + _ENFORCER = None + + +def init(policy_file=None, rules=None, default_rule=None, use_conf=True): + """Init an Enforcer class. + + :param policy_file: Custom policy file to use, if none is specified, + ``CONF.policy_file`` will be used. + :param rules: Default dictionary / Rules to use. It will be + considered just in the first instantiation. + :param default_rule: Default rule to use; ``CONF.default_rule`` will + be used if none is specified. + :param use_conf: Whether to load rules from config file. + """ + + global _ENFORCER + + if not _ENFORCER: + _ENFORCER = policy.Enforcer(CONF, + policy_file=policy_file, + rules=rules, + default_rule=default_rule, + use_conf=use_conf) + register_rules(_ENFORCER) def _do_enforce_rbac(action, context, do_raise=True): - policy_enforcer = context.policy_enforcer + init() + credentials = context.to_policy_values() target = {'project_id': context.project_id, 'user_id': context.user_id} exc = errors.PolicyNotAuthorized try: - # oslo.policy supports both enforce and authorize. authorize is + # `oslo.policy` supports both enforce and authorize. authorize is # stricter because it'll raise an exception if the policy action is # not found in the list of registered rules. This means that attempting # to enforce anything not found in ``deckhand.policies`` will error out # with a 'Policy not registered' message. - return policy_enforcer.authorize( + return _ENFORCER.authorize( action, target, context.to_dict(), do_raise=do_raise, exc=exc, action=action) except policy.PolicyNotRegistered as e: diff --git a/deckhand/tests/functional/gabbits/document-render-success-multiple-bucket-with-substitution.yaml b/deckhand/tests/functional/gabbits/document-render-success-multiple-bucket-with-substitution.yaml index 5745e270..6624ee8c 100644 --- a/deckhand/tests/functional/gabbits/document-render-success-multiple-bucket-with-substitution.yaml +++ b/deckhand/tests/functional/gabbits/document-render-success-multiple-bucket-with-substitution.yaml @@ -16,25 +16,22 @@ tests: desc: Begin testing from known state. DELETE: /api/v1.0/revisions status: 204 - skip: Not implemented. - name: add_bucket_a desc: Create documents for bucket a PUT: /api/v1.0/bucket/a/documents status: 200 - data: <@resources/design-doc-substition-sample-split-bucket-a.yaml - skip: Not implemented. + data: <@resources/design-doc-substitution-sample-split-bucket-a.yaml - name: add_bucket_b desc: Create documents for bucket b PUT: /api/v1.0/bucket/b/documents status: 200 - data: <@resources/design-doc-substition-sample-split-bucket-b.yaml - skip: Not implemented. + data: <@resources/design-doc-substitution-sample-split-bucket-b.yaml - name: verify_substitutions desc: Check for expected substitutions - GET: /api/v1.0/revisions/$RESPONSE['$.[0].revision']/rendered-documents?schema=armada/Chart/v1 + GET: /api/v1.0/revisions/$RESPONSE['$.[0].status.revision']/rendered-documents?schema=armada/Chart/v1 status: 200 response_multidoc_jsonpaths: $.[*].metadata.name: example-chart-01 @@ -49,4 +46,3 @@ tests: key: | KEY DATA some_url: http://admin:my-secret-password@service-name:8080/v1 - skip: Not implemented. diff --git a/deckhand/tests/functional/gabbits/document-render-success-single-bucket-with-substitution.yaml b/deckhand/tests/functional/gabbits/document-render-success-single-bucket-with-substitution.yaml index b9ebf190..1a34e472 100644 --- a/deckhand/tests/functional/gabbits/document-render-success-single-bucket-with-substitution.yaml +++ b/deckhand/tests/functional/gabbits/document-render-success-single-bucket-with-substitution.yaml @@ -15,18 +15,16 @@ tests: desc: Begin testing from known state. DELETE: /api/v1.0/revisions status: 204 - skip: Not implemented. - name: initialize desc: Create initial documents PUT: /api/v1.0/bucket/mop/documents status: 200 - data: <@resources/design-doc-substition-sample.yaml - skip: Not implemented. + data: <@resources/design-doc-substitution-sample.yaml - name: verify_substitutions desc: Check for expected substitutions - GET: /api/v1.0/revisions/$RESPONSE['$.[0].revision']/rendered-documents?schema=armada/Chart/v1 + GET: /api/v1.0/revisions/$RESPONSE['$.[0].status.revision']/rendered-documents?schema=armada/Chart/v1 status: 200 response_multidoc_jsonpaths: $.[*].metadata.name: example-chart-01 @@ -41,4 +39,3 @@ tests: key: | KEY DATA some_url: http://admin:my-secret-password@service-name:8080/v1 - skip: Not implemented. diff --git a/deckhand/tests/functional/gabbits/resources/design-doc-substitution-sample-split-bucket-a.yaml b/deckhand/tests/functional/gabbits/resources/design-doc-substitution-sample-split-bucket-a.yaml index 3b4fdf7c..42919292 100644 --- a/deckhand/tests/functional/gabbits/resources/design-doc-substitution-sample-split-bucket-a.yaml +++ b/deckhand/tests/functional/gabbits/resources/design-doc-substitution-sample-split-bucket-a.yaml @@ -2,9 +2,10 @@ schema: deckhand/CertificateKey/v1 metadata: name: example-key - storagePolicy: encrypted + schema: metadata/Document/v1 layeringDefinition: layer: site + storagePolicy: cleartext data: | KEY DATA ... diff --git a/deckhand/tests/functional/gabbits/resources/design-doc-substitution-sample-split-bucket-b.yaml b/deckhand/tests/functional/gabbits/resources/design-doc-substitution-sample-split-bucket-b.yaml index 229751ca..90e8f676 100644 --- a/deckhand/tests/functional/gabbits/resources/design-doc-substitution-sample-split-bucket-b.yaml +++ b/deckhand/tests/functional/gabbits/resources/design-doc-substitution-sample-split-bucket-b.yaml @@ -2,24 +2,26 @@ schema: deckhand/Certificate/v1 metadata: name: example-cert - storagePolicy: cleartext + schema: metadata/Document/v1 layeringDefinition: layer: site + storagePolicy: cleartext data: | CERTIFICATE DATA --- schema: deckhand/Passphrase/v1 metadata: name: example-password - storagePolicy: encrypted + schema: metadata/Document/v1 layeringDefinition: layer: site + storagePolicy: cleartext data: my-secret-password --- schema: armada/Chart/v1 metadata: name: example-chart-01 - storagePolicy: cleartext + schema: metadata/Document/v1 layeringDefinition: layer: region substitutions: diff --git a/deckhand/tests/functional/gabbits/resources/design-doc-substitution-sample.yaml b/deckhand/tests/functional/gabbits/resources/design-doc-substitution-sample.yaml index e8141341..3492d2c8 100644 --- a/deckhand/tests/functional/gabbits/resources/design-doc-substitution-sample.yaml +++ b/deckhand/tests/functional/gabbits/resources/design-doc-substitution-sample.yaml @@ -2,33 +2,36 @@ schema: deckhand/Certificate/v1 metadata: name: example-cert - storagePolicy: cleartext + schema: metadata/Document/v1 layeringDefinition: layer: site + storagePolicy: cleartext data: | CERTIFICATE DATA --- schema: deckhand/CertificateKey/v1 metadata: name: example-key - storagePolicy: encrypted + schema: metadata/Document/v1 layeringDefinition: layer: site + storagePolicy: cleartext data: | KEY DATA --- schema: deckhand/Passphrase/v1 metadata: name: example-password - storagePolicy: encrypted + schema: metadata/Document/v1 layeringDefinition: layer: site + storagePolicy: cleartext data: my-secret-password --- schema: armada/Chart/v1 metadata: name: example-chart-01 - storagePolicy: cleartext + schema: metadata/Document/v1 layeringDefinition: layer: region substitutions: diff --git a/deckhand/tests/functional/gabbits/revision-diff-success.yaml b/deckhand/tests/functional/gabbits/revision-diff-success.yaml index e84a0db6..f29df35f 100644 --- a/deckhand/tests/functional/gabbits/revision-diff-success.yaml +++ b/deckhand/tests/functional/gabbits/revision-diff-success.yaml @@ -196,6 +196,10 @@ tests: PUT: /api/v1.0/bucket/bucket_mistake/documents status: 200 data: "" + # Verification for whether a bucket_name was returned even though all the + # documents for this bucket were deleted. + response_multidoc_jsonpaths: + $.[*].status.bucket: bucket_mistake - name: verify_diff_between_created_and_deleted_mistake desc: Validates response for deletion between the last 2 revisions diff --git a/deckhand/tests/unit/control/base.py b/deckhand/tests/unit/control/base.py index ff98e58f..b4533712 100644 --- a/deckhand/tests/unit/control/base.py +++ b/deckhand/tests/unit/control/base.py @@ -16,6 +16,7 @@ from falcon import testing as falcon_testing from deckhand.control import api from deckhand.tests.unit import base as test_base +from deckhand.tests.unit import policy_fixture class BaseControllerTest(test_base.DeckhandWithDBTestCase, @@ -25,3 +26,4 @@ class BaseControllerTest(test_base.DeckhandWithDBTestCase, def setUp(self): super(BaseControllerTest, self).setUp() self.app = falcon_testing.TestClient(api.start_api()) + self.policy = self.useFixture(policy_fixture.RealPolicyFixture()) diff --git a/deckhand/tests/unit/control/test_api_initialization.py b/deckhand/tests/unit/control/test_api_initialization.py index 96895a89..1192566a 100644 --- a/deckhand/tests/unit/control/test_api_initialization.py +++ b/deckhand/tests/unit/control/test_api_initialization.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import inspect import mock from deckhand.control import api @@ -24,6 +25,7 @@ from deckhand.control import revisions from deckhand.control import rollback from deckhand.control import versions from deckhand.tests.unit import base as test_base +from deckhand import utils class TestApi(test_base.DeckhandTestCase): @@ -32,11 +34,16 @@ class TestApi(test_base.DeckhandTestCase): super(TestApi, self).setUp() for resource in (buckets, revision_diffing, revision_documents, revision_tags, revisions, rollback, versions): - resource_name = resource.__name__.split('.')[-1] - resource_obj = self.patchobject( - resource, '%sResource' % resource_name.title().replace( - '_', ''), autospec=True) - setattr(self, '%s_resource' % resource_name, resource_obj) + class_names = self._get_module_class_names(resource) + for class_name in class_names: + resource_obj = self.patchobject( + resource, class_name, autospec=True) + setattr(self, utils.to_snake_case(class_name), resource_obj) + + def _get_module_class_names(self, module): + class_names = [obj.__name__ for name, obj in inspect.getmembers(module) + if inspect.isclass(obj)] + return class_names @mock.patch.object(api, 'db_api', autospec=True) @mock.patch.object(api, 'logging', autospec=True) @@ -62,6 +69,8 @@ class TestApi(test_base.DeckhandTestCase): self.revision_diffing_resource()), mock.call('/api/v1.0/revisions/{revision_id}/documents', self.revision_documents_resource()), + mock.call('/api/v1.0/revisions/{revision_id}/rendered-documents', + self.rendered_documents_resource()), mock.call('/api/v1.0/revisions/{revision_id}/tags', self.revision_tags_resource()), mock.call('/api/v1.0/revisions/{revision_id}/tags/{tag}', diff --git a/deckhand/tests/unit/control/test_buckets_controller.py b/deckhand/tests/unit/control/test_buckets_controller.py index 2ee9a529..f1d90c36 100644 --- a/deckhand/tests/unit/control/test_buckets_controller.py +++ b/deckhand/tests/unit/control/test_buckets_controller.py @@ -12,17 +12,106 @@ # See the License for the specific language governing permissions and # limitations under the License. +import yaml + +import mock +from oslo_config import cfg + +from deckhand.control import buckets +from deckhand import factories from deckhand.tests.unit.control import base as test_base +CONF = cfg.CONF + class TestBucketsController(test_base.BaseControllerTest): - """Test suite for validating positive scenarios bucket controller.""" + """Test suite for validating positive scenarios for bucket controller.""" + + def test_put_bucket(self): + rules = {'deckhand:create_cleartext_documents': '@'} + self.policy.set_rules(rules) + + documents_factory = factories.DocumentFactory(2, [1, 1]) + document_mapping = { + "_GLOBAL_DATA_1_": {"data": {"a": {"x": 1, "y": 2}}}, + "_SITE_DATA_1_": {"data": {"a": {"x": 7, "z": 3}, "b": 4}}, + "_SITE_ACTIONS_1_": { + "actions": [{"method": "merge", "path": "."}]} + } + payload = documents_factory.gen_test(document_mapping) + + resp = self.app.simulate_put('/api/v1.0/bucket/mop/documents', + body=yaml.safe_dump_all(payload)) + self.assertEqual(200, resp.status_code) + created_documents = list(yaml.safe_load_all(resp.text)) + self.assertEqual(3, len(created_documents)) + expected = sorted([(d['schema'], d['metadata']['name']) + for d in payload]) + actual = sorted([(d['schema'], d['metadata']['name']) + for d in created_documents]) + self.assertEqual(expected, actual) + + def test_put_bucket_with_secret(self): + def _do_test(payload): + resp = self.app.simulate_put('/api/v1.0/bucket/mop/documents', + body=yaml.safe_dump_all(payload)) + self.assertEqual(200, resp.status_code) + created_documents = list(yaml.safe_load_all(resp.text)) + self.assertEqual(1, len(created_documents)) + expected = sorted([(d['schema'], d['metadata']['name']) + for d in payload]) + actual = sorted([(d['schema'], d['metadata']['name']) + for d in created_documents]) + self.assertEqual(expected, actual) + self.assertEqual({'secret': payload[0]['data']}, + created_documents[0]['data']) + + # Verify whether creating a cleartext secret works. + rules = {'deckhand:create_cleartext_documents': '@'} + self.policy.set_rules(rules) + + secrets_factory = factories.DocumentSecretFactory() + payload = [secrets_factory.gen_test('Certificate', 'cleartext')] + _do_test(payload) + + # Verify whether creating an encrypted secret works. + rules = {'deckhand:create_cleartext_documents': '@', + 'deckhand:create_encrypted_documents': '@'} + self.policy.set_rules(rules) + + secrets_factory = factories.DocumentSecretFactory() + payload = [secrets_factory.gen_test('Certificate', 'encrypted')] + + with mock.patch.object(buckets.BucketsResource, 'secrets_mgr', + autospec=True) as mock_secrets_mgr: + mock_secrets_mgr.create.return_value = { + 'secret': payload[0]['data']} + _do_test(payload) + + # Verify whether any document can be encrypted if its + # `metadata.storagePolicy`='encrypted'. In the case below, + # a generic document is tested. + documents_factory = factories.DocumentFactory(1, [1]) + document_mapping = { + "_GLOBAL_DATA_1_": {"data": {"a": {"x": 1, "y": 2}}} + } + payload = documents_factory.gen_test(document_mapping, + global_abstract=False) + payload[-1]['metadata']['storagePolicy'] = 'encrypted' + with mock.patch.object(buckets.BucketsResource, 'secrets_mgr', + autospec=True) as mock_secrets_mgr: + mock_secrets_mgr.create.return_value = { + 'secret': payload[-1]['data']} + _do_test([payload[-1]]) class TestBucketsControllerNegative(test_base.BaseControllerTest): - """Test suite for validating negative scenarios bucket controller.""" + """Test suite for validating negative scenarios for bucket controller.""" def test_put_bucket_with_invalid_document_payload(self): + rules = {'deckhand:create_cleartext_documents': '@'} + self.policy.set_rules(rules) + no_colon_spaces = """ name:foo schema: @@ -38,3 +127,42 @@ schema: body=payload) self.assertEqual(400, resp.status_code) self.assertRegexpMatches(resp.text, error_re[idx]) + + +class TestBucketsControllerNegativeRBAC(test_base.BaseControllerTest): + """Test suite for validating negative RBAC scenarios for bucket + controller. + """ + + def test_put_bucket_cleartext_documents_except_forbidden(self): + rules = {'deckhand:create_cleartext_documents': 'rule:admin_api'} + self.policy.set_rules(rules) + + documents_factory = factories.DocumentFactory(2, [1, 1]) + payload = documents_factory.gen_test({}) + + resp = self.app.simulate_put('/api/v1.0/bucket/mop/documents', + body=yaml.safe_dump_all(payload)) + self.assertEqual(403, resp.status_code) + + def test_put_bucket_cleartext_secret_except_forbidden(self): + rules = {'deckhand:create_cleartext_documents': 'rule:admin_api'} + self.policy.set_rules(rules) + + secrets_factory = factories.DocumentSecretFactory() + payload = [secrets_factory.gen_test('Certificate', 'cleartext')] + + resp = self.app.simulate_put('/api/v1.0/bucket/mop/documents', + body=yaml.safe_dump_all(payload)) + self.assertEqual(403, resp.status_code) + + def test_put_bucket_encrypted_secret_except_forbidden(self): + rules = {'deckhand:create_encrypted_documents': 'rule:admin_api'} + self.policy.set_rules(rules) + + secrets_factory = factories.DocumentSecretFactory() + payload = [secrets_factory.gen_test('Certificate', 'encrypted')] + + resp = self.app.simulate_put('/api/v1.0/bucket/mop/documents', + body=yaml.safe_dump_all(payload)) + self.assertEqual(403, resp.status_code) diff --git a/deckhand/tests/unit/db/base.py b/deckhand/tests/unit/db/base.py index 9d8948da..fd98670c 100644 --- a/deckhand/tests/unit/db/base.py +++ b/deckhand/tests/unit/db/base.py @@ -40,7 +40,8 @@ class DocumentFixture(object): 'layeringDefinition': { 'abstract': test_utils.rand_bool(), 'layer': test_utils.rand_name('layer') - } + }, + 'storagePolicy': test_utils.rand_name('storage_policy') }, 'schema': test_utils.rand_name('schema')} fixture.update(kwargs) diff --git a/deckhand/tests/unit/engine/test_secrets_manager.py b/deckhand/tests/unit/engine/test_secrets_manager.py index 8c5f59a6..a292534e 100644 --- a/deckhand/tests/unit/engine/test_secrets_manager.py +++ b/deckhand/tests/unit/engine/test_secrets_manager.py @@ -12,13 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +import copy + from deckhand.engine import secrets_manager from deckhand import factories from deckhand.tests import test_utils -from deckhand.tests.unit.db import base +from deckhand.tests.unit.db import base as test_base -class TestSecretsManager(base.TestDbBase): +class TestSecretsManager(test_base.TestDbBase): def setUp(self): super(TestSecretsManager, self).setUp() @@ -71,3 +73,215 @@ class TestSecretsManager(base.TestDbBase): def test_create_encrypted_passphrase(self): self._test_create_secret('encrypted', 'Passphrase') + + +class TestSecretsSubstitution(test_base.TestDbBase): + + def setUp(self): + super(TestSecretsSubstitution, self).setUp() + self.document_factory = factories.DocumentFactory(1, [1]) + self.secrets_factory = factories.DocumentSecretFactory() + + def _test_secret_substitution(self, document_mapping, secret_documents, + expected_data): + payload = self.document_factory.gen_test(document_mapping, + global_abstract=False) + bucket_name = test_utils.rand_name('bucket') + documents = self.create_documents( + bucket_name, secret_documents + [payload[-1]]) + + expected_documents = copy.deepcopy([documents[-1]]) + expected_documents[0]['data'] = expected_data + + secret_substitution = secrets_manager.SecretsSubstitution(documents) + substituted_docs = secret_substitution.substitute_all() + + self.assertEqual(expected_documents, substituted_docs) + + def test_secret_substitution_single_cleartext(self): + certificate = self.secrets_factory.gen_test( + 'Certificate', 'cleartext', data={'secret': 'CERTIFICATE DATA'}) + certificate['metadata']['name'] = 'example-cert' + + document_mapping = { + "_GLOBAL_SUBSTITUTIONS_1_": [{ + "dest": { + "path": ".chart.values.tls.certificate" + }, + "src": { + "schema": "deckhand/Certificate/v1", + "name": "example-cert", + "path": "." + } + + }] + } + expected_data = { + 'chart': { + 'values': { + 'tls': { + 'certificate': 'CERTIFICATE DATA' + } + } + } + } + self._test_secret_substitution( + document_mapping, [certificate], expected_data) + + def test_secret_substitution_single_cleartext_with_pattern(self): + passphrase = self.secrets_factory.gen_test( + 'Passphrase', 'cleartext', data={'secret': 'my-secret-password'}) + passphrase['metadata']['name'] = 'example-password' + + document_mapping = { + "_GLOBAL_DATA_1_": { + 'data': { + 'chart': { + 'values': { + 'some_url': ( + 'http://admin:INSERT_PASSWORD_HERE' + '@service-name:8080/v1') + } + } + } + }, + "_GLOBAL_SUBSTITUTIONS_1_": [{ + "dest": { + "path": ".chart.values.some_url", + "pattern": "INSERT_[A-Z]+_HERE" + }, + "src": { + "schema": "deckhand/Passphrase/v1", + "name": "example-password", + "path": "." + } + }] + } + expected_data = { + 'chart': { + 'values': { + 'some_url': ( + 'http://admin:my-secret-password@service-name:8080/v1') + } + } + } + self._test_secret_substitution( + document_mapping, [passphrase], expected_data) + + def test_secret_substitution_double_cleartext(self): + certificate = self.secrets_factory.gen_test( + 'Certificate', 'cleartext', data={'secret': 'CERTIFICATE DATA'}) + certificate['metadata']['name'] = 'example-cert' + + certificate_key = self.secrets_factory.gen_test( + 'CertificateKey', 'cleartext', data={'secret': 'KEY DATA'}) + certificate_key['metadata']['name'] = 'example-key' + + document_mapping = { + "_GLOBAL_SUBSTITUTIONS_1_": [{ + "dest": { + "path": ".chart.values.tls.certificate" + }, + "src": { + "schema": "deckhand/Certificate/v1", + "name": "example-cert", + "path": "." + } + + }, { + "dest": { + "path": ".chart.values.tls.key" + }, + "src": { + "schema": "deckhand/CertificateKey/v1", + "name": "example-key", + "path": "." + } + + }] + } + expected_data = { + 'chart': { + 'values': { + 'tls': { + 'certificate': 'CERTIFICATE DATA', + 'key': 'KEY DATA' + } + } + } + } + self._test_secret_substitution( + document_mapping, [certificate, certificate_key], expected_data) + + def test_secret_substitution_multiple_cleartext(self): + certificate = self.secrets_factory.gen_test( + 'Certificate', 'cleartext', data={'secret': 'CERTIFICATE DATA'}) + certificate['metadata']['name'] = 'example-cert' + + certificate_key = self.secrets_factory.gen_test( + 'CertificateKey', 'cleartext', data={'secret': 'KEY DATA'}) + certificate_key['metadata']['name'] = 'example-key' + + passphrase = self.secrets_factory.gen_test( + 'Passphrase', 'cleartext', data={'secret': 'my-secret-password'}) + passphrase['metadata']['name'] = 'example-password' + + document_mapping = { + "_GLOBAL_DATA_1_": { + 'data': { + 'chart': { + 'values': { + 'some_url': ( + 'http://admin:INSERT_PASSWORD_HERE' + '@service-name:8080/v1') + } + } + } + }, + "_GLOBAL_SUBSTITUTIONS_1_": [{ + "dest": { + "path": ".chart.values.tls.certificate" + }, + "src": { + "schema": "deckhand/Certificate/v1", + "name": "example-cert", + "path": "." + } + + }, { + "dest": { + "path": ".chart.values.tls.key" + }, + "src": { + "schema": "deckhand/CertificateKey/v1", + "name": "example-key", + "path": "." + } + + }, { + "dest": { + "path": ".chart.values.some_url", + "pattern": "INSERT_[A-Z]+_HERE" + }, + "src": { + "schema": "deckhand/Passphrase/v1", + "name": "example-password", + "path": "." + } + }] + } + expected_data = { + 'chart': { + 'values': { + 'tls': { + 'certificate': 'CERTIFICATE DATA', + 'key': 'KEY DATA' + }, + 'some_url': ( + 'http://admin:my-secret-password@service-name:8080/v1') + } + } + } + self._test_secret_substitution( + document_mapping, [certificate, certificate_key, passphrase], + expected_data) diff --git a/deckhand/tests/unit/fake_policy.py b/deckhand/tests/unit/fake_policy.py new file mode 100644 index 00000000..771fd770 --- /dev/null +++ b/deckhand/tests/unit/fake_policy.py @@ -0,0 +1,31 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +policy_data = """ +"admin_api": "role:admin" +"deckhand:create_cleartext_documents": "rule:admin_api" +"deckhand:create_encrypted_documents": "rule:admin_api" +"deckhand:list_cleartext_documents": "rule:admin_api" +"deckhand:list_encrypted_documents": "rule:admin_api" +"deckhand:show_revision": "rule:admin_api" +"deckhand:list_revisions": "rule:admin_api" +"deckhand:delete_revisions": "rule:admin_api" +"deckhand:show_revision_diff": "rule:admin_api" +"deckhand:create_tag": "rule:admin_api" +"deckhand:show_tag": "rule:admin_api" +"deckhand:list_tags": "rule:admin_api" +"deckhand:delete_tag": "rule:admin_api" +"deckhand:delete_tags": "rule:admin_api" +""" diff --git a/deckhand/tests/unit/policy_fixture.py b/deckhand/tests/unit/policy_fixture.py new file mode 100644 index 00000000..8610b5d6 --- /dev/null +++ b/deckhand/tests/unit/policy_fixture.py @@ -0,0 +1,74 @@ +# Copyright 2012 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import yaml + +import fixtures +from oslo_config import cfg +from oslo_policy import opts as policy_opts +from oslo_policy import policy as oslo_policy + +from deckhand import policies +import deckhand.policy +from deckhand.tests.unit import fake_policy + + +CONF = cfg.CONF + + +class RealPolicyFixture(fixtures.Fixture): + """Load the live policy for tests. + + A base policy fixture that starts with the assumption that you'd + like to load and enforce the shipped default policy in tests. + + """ + + def setUp(self): + super(RealPolicyFixture, self).setUp() + self.policy_dir = self.useFixture(fixtures.TempDir()) + self.policy_file = os.path.join(self.policy_dir.path, + 'policy.yaml') + # Load the fake_policy data and add the missing default rules. + policy_rules = yaml.safe_load(fake_policy.policy_data) + self.add_missing_default_rules(policy_rules) + with open(self.policy_file, 'w') as f: + yaml.safe_dump(policy_rules, f) + + policy_opts.set_defaults(CONF) + CONF.set_override('policy_dirs', [], group='oslo_policy') + CONF.set_override('policy_file', self.policy_file, group='oslo_policy') + + deckhand.policy.reset() + deckhand.policy.init() + self.addCleanup(deckhand.policy.reset) + + def add_missing_default_rules(self, rules): + """Adds default rules and their values to the given rules dict. + + The given rulen dict may have an incomplete set of policy rules. + This method will add the default policy rules and their values to + the dict. It will not override the existing rules. + """ + for rule in policies.list_rules(): + if rule.name not in rules: + rules[rule.name] = rule.check_str + + def set_rules(self, rules, overwrite=True): + if isinstance(rules, dict): + rules = oslo_policy.Rules.from_dict(rules) + + policy = deckhand.policy._ENFORCER + policy.set_rules(rules, overwrite=overwrite) diff --git a/deckhand/tests/unit/test_policy.py b/deckhand/tests/unit/test_policy.py index 54246e83..e5b71309 100644 --- a/deckhand/tests/unit/test_policy.py +++ b/deckhand/tests/unit/test_policy.py @@ -14,12 +14,10 @@ import falcon import mock from oslo_policy import policy as common_policy -from deckhand.conf import config from deckhand.control import base as api_base -from deckhand import policy +import deckhand.policy from deckhand.tests.unit import base as test_base - -CONF = config.CONF +from deckhand.tests.unit import policy_fixture class PolicyBaseTestCase(test_base.DeckhandTestCase): @@ -33,18 +31,18 @@ class PolicyBaseTestCase(test_base.DeckhandTestCase): "deckhand:create_cleartext_documents": [['@']], "deckhand:list_cleartext_documents": [['rule:admin_api']] } - self.policy_enforcer = common_policy.Enforcer(CONF) + + self.policy = self.useFixture(policy_fixture.RealPolicyFixture()) self._set_rules() def _set_rules(self): - rules = common_policy.Rules.from_dict(self.rules) - self.policy_enforcer.set_rules(rules) - self.addCleanup(self.policy_enforcer.clear) + these_rules = common_policy.Rules.from_dict(self.rules) + deckhand.policy._ENFORCER.set_rules(these_rules) def _enforce_policy(self, action): api_args = self._get_args() - @policy.authorize(action) + @deckhand.policy.authorize(action) def noop(*args, **kwargs): pass @@ -53,8 +51,7 @@ class PolicyBaseTestCase(test_base.DeckhandTestCase): def _get_args(self): # Returns the first two arguments that would be passed to any falcon # on_{HTTP_VERB} method: (self (which is mocked), falcon Request obj). - falcon_req = api_base.DeckhandRequest( - mock.MagicMock(), policy_enforcer=self.policy_enforcer) + falcon_req = api_base.DeckhandRequest(mock.MagicMock()) return (mock.Mock(), falcon_req) diff --git a/deckhand/utils.py b/deckhand/utils.py index aa9165ad..76421bf5 100644 --- a/deckhand/utils.py +++ b/deckhand/utils.py @@ -17,6 +17,8 @@ import string import jsonpath_ng +from deckhand import errors + def to_camel_case(s): """Convert string to camel case.""" @@ -30,30 +32,114 @@ def to_snake_case(name): return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower() -def jsonpath_parse(document, jsonpath): - """Parse value given JSON path in the document. +def jsonpath_parse(data, jsonpath): + """Parse value in the data for the given ``jsonpath``. - Retrieve the value corresponding to document[jsonpath] where ``jsonpath`` - is a multi-part key. A multi-key is a series of keys and nested keys - concatenated together with ".". For exampple, ``jsonpath`` of - ".foo.bar.baz" should mean that ``document`` has the format: + Retrieve the nested entry corresponding to ``data[jsonpath]``. For + example, a ``jsonpath`` of ".foo.bar.baz" means that the data section + should conform to: .. code-block:: yaml - --- - foo: - bar: - baz: + --- + foo: + bar: + baz: - :param document: Dictionary used for extracting nested entry. - :param jsonpath: A multi-part key that references nested data in a - dictionary. - :returns: Nested entry in ``document`` if present, else None. + :param data: The `data` section of a document. + :param jsonpath: A multi-part key that references a nested path in + ``data``. + :returns: Entry that corresponds to ``data[jsonpath]`` if present, + else None. + + Example:: + + src_name = sub['src']['name'] + src_path = sub['src']['path'] + src_doc = db_api.document_get(schema=src_schema, name=src_name) + src_secret = utils.jsonpath_parse(src_doc['data'], src_path) + # Do something with the extracted secret from the source document. """ if jsonpath.startswith('.'): jsonpath = '$' + jsonpath p = jsonpath_ng.parse(jsonpath) - matches = p.find(document) + matches = p.find(data) if matches: return matches[0].value + + +def jsonpath_replace(data, value, jsonpath, pattern=None): + """Update value in ``data`` at the path specified by ``jsonpath``. + + If the nested path corresponding to ``jsonpath`` isn't found in ``data``, + the path is created as an empty ``{}`` for each sub-path along the + ``jsonpath``. + + :param data: The `data` section of a document. + :param value: The new value for ``data[jsonpath]``. + :param jsonpath: A multi-part key that references a nested path in + ``data``. + :param pattern: A regular expression pattern. + :returns: Updated value at ``data[jsonpath]``. + :raises: MissingDocumentPattern if ``pattern`` is not None and + ``data[jsonpath]`` doesn't exist. + + Example:: + + doc = { + 'data': { + 'some_url': http://admin:INSERT_PASSWORD_HERE@svc-name:8080/v1 + } + } + secret = 'super-duper-secret' + path = '$.some_url' + pattern = 'INSERT_[A-Z]+_HERE' + replaced_data = utils.jsonpath_replace( + doc['data'], secret, path, pattern) + # The returned URL will look like: + # http://admin:super-duper-secret@svc-name:8080/v1 + doc['data'].update(replaced_data) + """ + data = data.copy() + if jsonpath.startswith('.'): + jsonpath = '$' + jsonpath + + def _do_replace(): + p = jsonpath_ng.parse(jsonpath) + p_to_change = p.find(data) + + if p_to_change: + _value = value + if pattern: + to_replace = p_to_change[0].value + # value represents the value to inject into to_replace that + # matches the pattern. + try: + _value = re.sub(pattern, value, to_replace) + except TypeError: + _value = None + return p.update(data, _value) + + result = _do_replace() + if result: + return result + + # A pattern requires us to look up the data located at data[jsonpath] + # and then figure out what re.match(data[jsonpath], pattern) is (in + # pseudocode). But raise an exception in case the path isn't present in the + # data and a pattern has been provided since it is impossible to do the + # look up. + if pattern: + raise errors.MissingDocumentPattern( + data=data, path=jsonpath, pattern=pattern) + + # However, Deckhand should be smart enough to create the nested keys in the + # data if they don't exist and a pattern isn't required. + d = data + for path in jsonpath.split('.')[1:]: + if path not in d: + d.setdefault(path, {}) + d = d.get(path) + + return _do_replace() diff --git a/doc/design.md b/doc/design.md index 0bc371b2..9648f25a 100644 --- a/doc/design.md +++ b/doc/design.md @@ -290,7 +290,7 @@ layer example above, which includes `global`, `region` and `site` layers, a document in the `region` layer could insert data from a document in the `site` layer. -Here is a sample set of documents demonstrating subistution: +Here is a sample set of documents demonstrating substitution: ```yaml --- diff --git a/etc/deckhand/policy.yaml.sample b/etc/deckhand/policy.yaml.sample index 28de03d7..d4f3cc90 100644 --- a/etc/deckhand/policy.yaml.sample +++ b/etc/deckhand/policy.yaml.sample @@ -7,10 +7,6 @@ # revision history, whereby the target revision's documents are re- # created for # the new revision. -# -# Conditionally enforced for the endpoints below if the any of the -# documents in -# the request body have a `metadata.storagePolicy` of "cleartext". # PUT /api/v1.0/bucket/{bucket_name}/documents # POST /api/v1.0/rollback/{target_revision_id} #"deckhand:create_cleartext_documents": "rule:admin_api" @@ -22,9 +18,11 @@ # the new # revision. # +# Only enforced after ``create_cleartext_documents`` passes. +# # Conditionally enforced for the endpoints below if the any of the # documents in -# the request body have a `metadata.storagePolicy` of "encrypted". +# the request body have a ``metadata.storagePolicy`` of "encrypted". # PUT /api/v1.0/bucket/{bucket_name}/documents # POST /api/v1.0/rollback/{target_revision_id} #"deckhand:create_encrypted_documents": "rule:admin_api" @@ -33,31 +31,28 @@ # substitution applied) as well as fully layered and substituted # concrete # documents. -# -# Conditionally enforced for the endpoints below if the any of the -# documents in -# the request body have a `metadata.storagePolicy` of "cleartext". If -# policy -# enforcement fails, cleartext documents are omitted. # GET api/v1.0/revisions/{revision_id}/documents # GET api/v1.0/revisions/{revision_id}/rendered-documents #"deckhand:list_cleartext_documents": "rule:admin_api" -# List cleartext documents for a revision (with no layering or +# List encrypted documents for a revision (with no layering or # substitution applied) as well as fully layered and substituted # concrete # documents. # -# Conditionally enforced for the endpoints below if the any of the -# documents in -# the request body have a `metadata.storagePolicy` of "encrypted". If +# Only enforced after ``list_cleartext_documents`` passes. +# +# Conditionally enforced for the endpoints below if any of the +# documents in the +# request body have a ``metadata.storagePolicy`` of "encrypted". If # policy -# enforcement fails, encrypted documents are omitted. +# enforcement fails, encrypted documents are exluded from the +# response. # GET api/v1.0/revisions/{revision_id}/documents # GET api/v1.0/revisions/{revision_id}/rendered-documents #"deckhand:list_encrypted_documents": "rule:admin_api" -# Show details for a revision tag. +# Show details for a revision. # GET /api/v1.0/revisions/{revision_id} #"deckhand:show_revision": "rule:admin_api" diff --git a/releasenotes/notes/secret-substitution-6eff2c93bf11d82e.yaml b/releasenotes/notes/secret-substitution-6eff2c93bf11d82e.yaml new file mode 100644 index 00000000..d26dc589 --- /dev/null +++ b/releasenotes/notes/secret-substitution-6eff2c93bf11d82e.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + Deckhand now supports secret substitution for documents. The endpoint + ``GET revisions/{revision_id}/rendered-documents`` has been added to + Deckhand, which allows the possibility of listing fully substituted + documents. Only documents with ``metadata.substitutions`` field undergo + secret substitution dynamically.