diff --git a/AUTHORS b/AUTHORS index c07cb36e..b8399261 100644 --- a/AUTHORS +++ b/AUTHORS @@ -1,7 +1,9 @@ Alan Meadows Anthony Lin +Bryan Strassner Felipe Monteiro Felipe Monteiro Mark Burnett Pete Birley Scott Hussey +Tin Lam diff --git a/deckhand/context.py b/deckhand/context.py index b4206e15..aa0d484c 100644 --- a/deckhand/context.py +++ b/deckhand/context.py @@ -14,9 +14,6 @@ from oslo_config import cfg from oslo_context import context -from oslo_policy import policy as common_policy - -from deckhand import policy CONF = cfg.CONF @@ -28,12 +25,10 @@ class RequestContext(context.RequestContext): accesses the system, as well as additional request information. """ - def __init__(self, policy_enforcer=None, project=None, **kwargs): + def __init__(self, project=None, **kwargs): if project: kwargs['tenant'] = project self.project = project - self.policy_enforcer = policy_enforcer or common_policy.Enforcer(CONF) - policy.register_rules(self.policy_enforcer) super(RequestContext, self).__init__(**kwargs) def to_dict(self): diff --git a/deckhand/control/api.py b/deckhand/control/api.py index fe6adbbc..23b064ea 100644 --- a/deckhand/control/api.py +++ b/deckhand/control/api.py @@ -67,6 +67,8 @@ def start_api(): revision_diffing.RevisionDiffingResource()), ('revisions/{revision_id}/documents', revision_documents.RevisionDocumentsResource()), + ('revisions/{revision_id}/rendered-documents', + revision_documents.RenderedDocumentsResource()), ('revisions/{revision_id}/tags', revision_tags.RevisionTagsResource()), ('revisions/{revision_id}/tags/{tag}', revision_tags.RevisionTagsResource()), diff --git a/deckhand/control/base.py b/deckhand/control/base.py index ff7ad319..b44d7c9f 100644 --- a/deckhand/control/base.py +++ b/deckhand/control/base.py @@ -51,10 +51,9 @@ class BaseResource(object): class DeckhandRequest(falcon.Request): - def __init__(self, env, options=None, policy_enforcer=None): + def __init__(self, env, options=None): super(DeckhandRequest, self).__init__(env, options) - self.context = context.RequestContext.from_environ( - self.env, policy_enforcer=policy_enforcer) + self.context = context.RequestContext.from_environ(self.env) @property def project_id(self): diff --git a/deckhand/control/buckets.py b/deckhand/control/buckets.py index f0575d55..2e1dee2a 100644 --- a/deckhand/control/buckets.py +++ b/deckhand/control/buckets.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import itertools import yaml import falcon @@ -37,6 +36,7 @@ class BucketsResource(api_base.BaseResource): view_builder = document_view.ViewBuilder() secrets_mgr = secrets_manager.SecretsManager() + @policy.authorize('deckhand:create_cleartext_documents') def on_put(self, req, resp, bucket_name=None): document_data = req.stream.read(req.content_length or 0) try: @@ -47,10 +47,34 @@ class BucketsResource(api_base.BaseResource): LOG.error(error_msg) raise falcon.HTTPBadRequest(description=six.text_type(e)) + # NOTE: Must validate documents before doing policy enforcement, + # because we expect certain formatting of the documents while doing + # policy enforcement. + validation_policies = self._create_validation_policies(documents) + + for document in documents: + if document['metadata'].get('storagePolicy') == 'encrypted': + policy.conditional_authorize( + 'deckhand:create_encrypted_documents', req.context) + break + + self._prepare_secret_documents(documents) + + # Save all the documents, including validation policies. + documents_to_create = documents + validation_policies + created_documents = self._create_revision_documents( + bucket_name, list(documents_to_create)) + + if created_documents: + resp.body = self.to_yaml_body( + self.view_builder.list(created_documents)) + resp.status = falcon.HTTP_200 + resp.append_header('Content-Type', 'application/x-yaml') + + def _create_validation_policies(self, documents): + # All concrete documents in the payload must successfully pass their + # JSON schema validations. Otherwise raise an error. try: - # NOTE: Must validate documents before doing policy enforcement, - # because we expect certain formatting of the documents while doing - # policy enforcement. validation_policies = document_validation.DocumentValidation( documents).validate_all() except deckhand_errors.InvalidDocumentFormat as e: @@ -58,42 +82,25 @@ class BucketsResource(api_base.BaseResource): # validation policy in the DB for future debugging, and only # afterward raise an exception. raise falcon.HTTPBadRequest(description=e.format_message()) + return validation_policies - cleartext_documents = [] - secret_documents = [] - - for document in documents: - if any([document['schema'].startswith(t) - for t in types.DOCUMENT_SECRET_TYPES]): - secret_documents.append(document) - else: - cleartext_documents.append(document) - - if secret_documents and any( - [d['metadata'].get('storagePolicy') == 'encrypted' - for d in secret_documents]): - policy.conditional_authorize('deckhand:create_encrypted_documents', - req.context) - if cleartext_documents: - policy.conditional_authorize('deckhand:create_cleartext_documents', - req.context) - + def _prepare_secret_documents(self, secret_documents): + # Encrypt data for secret documents, if any. for document in secret_documents: - secret_data = self.secrets_mgr.create(document) - document['data'] = secret_data + # TODO(fmontei): Move all of this to document validation directly. + if document['metadata'].get('storagePolicy') == 'encrypted': + secret_data = self.secrets_mgr.create(document) + document['data'] = secret_data + elif any([document['schema'].startswith(t) + for t in types.DOCUMENT_SECRET_TYPES]): + document['data'] = {'secret': document['data']} + def _create_revision_documents(self, bucket_name, documents): try: - documents_to_create = itertools.chain( - cleartext_documents, secret_documents, validation_policies) - created_documents = db_api.documents_create( - bucket_name, list(documents_to_create)) + created_documents = db_api.documents_create(bucket_name, documents) except deckhand_errors.DocumentExists as e: raise falcon.HTTPConflict(description=e.format_message()) except Exception as e: raise falcon.HTTPInternalServerError(description=six.text_type(e)) - if created_documents: - resp.body = self.to_yaml_body( - self.view_builder.list(created_documents)) - resp.status = falcon.HTTP_200 - resp.append_header('Content-Type', 'application/x-yaml') + return created_documents diff --git a/deckhand/control/revision_documents.py b/deckhand/control/revision_documents.py index a0a8cb00..9262c473 100644 --- a/deckhand/control/revision_documents.py +++ b/deckhand/control/revision_documents.py @@ -19,6 +19,7 @@ from deckhand.control import base as api_base from deckhand.control import common from deckhand.control.views import document as document_view from deckhand.db.sqlalchemy import api as db_api +from deckhand.engine import secrets_manager from deckhand import errors from deckhand import policy @@ -26,10 +27,11 @@ LOG = logging.getLogger(__name__) class RevisionDocumentsResource(api_base.BaseResource): - """API resource for realizing CRUD endpoints for revision documents.""" + """API resource for realizing revision documents endpoint.""" view_builder = document_view.ViewBuilder() + @policy.authorize('deckhand:list_cleartext_documents') @common.sanitize_params([ 'schema', 'metadata.name', 'metadata.layeringDefinition.abstract', 'metadata.layeringDefinition.layer', 'metadata.label', @@ -42,18 +44,13 @@ class RevisionDocumentsResource(api_base.BaseResource): documents will be as originally posted with no substitutions or layering applied. """ - include_cleartext = policy.conditional_authorize( - 'deckhand:list_cleartext_documents', req.context, do_raise=False) include_encrypted = policy.conditional_authorize( 'deckhand:list_encrypted_documents', req.context, do_raise=False) filters = sanitized_params.copy() - filters['metadata.storagePolicy'] = [] - if include_cleartext: - filters['metadata.storagePolicy'].append('cleartext') + filters['metadata.storagePolicy'] = ['cleartext'] if include_encrypted: filters['metadata.storagePolicy'].append('encrypted') - # Never return deleted documents to user. filters['deleted'] = False @@ -66,3 +63,51 @@ class RevisionDocumentsResource(api_base.BaseResource): resp.status = falcon.HTTP_200 resp.append_header('Content-Type', 'application/x-yaml') resp.body = self.to_yaml_body(self.view_builder.list(documents)) + + +class RenderedDocumentsResource(api_base.BaseResource): + """API resource for realizing rendered documents endpoint. + + Rendered documents are also revision documents, but unlike revision + documents, they are finalized documents, having undergone secret + substitution and document layering. + + Returns a multi-document YAML response containing all the documents + matching the filters specified via query string parameters. Returned + documents will have secrets substituted into them and be layered with + other documents in the revision, in accordance with the ``LayeringPolicy`` + that currently exists in the system. + """ + + view_builder = document_view.ViewBuilder() + + @policy.authorize('deckhand:list_cleartext_documents') + @common.sanitize_params([ + 'schema', 'metadata.name', 'metadata.label']) + def on_get(self, req, resp, sanitized_params, revision_id): + include_encrypted = policy.conditional_authorize( + 'deckhand:list_encrypted_documents', req.context, do_raise=False) + + filters = sanitized_params.copy() + filters['metadata.storagePolicy'] = ['cleartext'] + if include_encrypted: + filters['metadata.storagePolicy'].append('encrypted') + + try: + documents = db_api.revision_get_documents( + revision_id, **filters) + except (errors.RevisionNotFound) as e: + raise falcon.HTTPNotFound(description=e.format_message()) + + # TODO(fmontei): Currently the only phase of rendering that is + # performed is secret substitution, which can be done in any randomized + # order. However, secret substitution logic will have to be moved into + # a separate module that handles layering alongside substitution once + # layering has been fully integrated into this endpoint. + secrets_substitution = secrets_manager.SecretsSubstitution(documents) + rendered_documents = secrets_substitution.substitute_all() + + resp.status = falcon.HTTP_200 + resp.append_header('Content-Type', 'application/x-yaml') + resp.body = self.to_yaml_body( + self.view_builder.list(rendered_documents)) diff --git a/deckhand/control/rollback.py b/deckhand/control/rollback.py index 96f1c7b6..77169fb8 100644 --- a/deckhand/control/rollback.py +++ b/deckhand/control/rollback.py @@ -34,12 +34,10 @@ class RollbackResource(api_base.BaseResource): raise falcon.HTTPNotFound(description=e.format_message()) for document in latest_revision['documents']: - if document['metadata'].get('storagePolicy') == 'cleartext': - policy.conditional_authorize( - 'deckhand:create_cleartext_documents', req.context) - elif document['metadata'].get('storagePolicy') == 'encrypted': + if document['metadata'].get('storagePolicy') == 'encrypted': policy.conditional_authorize( 'deckhand:create_encrypted_documents', req.context) + break try: rollback_revision = db_api.revision_rollback( diff --git a/deckhand/control/views/document.py b/deckhand/control/views/document.py index ec126e05..ed9d47d7 100644 --- a/deckhand/control/views/document.py +++ b/deckhand/control/views/document.py @@ -13,6 +13,7 @@ # limitations under the License. from deckhand.control import common +from deckhand import types class ViewBuilder(common.ViewBuilder): @@ -30,34 +31,30 @@ class ViewBuilder(common.ViewBuilder): _collection_name = 'documents' def list(self, documents): - # Edge case for when all documents are deleted from a bucket. Still - # need to return bucket_id and revision_id. - if len(documents) == 1 and documents[0]['deleted']: - resp_obj = {'status': {}} - resp_obj['status']['bucket'] = documents[0]['bucket_name'] - resp_obj['status']['revision'] = documents[0]['revision_id'] - return [resp_obj] - resp_list = [] attrs = ['id', 'metadata', 'data', 'schema'] for document in documents: + if document['deleted']: + continue + if document['schema'].startswith(types.VALIDATION_POLICY_SCHEMA): + continue resp_obj = {x: document[x] for x in attrs} resp_obj.setdefault('status', {}) resp_obj['status']['bucket'] = document['bucket_name'] resp_obj['status']['revision'] = document['revision_id'] resp_list.append(resp_obj) - # In the case where no documents are passed to PUT - # buckets/{{bucket_name}}/documents, we need to mangle the response - # body a bit. The revision_id and buckete_id should be returned, as - # at the very least the revision_id will be needed by the user. + # Edge case for when all documents are deleted from a bucket. To detect + # the edge case, check whether ``resp_list`` is empty and whether there + # are still documents to be returned. This means that all the documents + # are either deleted or validation policies. Either way, we still need + # to return bucket_id and revision_id, which should be the same + # across all the documents in ``documents``. if not resp_list and documents: - resp_obj = {} - resp_obj.setdefault('status', {}) - resp_obj['status']['bucket'] = documents[0]['bucket_id'] + resp_obj = {'status': {}} + resp_obj['status']['bucket'] = documents[0]['bucket_name'] resp_obj['status']['revision'] = documents[0]['revision_id'] - - resp_list.append(resp_obj) + return [resp_obj] return resp_list diff --git a/deckhand/control/views/revision.py b/deckhand/control/views/revision.py index da15f554..569309df 100644 --- a/deckhand/control/views/revision.py +++ b/deckhand/control/views/revision.py @@ -59,7 +59,7 @@ class ViewBuilder(common.ViewBuilder): success_status = 'success' for vp in [d for d in revision['documents'] - if d['schema'] == types.VALIDATION_POLICY_SCHEMA]: + if d['schema'].startswith(types.VALIDATION_POLICY_SCHEMA)]: validation_policy = {} validation_policy['name'] = vp.get('name') validation_policy['url'] = self._gen_url(vp) diff --git a/deckhand/db/sqlalchemy/api.py b/deckhand/db/sqlalchemy/api.py index c08a1dca..c346dd12 100644 --- a/deckhand/db/sqlalchemy/api.py +++ b/deckhand/db/sqlalchemy/api.py @@ -261,18 +261,29 @@ def document_get(session=None, raw_dict=False, **filters): """ session = session or get_session() - # Retrieve the most recently created version of a document. Documents with - # the same metadata.name and schema can exist across different revisions, - # so it is necessary to use `first` instead of `one` to avoid errors. - document = session.query(models.Document)\ + # TODO(fmontei): Currently Deckhand doesn't support filtering by nested + # JSON fields via sqlalchemy. For now, filter the documents using all + # "regular" filters via sqlalchemy and all nested filters via Python. + nested_filters = {} + for f in filters.copy(): + if '.' in f: + nested_filters.setdefault(f, filters.pop(f)) + + # Documents with the the same metadata.name and schema can exist across + # different revisions, so it is necessary to order documents by creation + # date, then return the first document that matches all desired filters. + documents = session.query(models.Document)\ .filter_by(**filters)\ .order_by(models.Document.created_at.desc())\ - .first() + .all() - if not document: - raise errors.DocumentNotFound(document=filters) + for doc in documents: + d = doc.to_dict(raw_dict=raw_dict) + if _apply_filters(d, **nested_filters): + return d - return document.to_dict(raw_dict=raw_dict) + filters.update(nested_filters) + raise errors.DocumentNotFound(document=filters) #################### diff --git a/deckhand/engine/document.py b/deckhand/engine/document.py index af6587bc..d04f7783 100644 --- a/deckhand/engine/document.py +++ b/deckhand/engine/document.py @@ -72,6 +72,9 @@ class Document(object): def get_labels(self): return self._inner['metadata']['labels'] + def get_substitutions(self): + return self._inner['metadata'].get('substitutions', None) + def get_actions(self): try: return self._inner['metadata']['layeringDefinition']['actions'] @@ -118,4 +121,7 @@ class Document(object): return not self.__contains__(k) def __repr__(self): - return repr(self._inner) + return '(%s, %s)' % (self.get_schema(), self.get_name()) + + def __str__(self): + return str(self._inner) diff --git a/deckhand/engine/schema/v1_0/certificate_key_schema.py b/deckhand/engine/schema/v1_0/certificate_key_schema.py index 92c08fb8..93abd0a1 100644 --- a/deckhand/engine/schema/v1_0/certificate_key_schema.py +++ b/deckhand/engine/schema/v1_0/certificate_key_schema.py @@ -27,6 +27,13 @@ schema = { 'pattern': '^(metadata/Document/v[1]{1}(\.[0]{1}){0,1})$', }, 'name': {'type': 'string'}, + # Not strictly needed for secrets. + 'layeringDefinition': { + 'type': 'object', + 'properties': { + 'layer': {'type': 'string'} + } + }, 'storagePolicy': { 'type': 'string', 'enum': ['encrypted', 'cleartext'] diff --git a/deckhand/engine/schema/v1_0/certificate_schema.py b/deckhand/engine/schema/v1_0/certificate_schema.py index 9335b6db..25f63a66 100644 --- a/deckhand/engine/schema/v1_0/certificate_schema.py +++ b/deckhand/engine/schema/v1_0/certificate_schema.py @@ -27,6 +27,13 @@ schema = { 'pattern': '^(metadata/Document/v[1]{1}(\.[0]{1}){0,1})$', }, 'name': {'type': 'string'}, + # Not strictly needed for secrets. + 'layeringDefinition': { + 'type': 'object', + 'properties': { + 'layer': {'type': 'string'} + } + }, 'storagePolicy': { 'type': 'string', 'enum': ['encrypted', 'cleartext'] diff --git a/deckhand/engine/schema/v1_0/data_schema.py b/deckhand/engine/schema/v1_0/data_schema.py index 346b2d02..8185763a 100644 --- a/deckhand/engine/schema/v1_0/data_schema.py +++ b/deckhand/engine/schema/v1_0/data_schema.py @@ -33,6 +33,10 @@ schema = { # Labels are optional. 'labels': { 'type': 'object' + }, + 'storagePolicy': { + 'type': 'string', + 'enum': ['encrypted', 'cleartext'] } }, 'additionalProperties': False, diff --git a/deckhand/engine/schema/v1_0/document_schema.py b/deckhand/engine/schema/v1_0/document_schema.py index f903e2b2..eaf16d4e 100644 --- a/deckhand/engine/schema/v1_0/document_schema.py +++ b/deckhand/engine/schema/v1_0/document_schema.py @@ -84,6 +84,10 @@ schema = { 'substitutions': { 'type': 'array', 'items': substitution_schema + }, + 'storagePolicy': { + 'type': 'string', + 'enum': ['encrypted', 'cleartext'] } }, 'additionalProperties': False, diff --git a/deckhand/engine/schema/v1_0/layering_schema.py b/deckhand/engine/schema/v1_0/layering_schema.py index f03c4443..ace8e863 100644 --- a/deckhand/engine/schema/v1_0/layering_schema.py +++ b/deckhand/engine/schema/v1_0/layering_schema.py @@ -26,7 +26,11 @@ schema = { 'type': 'string', 'pattern': '^(metadata/Control/v[1]{1}(\.[0]{1}){0,1})$' }, - 'name': {'type': 'string'} + 'name': {'type': 'string'}, + 'storagePolicy': { + 'type': 'string', + 'enum': ['encrypted', 'cleartext'] + } }, 'additionalProperties': False, 'required': ['schema', 'name'] diff --git a/deckhand/engine/schema/v1_0/passphrase_schema.py b/deckhand/engine/schema/v1_0/passphrase_schema.py index 679a79e5..e9e7b211 100644 --- a/deckhand/engine/schema/v1_0/passphrase_schema.py +++ b/deckhand/engine/schema/v1_0/passphrase_schema.py @@ -27,6 +27,13 @@ schema = { 'pattern': '^(metadata/Document/v[1]{1}(\.[0]{1}){0,1})$', }, 'name': {'type': 'string'}, + # Not strictly needed for secrets. + 'layeringDefinition': { + 'type': 'object', + 'properties': { + 'layer': {'type': 'string'} + } + }, 'storagePolicy': { 'type': 'string', 'enum': ['encrypted', 'cleartext'] diff --git a/deckhand/engine/schema/v1_0/validation_schema.py b/deckhand/engine/schema/v1_0/validation_schema.py index 9681265d..8416bc2a 100644 --- a/deckhand/engine/schema/v1_0/validation_schema.py +++ b/deckhand/engine/schema/v1_0/validation_schema.py @@ -26,7 +26,11 @@ schema = { 'type': 'string', 'pattern': '^(metadata/Control/v[1]{1}(\.[0]{1}){0,1})$' }, - 'name': {'type': 'string'} + 'name': {'type': 'string'}, + 'storagePolicy': { + 'type': 'string', + 'enum': ['encrypted', 'cleartext'] + } }, 'additionalProperties': False, 'required': ['schema', 'name'] diff --git a/deckhand/engine/secrets_manager.py b/deckhand/engine/secrets_manager.py index a4179fce..41f0f80f 100644 --- a/deckhand/engine/secrets_manager.py +++ b/deckhand/engine/secrets_manager.py @@ -12,7 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +from oslo_log import log as logging + from deckhand.barbican import driver +from deckhand.db.sqlalchemy import api as db_api +from deckhand.engine import document as document_wrapper +from deckhand import utils + +LOG = logging.getLogger(__name__) CLEARTEXT = 'cleartext' ENCRYPTED = 'encrypted' @@ -34,22 +41,22 @@ class SecretsManager(object): documents with the schemas enumerated below) must be stored using a secure storage service like Barbican. - Documents with metadata.storagePolicy == "clearText" have their secrets - stored directly in Deckhand. + Documents with ``metadata.storagePolicy`` == "clearText" have their + secrets stored directly in Deckhand. - Documents with metadata.storagePolicy == "encrypted" are stored in + Documents with ``metadata.storagePolicy`` == "encrypted" are stored in Barbican directly. Deckhand in turn stores the reference returned by Barbican in Deckhand. :param secret_doc: A Deckhand document with one of the following schemas: - * deckhand/Certificate/v1 - * deckhand/CertificateKey/v1 - * deckhand/Passphrase/v1 + * ``deckhand/Certificate/v1`` + * ``deckhand/CertificateKey/v1`` + * ``deckhand/Passphrase/v1`` :returns: Dictionary representation of - `deckhand.db.sqlalchemy.models.DocumentSecret`. + ``deckhand.db.sqlalchemy.models.DocumentSecret``. """ encryption_type = secret_doc['metadata']['storagePolicy'] secret_type = self._get_secret_type(secret_doc['schema']) @@ -73,9 +80,9 @@ class SecretsManager(object): def _get_secret_type(self, schema): """Get the Barbican secret type based on the following mapping: - deckhand/Certificate/v1 => certificate - deckhand/CertificateKey/v1 => private - deckhand/Passphrase/v1 => passphrase + ``deckhand/Certificate/v1`` => certificate + ``deckhand/CertificateKey/v1`` => private + ``deckhand/Passphrase/v1`` => passphrase :param schema: The document's schema. :returns: The value corresponding to the mapping above. @@ -84,3 +91,65 @@ class SecretsManager(object): if _schema == 'certificatekey': return 'private' return _schema + + +class SecretsSubstitution(object): + """Class for document substitution logic for YAML files.""" + + def __init__(self, documents): + """SecretSubstitution constructor. + + :param documents: List of YAML documents in dictionary format that are + candidates for secret substitution. This class will automatically + detect documents that require substitution; documents need not be + filtered prior to being passed to the constructor. + """ + if not isinstance(documents, (list, tuple)): + documents = [documents] + substitute_docs = [document_wrapper.Document(d) for d in documents if + 'substitutions' in d['metadata']] + self.documents = substitute_docs + + def substitute_all(self): + """Substitute all documents that have a `metadata.substitutions` field. + + Concrete (non-abstract) documents can be used as a source of + substitution into other documents. This substitution is + layer-independent, a document in the region layer could insert data + from a document in the site layer. + + :returns: List of fully substituted documents. + """ + LOG.debug('Substituting secrets for documents: %s', self.documents) + substituted_docs = [] + + for doc in self.documents: + LOG.debug( + 'Checking for substitutions in schema=%s, metadata.name=%s', + doc.get_name(), doc.get_schema()) + for sub in doc.get_substitutions(): + src_schema = sub['src']['schema'] + src_name = sub['src']['name'] + src_path = sub['src']['path'] + if src_path == '.': + src_path = '.secret' + + # TODO(fmontei): Use secrets_manager for this logic. Need to + # check Barbican for the secret if it has been encrypted. + src_doc = db_api.document_get( + schema=src_schema, name=src_name, is_secret=True, + **{'metadata.layeringDefinition.abstract': False}) + src_secret = utils.jsonpath_parse(src_doc['data'], src_path) + + dest_path = sub['dest']['path'] + dest_pattern = sub['dest'].get('pattern', None) + + LOG.debug('Substituting from schema=%s name=%s src_path=%s ' + 'into dest_path=%s, dest_pattern=%s', src_schema, + src_name, src_path, dest_path, dest_pattern) + substituted_data = utils.jsonpath_replace( + doc['data'], src_secret, dest_path, dest_pattern) + doc['data'].update(substituted_data) + + substituted_docs.append(doc.to_dict()) + return substituted_docs diff --git a/deckhand/errors.py b/deckhand/errors.py index 45c8a062..30d3b4a3 100644 --- a/deckhand/errors.py +++ b/deckhand/errors.py @@ -23,18 +23,11 @@ class DeckhandException(Exception): code = 500 def __init__(self, message=None, **kwargs): - self.kwargs = kwargs - - if 'code' not in self.kwargs: - try: - self.kwargs['code'] = self.code - except AttributeError: - pass + kwargs.setdefault('code', DeckhandException.code) if not message: try: message = self.msg_fmt % kwargs - except Exception: message = self.msg_fmt @@ -58,15 +51,6 @@ class InvalidDocumentFormat(DeckhandException): super(InvalidDocumentFormat, self).__init__(**kwargs) -# TODO(fmontei): Remove this in a future commit. -class ApiError(Exception): - pass - - -class InvalidFormat(ApiError): - """The YAML file is incorrectly formatted and cannot be read.""" - - class DocumentExists(DeckhandException): msg_fmt = ("Document with schema %(schema)s and metadata.name " "%(name)s already exists in bucket %(bucket)s.") @@ -100,6 +84,12 @@ class MissingDocumentKey(DeckhandException): "Parent: %(parent)s. Child: %(child)s.") +class MissingDocumentPattern(DeckhandException): + msg_fmt = ("Substitution pattern %(pattern)s could not be found for the " + "JSON path %(path)s in the destination document data %(data)s.") + code = 400 + + class UnsupportedActionMethod(DeckhandException): msg_fmt = ("Method in %(actions)s is invalid for document %(document)s.") code = 400 diff --git a/deckhand/factories.py b/deckhand/factories.py index 76a16563..26f70889 100644 --- a/deckhand/factories.py +++ b/deckhand/factories.py @@ -61,7 +61,6 @@ class DocumentFactory(DeckhandFactory): "layeringDefinition": { "abstract": False, "layer": "", - "parentSelector": "", "actions": [] }, "name": "", @@ -92,7 +91,7 @@ class DocumentFactory(DeckhandFactory): ] :param num_layers: Total number of layers. Only supported values - include 2 or 3. + include 1, 2 or 3. :type num_layers: integer :param docs_per_layer: The number of documents to be included per layer. For example, if ``num_layers`` is 3, then ``docs_per_layer`` @@ -105,12 +104,14 @@ class DocumentFactory(DeckhandFactory): compatible with ``docs_per_layer``. """ # Set up the layering definition's layerOrder. - if num_layers == 2: + if num_layers == 1: + layer_order = ["global"] + elif num_layers == 2: layer_order = ["global", "site"] elif num_layers == 3: layer_order = ["global", "region", "site"] else: - raise ValueError("'num_layers' must either be 2 or 3.") + raise ValueError("'num_layers' must be a value between 1 - 3.") self.LAYERING_DEFINITION['data']['layerOrder'] = layer_order if not isinstance(docs_per_layer, (list, tuple)): @@ -225,14 +226,30 @@ class DocumentFactory(DeckhandFactory): data_key = "_%s_DATA_%d_" % (layer_name.upper(), count + 1) actions_key = "_%s_ACTIONS_%d_" % ( layer_name.upper(), count + 1) + sub_key = "_%s_SUBSTITUTIONS_%d_" % ( + layer_name.upper(), count + 1) try: layer_template['data'] = mapping[data_key]['data'] + except KeyError as e: + LOG.debug('Could not map %s because it was not found in ' + 'the `mapping` dict.', e.args[0]) + pass + + try: layer_template['metadata']['layeringDefinition'][ 'actions'] = mapping[actions_key]['actions'] except KeyError as e: - LOG.warning('Could not map %s because it was not found in ' - 'the `mapping` dict.', e.args[0]) + LOG.debug('Could not map %s because it was not found in ' + 'the `mapping` dict.', e.args[0]) + pass + + try: + layer_template['metadata']['substitutions'] = mapping[ + sub_key] + except KeyError as e: + LOG.debug('Could not map %s because it was not found in ' + 'the `mapping` dict.', e.args[0]) pass rendered_template.append(layer_template) diff --git a/deckhand/policies/document.py b/deckhand/policies/document.py index c54d7a3a..7433e950 100644 --- a/deckhand/policies/document.py +++ b/deckhand/policies/document.py @@ -24,10 +24,7 @@ document_policies = [ """Create a batch of documents specified in the request body, whereby a new revision is created. Also, roll back a revision to a previous one in the revision history, whereby the target revision's documents are re-created for -the new revision. - -Conditionally enforced for the endpoints below if the any of the documents in -the request body have a `metadata.storagePolicy` of "cleartext".""", +the new revision.""", [ { 'method': 'PUT', @@ -46,8 +43,10 @@ a new revision is created. Also, roll back a revision to a previous one in the history, whereby the target revision's documents are re-created for the new revision. +Only enforced after ``create_cleartext_documents`` passes. + Conditionally enforced for the endpoints below if the any of the documents in -the request body have a `metadata.storagePolicy` of "encrypted".""", +the request body have a ``metadata.storagePolicy`` of "encrypted".""", [ { 'method': 'PUT', @@ -63,11 +62,7 @@ the request body have a `metadata.storagePolicy` of "encrypted".""", base.RULE_ADMIN_API, """List cleartext documents for a revision (with no layering or substitution applied) as well as fully layered and substituted concrete -documents. - -Conditionally enforced for the endpoints below if the any of the documents in -the request body have a `metadata.storagePolicy` of "cleartext". If policy -enforcement fails, cleartext documents are omitted.""", +documents.""", [ { 'method': 'GET', @@ -81,13 +76,15 @@ enforcement fails, cleartext documents are omitted.""", policy.DocumentedRuleDefault( base.POLICY_ROOT % 'list_encrypted_documents', base.RULE_ADMIN_API, - """List cleartext documents for a revision (with no layering or + """List encrypted documents for a revision (with no layering or substitution applied) as well as fully layered and substituted concrete documents. -Conditionally enforced for the endpoints below if the any of the documents in -the request body have a `metadata.storagePolicy` of "encrypted". If policy -enforcement fails, encrypted documents are omitted.""", +Only enforced after ``list_cleartext_documents`` passes. + +Conditionally enforced for the endpoints below if any of the documents in the +request body have a ``metadata.storagePolicy`` of "encrypted". If policy +enforcement fails, encrypted documents are exluded from the response.""", [ { 'method': 'GET', diff --git a/deckhand/policy.py b/deckhand/policy.py index 9a2cb8ae..bda8b27b 100644 --- a/deckhand/policy.py +++ b/deckhand/policy.py @@ -25,22 +25,54 @@ from deckhand import policies CONF = cfg.CONF LOG = logging.getLogger(__name__) +_ENFORCER = None + + +def reset(): + global _ENFORCER + if _ENFORCER: + _ENFORCER.clear() + _ENFORCER = None + + +def init(policy_file=None, rules=None, default_rule=None, use_conf=True): + """Init an Enforcer class. + + :param policy_file: Custom policy file to use, if none is specified, + ``CONF.policy_file`` will be used. + :param rules: Default dictionary / Rules to use. It will be + considered just in the first instantiation. + :param default_rule: Default rule to use; ``CONF.default_rule`` will + be used if none is specified. + :param use_conf: Whether to load rules from config file. + """ + + global _ENFORCER + + if not _ENFORCER: + _ENFORCER = policy.Enforcer(CONF, + policy_file=policy_file, + rules=rules, + default_rule=default_rule, + use_conf=use_conf) + register_rules(_ENFORCER) def _do_enforce_rbac(action, context, do_raise=True): - policy_enforcer = context.policy_enforcer + init() + credentials = context.to_policy_values() target = {'project_id': context.project_id, 'user_id': context.user_id} exc = errors.PolicyNotAuthorized try: - # oslo.policy supports both enforce and authorize. authorize is + # `oslo.policy` supports both enforce and authorize. authorize is # stricter because it'll raise an exception if the policy action is # not found in the list of registered rules. This means that attempting # to enforce anything not found in ``deckhand.policies`` will error out # with a 'Policy not registered' message. - return policy_enforcer.authorize( + return _ENFORCER.authorize( action, target, context.to_dict(), do_raise=do_raise, exc=exc, action=action) except policy.PolicyNotRegistered as e: diff --git a/deckhand/tests/functional/gabbits/document-render-success-multiple-bucket-with-substitution.yaml b/deckhand/tests/functional/gabbits/document-render-success-multiple-bucket-with-substitution.yaml index 5745e270..6624ee8c 100644 --- a/deckhand/tests/functional/gabbits/document-render-success-multiple-bucket-with-substitution.yaml +++ b/deckhand/tests/functional/gabbits/document-render-success-multiple-bucket-with-substitution.yaml @@ -16,25 +16,22 @@ tests: desc: Begin testing from known state. DELETE: /api/v1.0/revisions status: 204 - skip: Not implemented. - name: add_bucket_a desc: Create documents for bucket a PUT: /api/v1.0/bucket/a/documents status: 200 - data: <@resources/design-doc-substition-sample-split-bucket-a.yaml - skip: Not implemented. + data: <@resources/design-doc-substitution-sample-split-bucket-a.yaml - name: add_bucket_b desc: Create documents for bucket b PUT: /api/v1.0/bucket/b/documents status: 200 - data: <@resources/design-doc-substition-sample-split-bucket-b.yaml - skip: Not implemented. + data: <@resources/design-doc-substitution-sample-split-bucket-b.yaml - name: verify_substitutions desc: Check for expected substitutions - GET: /api/v1.0/revisions/$RESPONSE['$.[0].revision']/rendered-documents?schema=armada/Chart/v1 + GET: /api/v1.0/revisions/$RESPONSE['$.[0].status.revision']/rendered-documents?schema=armada/Chart/v1 status: 200 response_multidoc_jsonpaths: $.[*].metadata.name: example-chart-01 @@ -49,4 +46,3 @@ tests: key: | KEY DATA some_url: http://admin:my-secret-password@service-name:8080/v1 - skip: Not implemented. diff --git a/deckhand/tests/functional/gabbits/document-render-success-single-bucket-with-substitution.yaml b/deckhand/tests/functional/gabbits/document-render-success-single-bucket-with-substitution.yaml index b9ebf190..1a34e472 100644 --- a/deckhand/tests/functional/gabbits/document-render-success-single-bucket-with-substitution.yaml +++ b/deckhand/tests/functional/gabbits/document-render-success-single-bucket-with-substitution.yaml @@ -15,18 +15,16 @@ tests: desc: Begin testing from known state. DELETE: /api/v1.0/revisions status: 204 - skip: Not implemented. - name: initialize desc: Create initial documents PUT: /api/v1.0/bucket/mop/documents status: 200 - data: <@resources/design-doc-substition-sample.yaml - skip: Not implemented. + data: <@resources/design-doc-substitution-sample.yaml - name: verify_substitutions desc: Check for expected substitutions - GET: /api/v1.0/revisions/$RESPONSE['$.[0].revision']/rendered-documents?schema=armada/Chart/v1 + GET: /api/v1.0/revisions/$RESPONSE['$.[0].status.revision']/rendered-documents?schema=armada/Chart/v1 status: 200 response_multidoc_jsonpaths: $.[*].metadata.name: example-chart-01 @@ -41,4 +39,3 @@ tests: key: | KEY DATA some_url: http://admin:my-secret-password@service-name:8080/v1 - skip: Not implemented. diff --git a/deckhand/tests/functional/gabbits/resources/design-doc-substitution-sample-split-bucket-a.yaml b/deckhand/tests/functional/gabbits/resources/design-doc-substitution-sample-split-bucket-a.yaml index 3b4fdf7c..42919292 100644 --- a/deckhand/tests/functional/gabbits/resources/design-doc-substitution-sample-split-bucket-a.yaml +++ b/deckhand/tests/functional/gabbits/resources/design-doc-substitution-sample-split-bucket-a.yaml @@ -2,9 +2,10 @@ schema: deckhand/CertificateKey/v1 metadata: name: example-key - storagePolicy: encrypted + schema: metadata/Document/v1 layeringDefinition: layer: site + storagePolicy: cleartext data: | KEY DATA ... diff --git a/deckhand/tests/functional/gabbits/resources/design-doc-substitution-sample-split-bucket-b.yaml b/deckhand/tests/functional/gabbits/resources/design-doc-substitution-sample-split-bucket-b.yaml index 229751ca..90e8f676 100644 --- a/deckhand/tests/functional/gabbits/resources/design-doc-substitution-sample-split-bucket-b.yaml +++ b/deckhand/tests/functional/gabbits/resources/design-doc-substitution-sample-split-bucket-b.yaml @@ -2,24 +2,26 @@ schema: deckhand/Certificate/v1 metadata: name: example-cert - storagePolicy: cleartext + schema: metadata/Document/v1 layeringDefinition: layer: site + storagePolicy: cleartext data: | CERTIFICATE DATA --- schema: deckhand/Passphrase/v1 metadata: name: example-password - storagePolicy: encrypted + schema: metadata/Document/v1 layeringDefinition: layer: site + storagePolicy: cleartext data: my-secret-password --- schema: armada/Chart/v1 metadata: name: example-chart-01 - storagePolicy: cleartext + schema: metadata/Document/v1 layeringDefinition: layer: region substitutions: diff --git a/deckhand/tests/functional/gabbits/resources/design-doc-substitution-sample.yaml b/deckhand/tests/functional/gabbits/resources/design-doc-substitution-sample.yaml index e8141341..3492d2c8 100644 --- a/deckhand/tests/functional/gabbits/resources/design-doc-substitution-sample.yaml +++ b/deckhand/tests/functional/gabbits/resources/design-doc-substitution-sample.yaml @@ -2,33 +2,36 @@ schema: deckhand/Certificate/v1 metadata: name: example-cert - storagePolicy: cleartext + schema: metadata/Document/v1 layeringDefinition: layer: site + storagePolicy: cleartext data: | CERTIFICATE DATA --- schema: deckhand/CertificateKey/v1 metadata: name: example-key - storagePolicy: encrypted + schema: metadata/Document/v1 layeringDefinition: layer: site + storagePolicy: cleartext data: | KEY DATA --- schema: deckhand/Passphrase/v1 metadata: name: example-password - storagePolicy: encrypted + schema: metadata/Document/v1 layeringDefinition: layer: site + storagePolicy: cleartext data: my-secret-password --- schema: armada/Chart/v1 metadata: name: example-chart-01 - storagePolicy: cleartext + schema: metadata/Document/v1 layeringDefinition: layer: region substitutions: diff --git a/deckhand/tests/functional/gabbits/revision-diff-success.yaml b/deckhand/tests/functional/gabbits/revision-diff-success.yaml index e84a0db6..f29df35f 100644 --- a/deckhand/tests/functional/gabbits/revision-diff-success.yaml +++ b/deckhand/tests/functional/gabbits/revision-diff-success.yaml @@ -196,6 +196,10 @@ tests: PUT: /api/v1.0/bucket/bucket_mistake/documents status: 200 data: "" + # Verification for whether a bucket_name was returned even though all the + # documents for this bucket were deleted. + response_multidoc_jsonpaths: + $.[*].status.bucket: bucket_mistake - name: verify_diff_between_created_and_deleted_mistake desc: Validates response for deletion between the last 2 revisions diff --git a/deckhand/tests/unit/control/base.py b/deckhand/tests/unit/control/base.py index ff98e58f..b4533712 100644 --- a/deckhand/tests/unit/control/base.py +++ b/deckhand/tests/unit/control/base.py @@ -16,6 +16,7 @@ from falcon import testing as falcon_testing from deckhand.control import api from deckhand.tests.unit import base as test_base +from deckhand.tests.unit import policy_fixture class BaseControllerTest(test_base.DeckhandWithDBTestCase, @@ -25,3 +26,4 @@ class BaseControllerTest(test_base.DeckhandWithDBTestCase, def setUp(self): super(BaseControllerTest, self).setUp() self.app = falcon_testing.TestClient(api.start_api()) + self.policy = self.useFixture(policy_fixture.RealPolicyFixture()) diff --git a/deckhand/tests/unit/control/test_api_initialization.py b/deckhand/tests/unit/control/test_api_initialization.py index 96895a89..1192566a 100644 --- a/deckhand/tests/unit/control/test_api_initialization.py +++ b/deckhand/tests/unit/control/test_api_initialization.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import inspect import mock from deckhand.control import api @@ -24,6 +25,7 @@ from deckhand.control import revisions from deckhand.control import rollback from deckhand.control import versions from deckhand.tests.unit import base as test_base +from deckhand import utils class TestApi(test_base.DeckhandTestCase): @@ -32,11 +34,16 @@ class TestApi(test_base.DeckhandTestCase): super(TestApi, self).setUp() for resource in (buckets, revision_diffing, revision_documents, revision_tags, revisions, rollback, versions): - resource_name = resource.__name__.split('.')[-1] - resource_obj = self.patchobject( - resource, '%sResource' % resource_name.title().replace( - '_', ''), autospec=True) - setattr(self, '%s_resource' % resource_name, resource_obj) + class_names = self._get_module_class_names(resource) + for class_name in class_names: + resource_obj = self.patchobject( + resource, class_name, autospec=True) + setattr(self, utils.to_snake_case(class_name), resource_obj) + + def _get_module_class_names(self, module): + class_names = [obj.__name__ for name, obj in inspect.getmembers(module) + if inspect.isclass(obj)] + return class_names @mock.patch.object(api, 'db_api', autospec=True) @mock.patch.object(api, 'logging', autospec=True) @@ -62,6 +69,8 @@ class TestApi(test_base.DeckhandTestCase): self.revision_diffing_resource()), mock.call('/api/v1.0/revisions/{revision_id}/documents', self.revision_documents_resource()), + mock.call('/api/v1.0/revisions/{revision_id}/rendered-documents', + self.rendered_documents_resource()), mock.call('/api/v1.0/revisions/{revision_id}/tags', self.revision_tags_resource()), mock.call('/api/v1.0/revisions/{revision_id}/tags/{tag}', diff --git a/deckhand/tests/unit/control/test_buckets_controller.py b/deckhand/tests/unit/control/test_buckets_controller.py index 2ee9a529..f1d90c36 100644 --- a/deckhand/tests/unit/control/test_buckets_controller.py +++ b/deckhand/tests/unit/control/test_buckets_controller.py @@ -12,17 +12,106 @@ # See the License for the specific language governing permissions and # limitations under the License. +import yaml + +import mock +from oslo_config import cfg + +from deckhand.control import buckets +from deckhand import factories from deckhand.tests.unit.control import base as test_base +CONF = cfg.CONF + class TestBucketsController(test_base.BaseControllerTest): - """Test suite for validating positive scenarios bucket controller.""" + """Test suite for validating positive scenarios for bucket controller.""" + + def test_put_bucket(self): + rules = {'deckhand:create_cleartext_documents': '@'} + self.policy.set_rules(rules) + + documents_factory = factories.DocumentFactory(2, [1, 1]) + document_mapping = { + "_GLOBAL_DATA_1_": {"data": {"a": {"x": 1, "y": 2}}}, + "_SITE_DATA_1_": {"data": {"a": {"x": 7, "z": 3}, "b": 4}}, + "_SITE_ACTIONS_1_": { + "actions": [{"method": "merge", "path": "."}]} + } + payload = documents_factory.gen_test(document_mapping) + + resp = self.app.simulate_put('/api/v1.0/bucket/mop/documents', + body=yaml.safe_dump_all(payload)) + self.assertEqual(200, resp.status_code) + created_documents = list(yaml.safe_load_all(resp.text)) + self.assertEqual(3, len(created_documents)) + expected = sorted([(d['schema'], d['metadata']['name']) + for d in payload]) + actual = sorted([(d['schema'], d['metadata']['name']) + for d in created_documents]) + self.assertEqual(expected, actual) + + def test_put_bucket_with_secret(self): + def _do_test(payload): + resp = self.app.simulate_put('/api/v1.0/bucket/mop/documents', + body=yaml.safe_dump_all(payload)) + self.assertEqual(200, resp.status_code) + created_documents = list(yaml.safe_load_all(resp.text)) + self.assertEqual(1, len(created_documents)) + expected = sorted([(d['schema'], d['metadata']['name']) + for d in payload]) + actual = sorted([(d['schema'], d['metadata']['name']) + for d in created_documents]) + self.assertEqual(expected, actual) + self.assertEqual({'secret': payload[0]['data']}, + created_documents[0]['data']) + + # Verify whether creating a cleartext secret works. + rules = {'deckhand:create_cleartext_documents': '@'} + self.policy.set_rules(rules) + + secrets_factory = factories.DocumentSecretFactory() + payload = [secrets_factory.gen_test('Certificate', 'cleartext')] + _do_test(payload) + + # Verify whether creating an encrypted secret works. + rules = {'deckhand:create_cleartext_documents': '@', + 'deckhand:create_encrypted_documents': '@'} + self.policy.set_rules(rules) + + secrets_factory = factories.DocumentSecretFactory() + payload = [secrets_factory.gen_test('Certificate', 'encrypted')] + + with mock.patch.object(buckets.BucketsResource, 'secrets_mgr', + autospec=True) as mock_secrets_mgr: + mock_secrets_mgr.create.return_value = { + 'secret': payload[0]['data']} + _do_test(payload) + + # Verify whether any document can be encrypted if its + # `metadata.storagePolicy`='encrypted'. In the case below, + # a generic document is tested. + documents_factory = factories.DocumentFactory(1, [1]) + document_mapping = { + "_GLOBAL_DATA_1_": {"data": {"a": {"x": 1, "y": 2}}} + } + payload = documents_factory.gen_test(document_mapping, + global_abstract=False) + payload[-1]['metadata']['storagePolicy'] = 'encrypted' + with mock.patch.object(buckets.BucketsResource, 'secrets_mgr', + autospec=True) as mock_secrets_mgr: + mock_secrets_mgr.create.return_value = { + 'secret': payload[-1]['data']} + _do_test([payload[-1]]) class TestBucketsControllerNegative(test_base.BaseControllerTest): - """Test suite for validating negative scenarios bucket controller.""" + """Test suite for validating negative scenarios for bucket controller.""" def test_put_bucket_with_invalid_document_payload(self): + rules = {'deckhand:create_cleartext_documents': '@'} + self.policy.set_rules(rules) + no_colon_spaces = """ name:foo schema: @@ -38,3 +127,42 @@ schema: body=payload) self.assertEqual(400, resp.status_code) self.assertRegexpMatches(resp.text, error_re[idx]) + + +class TestBucketsControllerNegativeRBAC(test_base.BaseControllerTest): + """Test suite for validating negative RBAC scenarios for bucket + controller. + """ + + def test_put_bucket_cleartext_documents_except_forbidden(self): + rules = {'deckhand:create_cleartext_documents': 'rule:admin_api'} + self.policy.set_rules(rules) + + documents_factory = factories.DocumentFactory(2, [1, 1]) + payload = documents_factory.gen_test({}) + + resp = self.app.simulate_put('/api/v1.0/bucket/mop/documents', + body=yaml.safe_dump_all(payload)) + self.assertEqual(403, resp.status_code) + + def test_put_bucket_cleartext_secret_except_forbidden(self): + rules = {'deckhand:create_cleartext_documents': 'rule:admin_api'} + self.policy.set_rules(rules) + + secrets_factory = factories.DocumentSecretFactory() + payload = [secrets_factory.gen_test('Certificate', 'cleartext')] + + resp = self.app.simulate_put('/api/v1.0/bucket/mop/documents', + body=yaml.safe_dump_all(payload)) + self.assertEqual(403, resp.status_code) + + def test_put_bucket_encrypted_secret_except_forbidden(self): + rules = {'deckhand:create_encrypted_documents': 'rule:admin_api'} + self.policy.set_rules(rules) + + secrets_factory = factories.DocumentSecretFactory() + payload = [secrets_factory.gen_test('Certificate', 'encrypted')] + + resp = self.app.simulate_put('/api/v1.0/bucket/mop/documents', + body=yaml.safe_dump_all(payload)) + self.assertEqual(403, resp.status_code) diff --git a/deckhand/tests/unit/db/base.py b/deckhand/tests/unit/db/base.py index 9d8948da..fd98670c 100644 --- a/deckhand/tests/unit/db/base.py +++ b/deckhand/tests/unit/db/base.py @@ -40,7 +40,8 @@ class DocumentFixture(object): 'layeringDefinition': { 'abstract': test_utils.rand_bool(), 'layer': test_utils.rand_name('layer') - } + }, + 'storagePolicy': test_utils.rand_name('storage_policy') }, 'schema': test_utils.rand_name('schema')} fixture.update(kwargs) diff --git a/deckhand/tests/unit/engine/test_secrets_manager.py b/deckhand/tests/unit/engine/test_secrets_manager.py index 8c5f59a6..a292534e 100644 --- a/deckhand/tests/unit/engine/test_secrets_manager.py +++ b/deckhand/tests/unit/engine/test_secrets_manager.py @@ -12,13 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +import copy + from deckhand.engine import secrets_manager from deckhand import factories from deckhand.tests import test_utils -from deckhand.tests.unit.db import base +from deckhand.tests.unit.db import base as test_base -class TestSecretsManager(base.TestDbBase): +class TestSecretsManager(test_base.TestDbBase): def setUp(self): super(TestSecretsManager, self).setUp() @@ -71,3 +73,215 @@ class TestSecretsManager(base.TestDbBase): def test_create_encrypted_passphrase(self): self._test_create_secret('encrypted', 'Passphrase') + + +class TestSecretsSubstitution(test_base.TestDbBase): + + def setUp(self): + super(TestSecretsSubstitution, self).setUp() + self.document_factory = factories.DocumentFactory(1, [1]) + self.secrets_factory = factories.DocumentSecretFactory() + + def _test_secret_substitution(self, document_mapping, secret_documents, + expected_data): + payload = self.document_factory.gen_test(document_mapping, + global_abstract=False) + bucket_name = test_utils.rand_name('bucket') + documents = self.create_documents( + bucket_name, secret_documents + [payload[-1]]) + + expected_documents = copy.deepcopy([documents[-1]]) + expected_documents[0]['data'] = expected_data + + secret_substitution = secrets_manager.SecretsSubstitution(documents) + substituted_docs = secret_substitution.substitute_all() + + self.assertEqual(expected_documents, substituted_docs) + + def test_secret_substitution_single_cleartext(self): + certificate = self.secrets_factory.gen_test( + 'Certificate', 'cleartext', data={'secret': 'CERTIFICATE DATA'}) + certificate['metadata']['name'] = 'example-cert' + + document_mapping = { + "_GLOBAL_SUBSTITUTIONS_1_": [{ + "dest": { + "path": ".chart.values.tls.certificate" + }, + "src": { + "schema": "deckhand/Certificate/v1", + "name": "example-cert", + "path": "." + } + + }] + } + expected_data = { + 'chart': { + 'values': { + 'tls': { + 'certificate': 'CERTIFICATE DATA' + } + } + } + } + self._test_secret_substitution( + document_mapping, [certificate], expected_data) + + def test_secret_substitution_single_cleartext_with_pattern(self): + passphrase = self.secrets_factory.gen_test( + 'Passphrase', 'cleartext', data={'secret': 'my-secret-password'}) + passphrase['metadata']['name'] = 'example-password' + + document_mapping = { + "_GLOBAL_DATA_1_": { + 'data': { + 'chart': { + 'values': { + 'some_url': ( + 'http://admin:INSERT_PASSWORD_HERE' + '@service-name:8080/v1') + } + } + } + }, + "_GLOBAL_SUBSTITUTIONS_1_": [{ + "dest": { + "path": ".chart.values.some_url", + "pattern": "INSERT_[A-Z]+_HERE" + }, + "src": { + "schema": "deckhand/Passphrase/v1", + "name": "example-password", + "path": "." + } + }] + } + expected_data = { + 'chart': { + 'values': { + 'some_url': ( + 'http://admin:my-secret-password@service-name:8080/v1') + } + } + } + self._test_secret_substitution( + document_mapping, [passphrase], expected_data) + + def test_secret_substitution_double_cleartext(self): + certificate = self.secrets_factory.gen_test( + 'Certificate', 'cleartext', data={'secret': 'CERTIFICATE DATA'}) + certificate['metadata']['name'] = 'example-cert' + + certificate_key = self.secrets_factory.gen_test( + 'CertificateKey', 'cleartext', data={'secret': 'KEY DATA'}) + certificate_key['metadata']['name'] = 'example-key' + + document_mapping = { + "_GLOBAL_SUBSTITUTIONS_1_": [{ + "dest": { + "path": ".chart.values.tls.certificate" + }, + "src": { + "schema": "deckhand/Certificate/v1", + "name": "example-cert", + "path": "." + } + + }, { + "dest": { + "path": ".chart.values.tls.key" + }, + "src": { + "schema": "deckhand/CertificateKey/v1", + "name": "example-key", + "path": "." + } + + }] + } + expected_data = { + 'chart': { + 'values': { + 'tls': { + 'certificate': 'CERTIFICATE DATA', + 'key': 'KEY DATA' + } + } + } + } + self._test_secret_substitution( + document_mapping, [certificate, certificate_key], expected_data) + + def test_secret_substitution_multiple_cleartext(self): + certificate = self.secrets_factory.gen_test( + 'Certificate', 'cleartext', data={'secret': 'CERTIFICATE DATA'}) + certificate['metadata']['name'] = 'example-cert' + + certificate_key = self.secrets_factory.gen_test( + 'CertificateKey', 'cleartext', data={'secret': 'KEY DATA'}) + certificate_key['metadata']['name'] = 'example-key' + + passphrase = self.secrets_factory.gen_test( + 'Passphrase', 'cleartext', data={'secret': 'my-secret-password'}) + passphrase['metadata']['name'] = 'example-password' + + document_mapping = { + "_GLOBAL_DATA_1_": { + 'data': { + 'chart': { + 'values': { + 'some_url': ( + 'http://admin:INSERT_PASSWORD_HERE' + '@service-name:8080/v1') + } + } + } + }, + "_GLOBAL_SUBSTITUTIONS_1_": [{ + "dest": { + "path": ".chart.values.tls.certificate" + }, + "src": { + "schema": "deckhand/Certificate/v1", + "name": "example-cert", + "path": "." + } + + }, { + "dest": { + "path": ".chart.values.tls.key" + }, + "src": { + "schema": "deckhand/CertificateKey/v1", + "name": "example-key", + "path": "." + } + + }, { + "dest": { + "path": ".chart.values.some_url", + "pattern": "INSERT_[A-Z]+_HERE" + }, + "src": { + "schema": "deckhand/Passphrase/v1", + "name": "example-password", + "path": "." + } + }] + } + expected_data = { + 'chart': { + 'values': { + 'tls': { + 'certificate': 'CERTIFICATE DATA', + 'key': 'KEY DATA' + }, + 'some_url': ( + 'http://admin:my-secret-password@service-name:8080/v1') + } + } + } + self._test_secret_substitution( + document_mapping, [certificate, certificate_key, passphrase], + expected_data) diff --git a/deckhand/tests/unit/fake_policy.py b/deckhand/tests/unit/fake_policy.py new file mode 100644 index 00000000..771fd770 --- /dev/null +++ b/deckhand/tests/unit/fake_policy.py @@ -0,0 +1,31 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +policy_data = """ +"admin_api": "role:admin" +"deckhand:create_cleartext_documents": "rule:admin_api" +"deckhand:create_encrypted_documents": "rule:admin_api" +"deckhand:list_cleartext_documents": "rule:admin_api" +"deckhand:list_encrypted_documents": "rule:admin_api" +"deckhand:show_revision": "rule:admin_api" +"deckhand:list_revisions": "rule:admin_api" +"deckhand:delete_revisions": "rule:admin_api" +"deckhand:show_revision_diff": "rule:admin_api" +"deckhand:create_tag": "rule:admin_api" +"deckhand:show_tag": "rule:admin_api" +"deckhand:list_tags": "rule:admin_api" +"deckhand:delete_tag": "rule:admin_api" +"deckhand:delete_tags": "rule:admin_api" +""" diff --git a/deckhand/tests/unit/policy_fixture.py b/deckhand/tests/unit/policy_fixture.py new file mode 100644 index 00000000..8610b5d6 --- /dev/null +++ b/deckhand/tests/unit/policy_fixture.py @@ -0,0 +1,74 @@ +# Copyright 2012 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import yaml + +import fixtures +from oslo_config import cfg +from oslo_policy import opts as policy_opts +from oslo_policy import policy as oslo_policy + +from deckhand import policies +import deckhand.policy +from deckhand.tests.unit import fake_policy + + +CONF = cfg.CONF + + +class RealPolicyFixture(fixtures.Fixture): + """Load the live policy for tests. + + A base policy fixture that starts with the assumption that you'd + like to load and enforce the shipped default policy in tests. + + """ + + def setUp(self): + super(RealPolicyFixture, self).setUp() + self.policy_dir = self.useFixture(fixtures.TempDir()) + self.policy_file = os.path.join(self.policy_dir.path, + 'policy.yaml') + # Load the fake_policy data and add the missing default rules. + policy_rules = yaml.safe_load(fake_policy.policy_data) + self.add_missing_default_rules(policy_rules) + with open(self.policy_file, 'w') as f: + yaml.safe_dump(policy_rules, f) + + policy_opts.set_defaults(CONF) + CONF.set_override('policy_dirs', [], group='oslo_policy') + CONF.set_override('policy_file', self.policy_file, group='oslo_policy') + + deckhand.policy.reset() + deckhand.policy.init() + self.addCleanup(deckhand.policy.reset) + + def add_missing_default_rules(self, rules): + """Adds default rules and their values to the given rules dict. + + The given rulen dict may have an incomplete set of policy rules. + This method will add the default policy rules and their values to + the dict. It will not override the existing rules. + """ + for rule in policies.list_rules(): + if rule.name not in rules: + rules[rule.name] = rule.check_str + + def set_rules(self, rules, overwrite=True): + if isinstance(rules, dict): + rules = oslo_policy.Rules.from_dict(rules) + + policy = deckhand.policy._ENFORCER + policy.set_rules(rules, overwrite=overwrite) diff --git a/deckhand/tests/unit/test_policy.py b/deckhand/tests/unit/test_policy.py index 54246e83..e5b71309 100644 --- a/deckhand/tests/unit/test_policy.py +++ b/deckhand/tests/unit/test_policy.py @@ -14,12 +14,10 @@ import falcon import mock from oslo_policy import policy as common_policy -from deckhand.conf import config from deckhand.control import base as api_base -from deckhand import policy +import deckhand.policy from deckhand.tests.unit import base as test_base - -CONF = config.CONF +from deckhand.tests.unit import policy_fixture class PolicyBaseTestCase(test_base.DeckhandTestCase): @@ -33,18 +31,18 @@ class PolicyBaseTestCase(test_base.DeckhandTestCase): "deckhand:create_cleartext_documents": [['@']], "deckhand:list_cleartext_documents": [['rule:admin_api']] } - self.policy_enforcer = common_policy.Enforcer(CONF) + + self.policy = self.useFixture(policy_fixture.RealPolicyFixture()) self._set_rules() def _set_rules(self): - rules = common_policy.Rules.from_dict(self.rules) - self.policy_enforcer.set_rules(rules) - self.addCleanup(self.policy_enforcer.clear) + these_rules = common_policy.Rules.from_dict(self.rules) + deckhand.policy._ENFORCER.set_rules(these_rules) def _enforce_policy(self, action): api_args = self._get_args() - @policy.authorize(action) + @deckhand.policy.authorize(action) def noop(*args, **kwargs): pass @@ -53,8 +51,7 @@ class PolicyBaseTestCase(test_base.DeckhandTestCase): def _get_args(self): # Returns the first two arguments that would be passed to any falcon # on_{HTTP_VERB} method: (self (which is mocked), falcon Request obj). - falcon_req = api_base.DeckhandRequest( - mock.MagicMock(), policy_enforcer=self.policy_enforcer) + falcon_req = api_base.DeckhandRequest(mock.MagicMock()) return (mock.Mock(), falcon_req) diff --git a/deckhand/utils.py b/deckhand/utils.py index aa9165ad..76421bf5 100644 --- a/deckhand/utils.py +++ b/deckhand/utils.py @@ -17,6 +17,8 @@ import string import jsonpath_ng +from deckhand import errors + def to_camel_case(s): """Convert string to camel case.""" @@ -30,30 +32,114 @@ def to_snake_case(name): return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower() -def jsonpath_parse(document, jsonpath): - """Parse value given JSON path in the document. +def jsonpath_parse(data, jsonpath): + """Parse value in the data for the given ``jsonpath``. - Retrieve the value corresponding to document[jsonpath] where ``jsonpath`` - is a multi-part key. A multi-key is a series of keys and nested keys - concatenated together with ".". For exampple, ``jsonpath`` of - ".foo.bar.baz" should mean that ``document`` has the format: + Retrieve the nested entry corresponding to ``data[jsonpath]``. For + example, a ``jsonpath`` of ".foo.bar.baz" means that the data section + should conform to: .. code-block:: yaml - --- - foo: - bar: - baz: + --- + foo: + bar: + baz: - :param document: Dictionary used for extracting nested entry. - :param jsonpath: A multi-part key that references nested data in a - dictionary. - :returns: Nested entry in ``document`` if present, else None. + :param data: The `data` section of a document. + :param jsonpath: A multi-part key that references a nested path in + ``data``. + :returns: Entry that corresponds to ``data[jsonpath]`` if present, + else None. + + Example:: + + src_name = sub['src']['name'] + src_path = sub['src']['path'] + src_doc = db_api.document_get(schema=src_schema, name=src_name) + src_secret = utils.jsonpath_parse(src_doc['data'], src_path) + # Do something with the extracted secret from the source document. """ if jsonpath.startswith('.'): jsonpath = '$' + jsonpath p = jsonpath_ng.parse(jsonpath) - matches = p.find(document) + matches = p.find(data) if matches: return matches[0].value + + +def jsonpath_replace(data, value, jsonpath, pattern=None): + """Update value in ``data`` at the path specified by ``jsonpath``. + + If the nested path corresponding to ``jsonpath`` isn't found in ``data``, + the path is created as an empty ``{}`` for each sub-path along the + ``jsonpath``. + + :param data: The `data` section of a document. + :param value: The new value for ``data[jsonpath]``. + :param jsonpath: A multi-part key that references a nested path in + ``data``. + :param pattern: A regular expression pattern. + :returns: Updated value at ``data[jsonpath]``. + :raises: MissingDocumentPattern if ``pattern`` is not None and + ``data[jsonpath]`` doesn't exist. + + Example:: + + doc = { + 'data': { + 'some_url': http://admin:INSERT_PASSWORD_HERE@svc-name:8080/v1 + } + } + secret = 'super-duper-secret' + path = '$.some_url' + pattern = 'INSERT_[A-Z]+_HERE' + replaced_data = utils.jsonpath_replace( + doc['data'], secret, path, pattern) + # The returned URL will look like: + # http://admin:super-duper-secret@svc-name:8080/v1 + doc['data'].update(replaced_data) + """ + data = data.copy() + if jsonpath.startswith('.'): + jsonpath = '$' + jsonpath + + def _do_replace(): + p = jsonpath_ng.parse(jsonpath) + p_to_change = p.find(data) + + if p_to_change: + _value = value + if pattern: + to_replace = p_to_change[0].value + # value represents the value to inject into to_replace that + # matches the pattern. + try: + _value = re.sub(pattern, value, to_replace) + except TypeError: + _value = None + return p.update(data, _value) + + result = _do_replace() + if result: + return result + + # A pattern requires us to look up the data located at data[jsonpath] + # and then figure out what re.match(data[jsonpath], pattern) is (in + # pseudocode). But raise an exception in case the path isn't present in the + # data and a pattern has been provided since it is impossible to do the + # look up. + if pattern: + raise errors.MissingDocumentPattern( + data=data, path=jsonpath, pattern=pattern) + + # However, Deckhand should be smart enough to create the nested keys in the + # data if they don't exist and a pattern isn't required. + d = data + for path in jsonpath.split('.')[1:]: + if path not in d: + d.setdefault(path, {}) + d = d.get(path) + + return _do_replace() diff --git a/doc/design.md b/doc/design.md index 0bc371b2..9648f25a 100644 --- a/doc/design.md +++ b/doc/design.md @@ -290,7 +290,7 @@ layer example above, which includes `global`, `region` and `site` layers, a document in the `region` layer could insert data from a document in the `site` layer. -Here is a sample set of documents demonstrating subistution: +Here is a sample set of documents demonstrating substitution: ```yaml --- diff --git a/etc/deckhand/policy.yaml.sample b/etc/deckhand/policy.yaml.sample index 28de03d7..d4f3cc90 100644 --- a/etc/deckhand/policy.yaml.sample +++ b/etc/deckhand/policy.yaml.sample @@ -7,10 +7,6 @@ # revision history, whereby the target revision's documents are re- # created for # the new revision. -# -# Conditionally enforced for the endpoints below if the any of the -# documents in -# the request body have a `metadata.storagePolicy` of "cleartext". # PUT /api/v1.0/bucket/{bucket_name}/documents # POST /api/v1.0/rollback/{target_revision_id} #"deckhand:create_cleartext_documents": "rule:admin_api" @@ -22,9 +18,11 @@ # the new # revision. # +# Only enforced after ``create_cleartext_documents`` passes. +# # Conditionally enforced for the endpoints below if the any of the # documents in -# the request body have a `metadata.storagePolicy` of "encrypted". +# the request body have a ``metadata.storagePolicy`` of "encrypted". # PUT /api/v1.0/bucket/{bucket_name}/documents # POST /api/v1.0/rollback/{target_revision_id} #"deckhand:create_encrypted_documents": "rule:admin_api" @@ -33,31 +31,28 @@ # substitution applied) as well as fully layered and substituted # concrete # documents. -# -# Conditionally enforced for the endpoints below if the any of the -# documents in -# the request body have a `metadata.storagePolicy` of "cleartext". If -# policy -# enforcement fails, cleartext documents are omitted. # GET api/v1.0/revisions/{revision_id}/documents # GET api/v1.0/revisions/{revision_id}/rendered-documents #"deckhand:list_cleartext_documents": "rule:admin_api" -# List cleartext documents for a revision (with no layering or +# List encrypted documents for a revision (with no layering or # substitution applied) as well as fully layered and substituted # concrete # documents. # -# Conditionally enforced for the endpoints below if the any of the -# documents in -# the request body have a `metadata.storagePolicy` of "encrypted". If +# Only enforced after ``list_cleartext_documents`` passes. +# +# Conditionally enforced for the endpoints below if any of the +# documents in the +# request body have a ``metadata.storagePolicy`` of "encrypted". If # policy -# enforcement fails, encrypted documents are omitted. +# enforcement fails, encrypted documents are exluded from the +# response. # GET api/v1.0/revisions/{revision_id}/documents # GET api/v1.0/revisions/{revision_id}/rendered-documents #"deckhand:list_encrypted_documents": "rule:admin_api" -# Show details for a revision tag. +# Show details for a revision. # GET /api/v1.0/revisions/{revision_id} #"deckhand:show_revision": "rule:admin_api" diff --git a/releasenotes/notes/secret-substitution-6eff2c93bf11d82e.yaml b/releasenotes/notes/secret-substitution-6eff2c93bf11d82e.yaml new file mode 100644 index 00000000..d26dc589 --- /dev/null +++ b/releasenotes/notes/secret-substitution-6eff2c93bf11d82e.yaml @@ -0,0 +1,8 @@ +--- +features: + - | + Deckhand now supports secret substitution for documents. The endpoint + ``GET revisions/{revision_id}/rendered-documents`` has been added to + Deckhand, which allows the possibility of listing fully substituted + documents. Only documents with ``metadata.substitutions`` field undergo + secret substitution dynamically.