airship
/
deckhand
Code Issues Proposed changes

DECKHAND-66: Document substitution implementation 

This PS implements documentation substitution and
the rendered-documents endpoint. Each time the
rendered-documents is queried, the documents for
the reqeust revision_id dynamically undergo
secret substitution.

All functional tests related to secret substitution
have been unskipped.

Deckhand currently does not real testing for
verifying that secret substitution works
for encrypted documents. This will only happen
when integration testing is added to Deckhand to
test its interaction with Keystone and Barbican.

Included in this PS:
  - basic implementation for secret substitution
  - introduction of jsonpath_ng for searching for and
    updating jsonpaths in documents
  - rendered-documents endpoint
  - unit tests
  - all relevant functional tests unskipped
  - additional bucket controller tests include RBAC
    tests and framework testing RBAC via unit tests

Change-Id: I86f269a5b616b518e5f742a4005891412226fe2a
This commit is contained in:
Felipe Monteiro 2017-10-03 20:18:47 +01:00
parent 698f90a4cb
commit d2d2312af9
41 changed files with 959 additions and 207 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
AUTHORS
View File

@ -1,7 +1,9 @@
Alan Meadows <alan.meadows@gmail.com>
Anthony Lin <anthony.jclin@gmail.com>
Bryan Strassner <bryan.strassner@gmail.com>
Felipe Monteiro <felipe.monteiro@att.com>
Felipe Monteiro <fmontei@users.noreply.github.com>
Mark Burnett <mark.m.burnett@gmail.com>
Pete Birley <pete@port.direct>
Scott Hussey <sh8121@att.com>
Tin Lam <tin@irrational.io>

7
deckhand/context.py
View File

@ -14,9 +14,6 @@
from oslo_config import cfg
from oslo_context import context
from oslo_policy import policy as common_policy
from deckhand import policy
CONF = cfg.CONF
@ -28,12 +25,10 @@ class RequestContext(context.RequestContext):
accesses the system, as well as additional request information.
"""
def __init__(self, policy_enforcer=None, project=None, **kwargs):
def __init__(self, project=None, **kwargs):
if project:
kwargs['tenant'] = project
self.project = project
self.policy_enforcer = policy_enforcer or common_policy.Enforcer(CONF)
policy.register_rules(self.policy_enforcer)
super(RequestContext, self).__init__(**kwargs)
def to_dict(self):

2
deckhand/control/api.py
View File

@ -67,6 +67,8 @@ def start_api():
revision_diffing.RevisionDiffingResource()),
('revisions/{revision_id}/documents',
revision_documents.RevisionDocumentsResource()),
('revisions/{revision_id}/rendered-documents',
revision_documents.RenderedDocumentsResource()),
('revisions/{revision_id}/tags', revision_tags.RevisionTagsResource()),
('revisions/{revision_id}/tags/{tag}',
revision_tags.RevisionTagsResource()),

5
deckhand/control/base.py
View File

@ -51,10 +51,9 @@ class BaseResource(object):
class DeckhandRequest(falcon.Request):
def __init__(self, env, options=None, policy_enforcer=None):
def __init__(self, env, options=None):
super(DeckhandRequest, self).__init__(env, options)
self.context = context.RequestContext.from_environ(
self.env, policy_enforcer=policy_enforcer)
self.context = context.RequestContext.from_environ(self.env)
@property
def project_id(self):

75
deckhand/control/buckets.py
View File

@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import itertools
import yaml
import falcon
@ -37,6 +36,7 @@ class BucketsResource(api_base.BaseResource):
view_builder = document_view.ViewBuilder()
secrets_mgr = secrets_manager.SecretsManager()
@policy.authorize('deckhand:create_cleartext_documents')
def on_put(self, req, resp, bucket_name=None):
document_data = req.stream.read(req.content_length or 0)
try:
@ -47,10 +47,34 @@ class BucketsResource(api_base.BaseResource):
LOG.error(error_msg)
raise falcon.HTTPBadRequest(description=six.text_type(e))
# NOTE: Must validate documents before doing policy enforcement,
# because we expect certain formatting of the documents while doing
# policy enforcement.
validation_policies = self._create_validation_policies(documents)
for document in documents:
if document['metadata'].get('storagePolicy') == 'encrypted':
policy.conditional_authorize(
'deckhand:create_encrypted_documents', req.context)
break
self._prepare_secret_documents(documents)
# Save all the documents, including validation policies.
documents_to_create = documents + validation_policies
created_documents = self._create_revision_documents(
bucket_name, list(documents_to_create))
if created_documents:
resp.body = self.to_yaml_body(
self.view_builder.list(created_documents))
resp.status = falcon.HTTP_200
resp.append_header('Content-Type', 'application/x-yaml')
def _create_validation_policies(self, documents):
# All concrete documents in the payload must successfully pass their
# JSON schema validations. Otherwise raise an error.
try:
# NOTE: Must validate documents before doing policy enforcement,
# because we expect certain formatting of the documents while doing
# policy enforcement.
validation_policies = document_validation.DocumentValidation(
documents).validate_all()
except deckhand_errors.InvalidDocumentFormat as e:
@ -58,42 +82,25 @@ class BucketsResource(api_base.BaseResource):
# validation policy in the DB for future debugging, and only
# afterward raise an exception.
raise falcon.HTTPBadRequest(description=e.format_message())
return validation_policies
cleartext_documents = []
secret_documents = []
for document in documents:
if any([document['schema'].startswith(t)
for t in types.DOCUMENT_SECRET_TYPES]):
secret_documents.append(document)
else:
cleartext_documents.append(document)
if secret_documents and any(
[d['metadata'].get('storagePolicy') == 'encrypted'
for d in secret_documents]):
policy.conditional_authorize('deckhand:create_encrypted_documents',
req.context)
if cleartext_documents:
policy.conditional_authorize('deckhand:create_cleartext_documents',
req.context)
def _prepare_secret_documents(self, secret_documents):
# Encrypt data for secret documents, if any.
for document in secret_documents:
secret_data = self.secrets_mgr.create(document)
document['data'] = secret_data
# TODO(fmontei): Move all of this to document validation directly.
if document['metadata'].get('storagePolicy') == 'encrypted':
secret_data = self.secrets_mgr.create(document)
document['data'] = secret_data
elif any([document['schema'].startswith(t)
for t in types.DOCUMENT_SECRET_TYPES]):
document['data'] = {'secret': document['data']}
def _create_revision_documents(self, bucket_name, documents):
try:
documents_to_create = itertools.chain(
cleartext_documents, secret_documents, validation_policies)
created_documents = db_api.documents_create(
bucket_name, list(documents_to_create))
created_documents = db_api.documents_create(bucket_name, documents)
except deckhand_errors.DocumentExists as e:
raise falcon.HTTPConflict(description=e.format_message())
except Exception as e:
raise falcon.HTTPInternalServerError(description=six.text_type(e))
if created_documents:
resp.body = self.to_yaml_body(
self.view_builder.list(created_documents))
resp.status = falcon.HTTP_200
resp.append_header('Content-Type', 'application/x-yaml')
return created_documents

59
deckhand/control/revision_documents.py
View File

@ -19,6 +19,7 @@ from deckhand.control import base as api_base
from deckhand.control import common
from deckhand.control.views import document as document_view
from deckhand.db.sqlalchemy import api as db_api
from deckhand.engine import secrets_manager
from deckhand import errors
from deckhand import policy
@ -26,10 +27,11 @@ LOG = logging.getLogger(__name__)
class RevisionDocumentsResource(api_base.BaseResource):
"""API resource for realizing CRUD endpoints for revision documents."""
"""API resource for realizing revision documents endpoint."""
view_builder = document_view.ViewBuilder()
@policy.authorize('deckhand:list_cleartext_documents')
@common.sanitize_params([
'schema', 'metadata.name', 'metadata.layeringDefinition.abstract',
'metadata.layeringDefinition.layer', 'metadata.label',
@ -42,18 +44,13 @@ class RevisionDocumentsResource(api_base.BaseResource):
documents will be as originally posted with no substitutions or
layering applied.
"""
include_cleartext = policy.conditional_authorize(
'deckhand:list_cleartext_documents', req.context, do_raise=False)
include_encrypted = policy.conditional_authorize(
'deckhand:list_encrypted_documents', req.context, do_raise=False)
filters = sanitized_params.copy()
filters['metadata.storagePolicy'] = []
if include_cleartext:
filters['metadata.storagePolicy'].append('cleartext')
filters['metadata.storagePolicy'] = ['cleartext']
if include_encrypted:
filters['metadata.storagePolicy'].append('encrypted')
# Never return deleted documents to user.
filters['deleted'] = False
@ -66,3 +63,51 @@ class RevisionDocumentsResource(api_base.BaseResource):
resp.status = falcon.HTTP_200
resp.append_header('Content-Type', 'application/x-yaml')
resp.body = self.to_yaml_body(self.view_builder.list(documents))
class RenderedDocumentsResource(api_base.BaseResource):
"""API resource for realizing rendered documents endpoint.
Rendered documents are also revision documents, but unlike revision
documents, they are finalized documents, having undergone secret
substitution and document layering.
Returns a multi-document YAML response containing all the documents
matching the filters specified via query string parameters. Returned
documents will have secrets substituted into them and be layered with
other documents in the revision, in accordance with the ``LayeringPolicy``
that currently exists in the system.
"""
view_builder = document_view.ViewBuilder()
@policy.authorize('deckhand:list_cleartext_documents')
@common.sanitize_params([
'schema', 'metadata.name', 'metadata.label'])
def on_get(self, req, resp, sanitized_params, revision_id):
include_encrypted = policy.conditional_authorize(
'deckhand:list_encrypted_documents', req.context, do_raise=False)
filters = sanitized_params.copy()
filters['metadata.storagePolicy'] = ['cleartext']
if include_encrypted:
filters['metadata.storagePolicy'].append('encrypted')
try:
documents = db_api.revision_get_documents(
revision_id, **filters)
except (errors.RevisionNotFound) as e:
raise falcon.HTTPNotFound(description=e.format_message())
# TODO(fmontei): Currently the only phase of rendering that is
# performed is secret substitution, which can be done in any randomized
# order. However, secret substitution logic will have to be moved into
# a separate module that handles layering alongside substitution once
# layering has been fully integrated into this endpoint.
secrets_substitution = secrets_manager.SecretsSubstitution(documents)
rendered_documents = secrets_substitution.substitute_all()
resp.status = falcon.HTTP_200
resp.append_header('Content-Type', 'application/x-yaml')
resp.body = self.to_yaml_body(
self.view_builder.list(rendered_documents))

6
deckhand/control/rollback.py
View File

@ -34,12 +34,10 @@ class RollbackResource(api_base.BaseResource):
raise falcon.HTTPNotFound(description=e.format_message())
for document in latest_revision['documents']:
if document['metadata'].get('storagePolicy') == 'cleartext':
policy.conditional_authorize(
'deckhand:create_cleartext_documents', req.context)
elif document['metadata'].get('storagePolicy') == 'encrypted':
if document['metadata'].get('storagePolicy') == 'encrypted':
policy.conditional_authorize(
'deckhand:create_encrypted_documents', req.context)
break
try:
rollback_revision = db_api.revision_rollback(

31
deckhand/control/views/document.py
View File

@ -13,6 +13,7 @@
# limitations under the License.
from deckhand.control import common
from deckhand import types
class ViewBuilder(common.ViewBuilder):
@ -30,34 +31,30 @@ class ViewBuilder(common.ViewBuilder):
_collection_name = 'documents'
def list(self, documents):
# Edge case for when all documents are deleted from a bucket. Still
# need to return bucket_id and revision_id.
if len(documents) == 1 and documents[0]['deleted']:
resp_obj = {'status': {}}
resp_obj['status']['bucket'] = documents[0]['bucket_name']
resp_obj['status']['revision'] = documents[0]['revision_id']
return [resp_obj]
resp_list = []
attrs = ['id', 'metadata', 'data', 'schema']
for document in documents:
if document['deleted']:
continue
if document['schema'].startswith(types.VALIDATION_POLICY_SCHEMA):
continue
resp_obj = {x: document[x] for x in attrs}
resp_obj.setdefault('status', {})
resp_obj['status']['bucket'] = document['bucket_name']
resp_obj['status']['revision'] = document['revision_id']
resp_list.append(resp_obj)
# In the case where no documents are passed to PUT
# buckets/{{bucket_name}}/documents, we need to mangle the response
# body a bit. The revision_id and buckete_id should be returned, as
# at the very least the revision_id will be needed by the user.
# Edge case for when all documents are deleted from a bucket. To detect
# the edge case, check whether ``resp_list`` is empty and whether there
# are still documents to be returned. This means that all the documents
# are either deleted or validation policies. Either way, we still need
# to return bucket_id and revision_id, which should be the same
# across all the documents in ``documents``.
if not resp_list and documents:
resp_obj = {}
resp_obj.setdefault('status', {})
resp_obj['status']['bucket'] = documents[0]['bucket_id']
resp_obj = {'status': {}}
resp_obj['status']['bucket'] = documents[0]['bucket_name']
resp_obj['status']['revision'] = documents[0]['revision_id']
resp_list.append(resp_obj)
return [resp_obj]
return resp_list

2
deckhand/control/views/revision.py
View File

@ -59,7 +59,7 @@ class ViewBuilder(common.ViewBuilder):
success_status = 'success'
for vp in [d for d in revision['documents']
if d['schema'] == types.VALIDATION_POLICY_SCHEMA]:
if d['schema'].startswith(types.VALIDATION_POLICY_SCHEMA)]:
validation_policy = {}
validation_policy['name'] = vp.get('name')
validation_policy['url'] = self._gen_url(vp)

27
deckhand/db/sqlalchemy/api.py
View File

@ -261,18 +261,29 @@ def document_get(session=None, raw_dict=False, **filters):
"""
session = session or get_session()
# Retrieve the most recently created version of a document. Documents with
# the same metadata.name and schema can exist across different revisions,
# so it is necessary to use `first` instead of `one` to avoid errors.
document = session.query(models.Document)\
# TODO(fmontei): Currently Deckhand doesn't support filtering by nested
# JSON fields via sqlalchemy. For now, filter the documents using all
# "regular" filters via sqlalchemy and all nested filters via Python.
nested_filters = {}
for f in filters.copy():
if '.' in f:
nested_filters.setdefault(f, filters.pop(f))
# Documents with the the same metadata.name and schema can exist across
# different revisions, so it is necessary to order documents by creation
# date, then return the first document that matches all desired filters.
documents = session.query(models.Document)\
.filter_by(**filters)\
.order_by(models.Document.created_at.desc())\
.first()
.all()
if not document:
raise errors.DocumentNotFound(document=filters)
for doc in documents:
d = doc.to_dict(raw_dict=raw_dict)
if _apply_filters(d, **nested_filters):
return d
return document.to_dict(raw_dict=raw_dict)
filters.update(nested_filters)
raise errors.DocumentNotFound(document=filters)
####################

8
deckhand/engine/document.py
View File

@ -72,6 +72,9 @@ class Document(object):
def get_labels(self):
return self._inner['metadata']['labels']
def get_substitutions(self):
return self._inner['metadata'].get('substitutions', None)
def get_actions(self):
try:
return self._inner['metadata']['layeringDefinition']['actions']
@ -118,4 +121,7 @@ class Document(object):
return not self.__contains__(k)
def __repr__(self):
return repr(self._inner)
return '(%s, %s)' % (self.get_schema(), self.get_name())
def __str__(self):
return str(self._inner)

7
deckhand/engine/schema/v1_0/certificate_key_schema.py
View File

@ -27,6 +27,13 @@ schema = {
'pattern': '^(metadata/Document/v[1]{1}(\.[0]{1}){0,1})$',
},
'name': {'type': 'string'},
# Not strictly needed for secrets.
'layeringDefinition': {
'type': 'object',
'properties': {
'layer': {'type': 'string'}
}
},
'storagePolicy': {
'type': 'string',
'enum': ['encrypted', 'cleartext']

7
deckhand/engine/schema/v1_0/certificate_schema.py
View File

@ -27,6 +27,13 @@ schema = {
'pattern': '^(metadata/Document/v[1]{1}(\.[0]{1}){0,1})$',
},
'name': {'type': 'string'},
# Not strictly needed for secrets.
'layeringDefinition': {
'type': 'object',
'properties': {
'layer': {'type': 'string'}
}
},
'storagePolicy': {
'type': 'string',
'enum': ['encrypted', 'cleartext']

4
deckhand/engine/schema/v1_0/data_schema.py
View File

@ -33,6 +33,10 @@ schema = {
# Labels are optional.
'labels': {
'type': 'object'
},
'storagePolicy': {
'type': 'string',
'enum': ['encrypted', 'cleartext']
}
},
'additionalProperties': False,

4
deckhand/engine/schema/v1_0/document_schema.py
View File

@ -84,6 +84,10 @@ schema = {
'substitutions': {
'type': 'array',
'items': substitution_schema
},
'storagePolicy': {
'type': 'string',
'enum': ['encrypted', 'cleartext']
}
},
'additionalProperties': False,

6
deckhand/engine/schema/v1_0/layering_schema.py
View File

@ -26,7 +26,11 @@ schema = {
'type': 'string',
'pattern': '^(metadata/Control/v[1]{1}(\.[0]{1}){0,1})$'
},
'name': {'type': 'string'}
'name': {'type': 'string'},
'storagePolicy': {
'type': 'string',
'enum': ['encrypted', 'cleartext']
}
},
'additionalProperties': False,
'required': ['schema', 'name']

7
deckhand/engine/schema/v1_0/passphrase_schema.py
View File

@ -27,6 +27,13 @@ schema = {
'pattern': '^(metadata/Document/v[1]{1}(\.[0]{1}){0,1})$',
},
'name': {'type': 'string'},
# Not strictly needed for secrets.
'layeringDefinition': {
'type': 'object',
'properties': {
'layer': {'type': 'string'}
}
},
'storagePolicy': {
'type': 'string',
'enum': ['encrypted', 'cleartext']

6
deckhand/engine/schema/v1_0/validation_schema.py
View File

@ -26,7 +26,11 @@ schema = {
'type': 'string',
'pattern': '^(metadata/Control/v[1]{1}(\.[0]{1}){0,1})$'
},
'name': {'type': 'string'}
'name': {'type': 'string'},
'storagePolicy': {
'type': 'string',
'enum': ['encrypted', 'cleartext']
}
},
'additionalProperties': False,
'required': ['schema', 'name']

89
deckhand/engine/secrets_manager.py
View File

@ -12,7 +12,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_log import log as logging
from deckhand.barbican import driver
from deckhand.db.sqlalchemy import api as db_api
from deckhand.engine import document as document_wrapper
from deckhand import utils
LOG = logging.getLogger(__name__)
CLEARTEXT = 'cleartext'
ENCRYPTED = 'encrypted'
@ -34,22 +41,22 @@ class SecretsManager(object):
documents with the schemas enumerated below) must be stored using a
secure storage service like Barbican.
Documents with metadata.storagePolicy == "clearText" have their secrets
stored directly in Deckhand.
Documents with ``metadata.storagePolicy`` == "clearText" have their
secrets stored directly in Deckhand.
Documents with metadata.storagePolicy == "encrypted" are stored in
Documents with ``metadata.storagePolicy`` == "encrypted" are stored in
Barbican directly. Deckhand in turn stores the reference returned
by Barbican in Deckhand.
:param secret_doc: A Deckhand document with one of the following
schemas:
* deckhand/Certificate/v1
* deckhand/CertificateKey/v1
* deckhand/Passphrase/v1
* ``deckhand/Certificate/v1``
* ``deckhand/CertificateKey/v1``
* ``deckhand/Passphrase/v1``
:returns: Dictionary representation of
`deckhand.db.sqlalchemy.models.DocumentSecret`.
``deckhand.db.sqlalchemy.models.DocumentSecret``.
"""
encryption_type = secret_doc['metadata']['storagePolicy']
secret_type = self._get_secret_type(secret_doc['schema'])
@ -73,9 +80,9 @@ class SecretsManager(object):
def _get_secret_type(self, schema):
"""Get the Barbican secret type based on the following mapping:
deckhand/Certificate/v1 => certificate
deckhand/CertificateKey/v1 => private
deckhand/Passphrase/v1 => passphrase
``deckhand/Certificate/v1`` => certificate
``deckhand/CertificateKey/v1`` => private
``deckhand/Passphrase/v1`` => passphrase
:param schema: The document's schema.
:returns: The value corresponding to the mapping above.
@ -84,3 +91,65 @@ class SecretsManager(object):
if _schema == 'certificatekey':
return 'private'
return _schema
class SecretsSubstitution(object):
"""Class for document substitution logic for YAML files."""
def __init__(self, documents):
"""SecretSubstitution constructor.
:param documents: List of YAML documents in dictionary format that are
candidates for secret substitution. This class will automatically
detect documents that require substitution; documents need not be
filtered prior to being passed to the constructor.
"""
if not isinstance(documents, (list, tuple)):
documents = [documents]
substitute_docs = [document_wrapper.Document(d) for d in documents if
'substitutions' in d['metadata']]
self.documents = substitute_docs
def substitute_all(self):
"""Substitute all documents that have a `metadata.substitutions` field.
Concrete (non-abstract) documents can be used as a source of
substitution into other documents. This substitution is
layer-independent, a document in the region layer could insert data
from a document in the site layer.
:returns: List of fully substituted documents.
"""
LOG.debug('Substituting secrets for documents: %s', self.documents)
substituted_docs = []
for doc in self.documents:
LOG.debug(
'Checking for substitutions in schema=%s, metadata.name=%s',
doc.get_name(), doc.get_schema())
for sub in doc.get_substitutions():
src_schema = sub['src']['schema']
src_name = sub['src']['name']
src_path = sub['src']['path']
if src_path == '.':
src_path = '.secret'
# TODO(fmontei): Use secrets_manager for this logic. Need to
# check Barbican for the secret if it has been encrypted.
src_doc = db_api.document_get(
schema=src_schema, name=src_name, is_secret=True,
**{'metadata.layeringDefinition.abstract': False})
src_secret = utils.jsonpath_parse(src_doc['data'], src_path)
dest_path = sub['dest']['path']
dest_pattern = sub['dest'].get('pattern', None)
LOG.debug('Substituting from schema=%s name=%s src_path=%s '
'into dest_path=%s, dest_pattern=%s', src_schema,
src_name, src_path, dest_path, dest_pattern)
substituted_data = utils.jsonpath_replace(
doc['data'], src_secret, dest_path, dest_pattern)
doc['data'].update(substituted_data)
substituted_docs.append(doc.to_dict())
return substituted_docs

24
deckhand/errors.py
View File

@ -23,18 +23,11 @@ class DeckhandException(Exception):
code = 500
def __init__(self, message=None, **kwargs):
self.kwargs = kwargs
if 'code' not in self.kwargs:
try:
self.kwargs['code'] = self.code
except AttributeError:
pass
kwargs.setdefault('code', DeckhandException.code)
if not message:
try:
message = self.msg_fmt % kwargs
except Exception:
message = self.msg_fmt
@ -58,15 +51,6 @@ class InvalidDocumentFormat(DeckhandException):
super(InvalidDocumentFormat, self).__init__(**kwargs)
# TODO(fmontei): Remove this in a future commit.
class ApiError(Exception):
pass
class InvalidFormat(ApiError):
"""The YAML file is incorrectly formatted and cannot be read."""
class DocumentExists(DeckhandException):
msg_fmt = ("Document with schema %(schema)s and metadata.name "
"%(name)s already exists in bucket %(bucket)s.")
@ -100,6 +84,12 @@ class MissingDocumentKey(DeckhandException):
"Parent: %(parent)s. Child: %(child)s.")
class MissingDocumentPattern(DeckhandException):
msg_fmt = ("Substitution pattern %(pattern)s could not be found for the "
"JSON path %(path)s in the destination document data %(data)s.")
code = 400
class UnsupportedActionMethod(DeckhandException):
msg_fmt = ("Method in %(actions)s is invalid for document %(document)s.")
code = 400

29
deckhand/factories.py
View File

@ -61,7 +61,6 @@ class DocumentFactory(DeckhandFactory):
"layeringDefinition": {
"abstract": False,
"layer": "",
"parentSelector": "",
"actions": []
},
"name": "",
@ -92,7 +91,7 @@ class DocumentFactory(DeckhandFactory):
]
:param num_layers: Total number of layers. Only supported values
include 2 or 3.
include 1, 2 or 3.
:type num_layers: integer
:param docs_per_layer: The number of documents to be included per
layer. For example, if ``num_layers`` is 3, then ``docs_per_layer``
@ -105,12 +104,14 @@ class DocumentFactory(DeckhandFactory):
compatible with ``docs_per_layer``.
"""
# Set up the layering definition's layerOrder.
if num_layers == 2:
if num_layers == 1:
layer_order = ["global"]
elif num_layers == 2:
layer_order = ["global", "site"]
elif num_layers == 3:
layer_order = ["global", "region", "site"]
else:
raise ValueError("'num_layers' must either be 2 or 3.")
raise ValueError("'num_layers' must be a value between 1 - 3.")
self.LAYERING_DEFINITION['data']['layerOrder'] = layer_order
if not isinstance(docs_per_layer, (list, tuple)):
@ -225,14 +226,30 @@ class DocumentFactory(DeckhandFactory):
data_key = "_%s_DATA_%d_" % (layer_name.upper(), count + 1)
actions_key = "_%s_ACTIONS_%d_" % (
layer_name.upper(), count + 1)
sub_key = "_%s_SUBSTITUTIONS_%d_" % (
layer_name.upper(), count + 1)
try:
layer_template['data'] = mapping[data_key]['data']
except KeyError as e:
LOG.debug('Could not map %s because it was not found in '
'the `mapping` dict.', e.args[0])
pass
try:
layer_template['metadata']['layeringDefinition'][
'actions'] = mapping[actions_key]['actions']
except KeyError as e:
LOG.warning('Could not map %s because it was not found in '
'the `mapping` dict.', e.args[0])
LOG.debug('Could not map %s because it was not found in '
'the `mapping` dict.', e.args[0])
pass
try:
layer_template['metadata']['substitutions'] = mapping[
sub_key]
except KeyError as e:
LOG.debug('Could not map %s because it was not found in '
'the `mapping` dict.', e.args[0])
pass
rendered_template.append(layer_template)

25
deckhand/policies/document.py
View File

@ -24,10 +24,7 @@ document_policies = [
"""Create a batch of documents specified in the request body, whereby
a new revision is created. Also, roll back a revision to a previous one in the
revision history, whereby the target revision's documents are re-created for
the new revision.
Conditionally enforced for the endpoints below if the any of the documents in
the request body have a `metadata.storagePolicy` of "cleartext".""",
the new revision.""",
[
{
'method': 'PUT',
@ -46,8 +43,10 @@ a new revision is created. Also, roll back a revision to a previous one in the
history, whereby the target revision's documents are re-created for the new
revision.
Only enforced after ``create_cleartext_documents`` passes.
Conditionally enforced for the endpoints below if the any of the documents in
the request body have a `metadata.storagePolicy` of "encrypted".""",
the request body have a ``metadata.storagePolicy`` of "encrypted".""",
[
{
'method': 'PUT',
@ -63,11 +62,7 @@ the request body have a `metadata.storagePolicy` of "encrypted".""",
base.RULE_ADMIN_API,
"""List cleartext documents for a revision (with no layering or
substitution applied) as well as fully layered and substituted concrete
documents.
Conditionally enforced for the endpoints below if the any of the documents in
the request body have a `metadata.storagePolicy` of "cleartext". If policy
enforcement fails, cleartext documents are omitted.""",
documents.""",
[
{
'method': 'GET',
@ -81,13 +76,15 @@ enforcement fails, cleartext documents are omitted.""",
policy.DocumentedRuleDefault(
base.POLICY_ROOT % 'list_encrypted_documents',
base.RULE_ADMIN_API,
"""List cleartext documents for a revision (with no layering or
"""List encrypted documents for a revision (with no layering or
substitution applied) as well as fully layered and substituted concrete
documents.
Conditionally enforced for the endpoints below if the any of the documents in
the request body have a `metadata.storagePolicy` of "encrypted". If policy
enforcement fails, encrypted documents are omitted.""",
Only enforced after ``list_cleartext_documents`` passes.
Conditionally enforced for the endpoints below if any of the documents in the
request body have a ``metadata.storagePolicy`` of "encrypted". If policy
enforcement fails, encrypted documents are exluded from the response.""",
[
{
'method': 'GET',

38
deckhand/policy.py
View File

@ -25,22 +25,54 @@ from deckhand import policies
CONF = cfg.CONF
LOG = logging.getLogger(__name__)
_ENFORCER = None
def reset():
global _ENFORCER
if _ENFORCER:
_ENFORCER.clear()
_ENFORCER = None
def init(policy_file=None, rules=None, default_rule=None, use_conf=True):
"""Init an Enforcer class.
:param policy_file: Custom policy file to use, if none is specified,
``CONF.policy_file`` will be used.
:param rules: Default dictionary / Rules to use. It will be
considered just in the first instantiation.
:param default_rule: Default rule to use; ``CONF.default_rule`` will
be used if none is specified.
:param use_conf: Whether to load rules from config file.
"""
global _ENFORCER
if not _ENFORCER:
_ENFORCER = policy.Enforcer(CONF,
policy_file=policy_file,
rules=rules,
default_rule=default_rule,
use_conf=use_conf)
register_rules(_ENFORCER)
def _do_enforce_rbac(action, context, do_raise=True):
policy_enforcer = context.policy_enforcer
init()
credentials = context.to_policy_values()
target = {'project_id': context.project_id,
'user_id': context.user_id}
exc = errors.PolicyNotAuthorized
try:
# oslo.policy supports both enforce and authorize. authorize is
# `oslo.policy` supports both enforce and authorize. authorize is
# stricter because it'll raise an exception if the policy action is
# not found in the list of registered rules. This means that attempting
# to enforce anything not found in ``deckhand.policies`` will error out
# with a 'Policy not registered' message.
return policy_enforcer.authorize(
return _ENFORCER.authorize(
action, target, context.to_dict(), do_raise=do_raise,
exc=exc, action=action)
except policy.PolicyNotRegistered as e:

10
deckhand/tests/functional/gabbits/document-render-success-multiple-bucket-with-substitution.yaml
View File

@ -16,25 +16,22 @@ tests:
desc: Begin testing from known state.
DELETE: /api/v1.0/revisions
status: 204
skip: Not implemented.
- name: add_bucket_a
desc: Create documents for bucket a
PUT: /api/v1.0/bucket/a/documents
status: 200
data: <@resources/design-doc-substition-sample-split-bucket-a.yaml
skip: Not implemented.
data: <@resources/design-doc-substitution-sample-split-bucket-a.yaml
- name: add_bucket_b
desc: Create documents for bucket b
PUT: /api/v1.0/bucket/b/documents
status: 200
data: <@resources/design-doc-substition-sample-split-bucket-b.yaml
skip: Not implemented.
data: <@resources/design-doc-substitution-sample-split-bucket-b.yaml
- name: verify_substitutions
desc: Check for expected substitutions
GET: /api/v1.0/revisions/$RESPONSE['$.[0].revision']/rendered-documents?schema=armada/Chart/v1
GET: /api/v1.0/revisions/$RESPONSE['$.[0].status.revision']/rendered-documents?schema=armada/Chart/v1
status: 200
response_multidoc_jsonpaths:
$.[*].metadata.name: example-chart-01
@ -49,4 +46,3 @@ tests:
key: |
KEY DATA
some_url: http://admin:my-secret-password@service-name:8080/v1
skip: Not implemented.

7
deckhand/tests/functional/gabbits/document-render-success-single-bucket-with-substitution.yaml
View File

@ -15,18 +15,16 @@ tests:
desc: Begin testing from known state.
DELETE: /api/v1.0/revisions
status: 204
skip: Not implemented.
- name: initialize
desc: Create initial documents
PUT: /api/v1.0/bucket/mop/documents
status: 200
data: <@resources/design-doc-substition-sample.yaml
skip: Not implemented.
data: <@resources/design-doc-substitution-sample.yaml
- name: verify_substitutions
desc: Check for expected substitutions
GET: /api/v1.0/revisions/$RESPONSE['$.[0].revision']/rendered-documents?schema=armada/Chart/v1
GET: /api/v1.0/revisions/$RESPONSE['$.[0].status.revision']/rendered-documents?schema=armada/Chart/v1
status: 200
response_multidoc_jsonpaths:
$.[*].metadata.name: example-chart-01
@ -41,4 +39,3 @@ tests:
key: |
KEY DATA
some_url: http://admin:my-secret-password@service-name:8080/v1
skip: Not implemented.

3
deckhand/tests/functional/gabbits/resources/design-doc-substitution-sample-split-bucket-a.yaml
View File

@ -2,9 +2,10 @@
schema: deckhand/CertificateKey/v1
metadata:
name: example-key
storagePolicy: encrypted
schema: metadata/Document/v1
layeringDefinition:
layer: site
storagePolicy: cleartext
data: |
KEY DATA
...

8
deckhand/tests/functional/gabbits/resources/design-doc-substitution-sample-split-bucket-b.yaml
View File

@ -2,24 +2,26 @@
schema: deckhand/Certificate/v1
metadata:
name: example-cert
storagePolicy: cleartext
schema: metadata/Document/v1
layeringDefinition:
layer: site
storagePolicy: cleartext
data: |
CERTIFICATE DATA
---
schema: deckhand/Passphrase/v1
metadata:
name: example-password
storagePolicy: encrypted
schema: metadata/Document/v1
layeringDefinition:
layer: site
storagePolicy: cleartext
data: my-secret-password
---
schema: armada/Chart/v1
metadata:
name: example-chart-01
storagePolicy: cleartext
schema: metadata/Document/v1
layeringDefinition:
layer: region
substitutions:

11
deckhand/tests/functional/gabbits/resources/design-doc-substitution-sample.yaml
View File

@ -2,33 +2,36 @@
schema: deckhand/Certificate/v1
metadata:
name: example-cert
storagePolicy: cleartext
schema: metadata/Document/v1
layeringDefinition:
layer: site
storagePolicy: cleartext
data: |
CERTIFICATE DATA
---
schema: deckhand/CertificateKey/v1
metadata:
name: example-key
storagePolicy: encrypted
schema: metadata/Document/v1
layeringDefinition:
layer: site
storagePolicy: cleartext
data: |
KEY DATA
---
schema: deckhand/Passphrase/v1
metadata:
name: example-password
storagePolicy: encrypted
schema: metadata/Document/v1
layeringDefinition:
layer: site
storagePolicy: cleartext
data: my-secret-password
---
schema: armada/Chart/v1
metadata:
name: example-chart-01
storagePolicy: cleartext
schema: metadata/Document/v1
layeringDefinition:
layer: region
substitutions:

4
deckhand/tests/functional/gabbits/revision-diff-success.yaml
View File

@ -196,6 +196,10 @@ tests:
PUT: /api/v1.0/bucket/bucket_mistake/documents
status: 200
data: ""
# Verification for whether a bucket_name was returned even though all the
# documents for this bucket were deleted.
response_multidoc_jsonpaths:
$.[*].status.bucket: bucket_mistake
- name: verify_diff_between_created_and_deleted_mistake
desc: Validates response for deletion between the last 2 revisions

2
deckhand/tests/unit/control/base.py
View File

@ -16,6 +16,7 @@ from falcon import testing as falcon_testing
from deckhand.control import api
from deckhand.tests.unit import base as test_base
from deckhand.tests.unit import policy_fixture
class BaseControllerTest(test_base.DeckhandWithDBTestCase,
@ -25,3 +26,4 @@ class BaseControllerTest(test_base.DeckhandWithDBTestCase,
def setUp(self):
super(BaseControllerTest, self).setUp()
self.app = falcon_testing.TestClient(api.start_api())
self.policy = self.useFixture(policy_fixture.RealPolicyFixture())

19
deckhand/tests/unit/control/test_api_initialization.py
View File

@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import inspect
import mock
from deckhand.control import api
@ -24,6 +25,7 @@ from deckhand.control import revisions
from deckhand.control import rollback
from deckhand.control import versions
from deckhand.tests.unit import base as test_base
from deckhand import utils
class TestApi(test_base.DeckhandTestCase):
@ -32,11 +34,16 @@ class TestApi(test_base.DeckhandTestCase):
super(TestApi, self).setUp()
for resource in (buckets, revision_diffing, revision_documents,
revision_tags, revisions, rollback, versions):
resource_name = resource.__name__.split('.')[-1]
resource_obj = self.patchobject(
resource, '%sResource' % resource_name.title().replace(
'_', ''), autospec=True)
setattr(self, '%s_resource' % resource_name, resource_obj)
class_names = self._get_module_class_names(resource)
for class_name in class_names:
resource_obj = self.patchobject(
resource, class_name, autospec=True)
setattr(self, utils.to_snake_case(class_name), resource_obj)
def _get_module_class_names(self, module):
class_names = [obj.__name__ for name, obj in inspect.getmembers(module)
if inspect.isclass(obj)]
return class_names
@mock.patch.object(api, 'db_api', autospec=True)
@mock.patch.object(api, 'logging', autospec=True)
@ -62,6 +69,8 @@ class TestApi(test_base.DeckhandTestCase):
self.revision_diffing_resource()),
mock.call('/api/v1.0/revisions/{revision_id}/documents',
self.revision_documents_resource()),
mock.call('/api/v1.0/revisions/{revision_id}/rendered-documents',
self.rendered_documents_resource()),
mock.call('/api/v1.0/revisions/{revision_id}/tags',
self.revision_tags_resource()),
mock.call('/api/v1.0/revisions/{revision_id}/tags/{tag}',

132
deckhand/tests/unit/control/test_buckets_controller.py
View File

@ -12,17 +12,106 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import yaml
import mock
from oslo_config import cfg
from deckhand.control import buckets
from deckhand import factories
from deckhand.tests.unit.control import base as test_base
CONF = cfg.CONF
class TestBucketsController(test_base.BaseControllerTest):
"""Test suite for validating positive scenarios bucket controller."""
"""Test suite for validating positive scenarios for bucket controller."""
def test_put_bucket(self):
rules = {'deckhand:create_cleartext_documents': '@'}
self.policy.set_rules(rules)
documents_factory = factories.DocumentFactory(2, [1, 1])
document_mapping = {
"_GLOBAL_DATA_1_": {"data": {"a": {"x": 1, "y": 2}}},
"_SITE_DATA_1_": {"data": {"a": {"x": 7, "z": 3}, "b": 4}},
"_SITE_ACTIONS_1_": {
"actions": [{"method": "merge", "path": "."}]}
}
payload = documents_factory.gen_test(document_mapping)
resp = self.app.simulate_put('/api/v1.0/bucket/mop/documents',
body=yaml.safe_dump_all(payload))
self.assertEqual(200, resp.status_code)
created_documents = list(yaml.safe_load_all(resp.text))
self.assertEqual(3, len(created_documents))
expected = sorted([(d['schema'], d['metadata']['name'])
for d in payload])
actual = sorted([(d['schema'], d['metadata']['name'])
for d in created_documents])
self.assertEqual(expected, actual)
def test_put_bucket_with_secret(self):
def _do_test(payload):
resp = self.app.simulate_put('/api/v1.0/bucket/mop/documents',
body=yaml.safe_dump_all(payload))
self.assertEqual(200, resp.status_code)
created_documents = list(yaml.safe_load_all(resp.text))
self.assertEqual(1, len(created_documents))
expected = sorted([(d['schema'], d['metadata']['name'])
for d in payload])
actual = sorted([(d['schema'], d['metadata']['name'])
for d in created_documents])
self.assertEqual(expected, actual)
self.assertEqual({'secret': payload[0]['data']},
created_documents[0]['data'])
# Verify whether creating a cleartext secret works.
rules = {'deckhand:create_cleartext_documents': '@'}
self.policy.set_rules(rules)
secrets_factory = factories.DocumentSecretFactory()
payload = [secrets_factory.gen_test('Certificate', 'cleartext')]
_do_test(payload)
# Verify whether creating an encrypted secret works.
rules = {'deckhand:create_cleartext_documents': '@',
'deckhand:create_encrypted_documents': '@'}
self.policy.set_rules(rules)
secrets_factory = factories.DocumentSecretFactory()
payload = [secrets_factory.gen_test('Certificate', 'encrypted')]
with mock.patch.object(buckets.BucketsResource, 'secrets_mgr',
autospec=True) as mock_secrets_mgr:
mock_secrets_mgr.create.return_value = {
'secret': payload[0]['data']}
_do_test(payload)
# Verify whether any document can be encrypted if its
# `metadata.storagePolicy`='encrypted'. In the case below,
# a generic document is tested.
documents_factory = factories.DocumentFactory(1, [1])
document_mapping = {
"_GLOBAL_DATA_1_": {"data": {"a": {"x": 1, "y": 2}}}
}
payload = documents_factory.gen_test(document_mapping,
global_abstract=False)
payload[-1]['metadata']['storagePolicy'] = 'encrypted'
with mock.patch.object(buckets.BucketsResource, 'secrets_mgr',
autospec=True) as mock_secrets_mgr:
mock_secrets_mgr.create.return_value = {
'secret': payload[-1]['data']}
_do_test([payload[-1]])
class TestBucketsControllerNegative(test_base.BaseControllerTest):
"""Test suite for validating negative scenarios bucket controller."""
"""Test suite for validating negative scenarios for bucket controller."""
def test_put_bucket_with_invalid_document_payload(self):
rules = {'deckhand:create_cleartext_documents': '@'}
self.policy.set_rules(rules)
no_colon_spaces = """
name:foo
schema:
@ -38,3 +127,42 @@ schema:
body=payload)
self.assertEqual(400, resp.status_code)
self.assertRegexpMatches(resp.text, error_re[idx])
class TestBucketsControllerNegativeRBAC(test_base.BaseControllerTest):
"""Test suite for validating negative RBAC scenarios for bucket
controller.
"""
def test_put_bucket_cleartext_documents_except_forbidden(self):
rules = {'deckhand:create_cleartext_documents': 'rule:admin_api'}
self.policy.set_rules(rules)
documents_factory = factories.DocumentFactory(2, [1, 1])
payload = documents_factory.gen_test({})
resp = self.app.simulate_put('/api/v1.0/bucket/mop/documents',
body=yaml.safe_dump_all(payload))
self.assertEqual(403, resp.status_code)
def test_put_bucket_cleartext_secret_except_forbidden(self):
rules = {'deckhand:create_cleartext_documents': 'rule:admin_api'}
self.policy.set_rules(rules)
secrets_factory = factories.DocumentSecretFactory()
payload = [secrets_factory.gen_test('Certificate', 'cleartext')]
resp = self.app.simulate_put('/api/v1.0/bucket/mop/documents',
body=yaml.safe_dump_all(payload))
self.assertEqual(403, resp.status_code)
def test_put_bucket_encrypted_secret_except_forbidden(self):
rules = {'deckhand:create_encrypted_documents': 'rule:admin_api'}
self.policy.set_rules(rules)
secrets_factory = factories.DocumentSecretFactory()
payload = [secrets_factory.gen_test('Certificate', 'encrypted')]
resp = self.app.simulate_put('/api/v1.0/bucket/mop/documents',
body=yaml.safe_dump_all(payload))
self.assertEqual(403, resp.status_code)

3
deckhand/tests/unit/db/base.py
View File

@ -40,7 +40,8 @@ class DocumentFixture(object):
'layeringDefinition': {
'abstract': test_utils.rand_bool(),
'layer': test_utils.rand_name('layer')
}
},
'storagePolicy': test_utils.rand_name('storage_policy')
},
'schema': test_utils.rand_name('schema')}
fixture.update(kwargs)

218
deckhand/tests/unit/engine/test_secrets_manager.py
View File

@ -12,13 +12,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import copy
from deckhand.engine import secrets_manager
from deckhand import factories
from deckhand.tests import test_utils
from deckhand.tests.unit.db import base
from deckhand.tests.unit.db import base as test_base
class TestSecretsManager(base.TestDbBase):
class TestSecretsManager(test_base.TestDbBase):
def setUp(self):
super(TestSecretsManager, self).setUp()
@ -71,3 +73,215 @@ class TestSecretsManager(base.TestDbBase):
def test_create_encrypted_passphrase(self):
self._test_create_secret('encrypted', 'Passphrase')
class TestSecretsSubstitution(test_base.TestDbBase):
def setUp(self):
super(TestSecretsSubstitution, self).setUp()
self.document_factory = factories.DocumentFactory(1, [1])
self.secrets_factory = factories.DocumentSecretFactory()
def _test_secret_substitution(self, document_mapping, secret_documents,
expected_data):
payload = self.document_factory.gen_test(document_mapping,
global_abstract=False)
bucket_name = test_utils.rand_name('bucket')
documents = self.create_documents(
bucket_name, secret_documents + [payload[-1]])
expected_documents = copy.deepcopy([documents[-1]])
expected_documents[0]['data'] = expected_data
secret_substitution = secrets_manager.SecretsSubstitution(documents)
substituted_docs = secret_substitution.substitute_all()
self.assertEqual(expected_documents, substituted_docs)
def test_secret_substitution_single_cleartext(self):
certificate = self.secrets_factory.gen_test(
'Certificate', 'cleartext', data={'secret': 'CERTIFICATE DATA'})
certificate['metadata']['name'] = 'example-cert'
document_mapping = {
"_GLOBAL_SUBSTITUTIONS_1_": [{
"dest": {
"path": ".chart.values.tls.certificate"
},
"src": {
"schema": "deckhand/Certificate/v1",
"name": "example-cert",
"path": "."
}
}]
}
expected_data = {
'chart': {
'values': {
'tls': {
'certificate': 'CERTIFICATE DATA'
}
}
}
}
self._test_secret_substitution(
document_mapping, [certificate], expected_data)
def test_secret_substitution_single_cleartext_with_pattern(self):
passphrase = self.secrets_factory.gen_test(
'Passphrase', 'cleartext', data={'secret': 'my-secret-password'})
passphrase['metadata']['name'] = 'example-password'
document_mapping = {
"_GLOBAL_DATA_1_": {
'data': {
'chart': {
'values': {
'some_url': (
'http://admin:INSERT_PASSWORD_HERE'
'@service-name:8080/v1')
}
}
}
},
"_GLOBAL_SUBSTITUTIONS_1_": [{
"dest": {
"path": ".chart.values.some_url",
"pattern": "INSERT_[A-Z]+_HERE"
},
"src": {
"schema": "deckhand/Passphrase/v1",
"name": "example-password",
"path": "."
}
}]
}
expected_data = {
'chart': {
'values': {
'some_url': (
'http://admin:my-secret-password@service-name:8080/v1')
}
}
}
self._test_secret_substitution(
document_mapping, [passphrase], expected_data)
def test_secret_substitution_double_cleartext(self):
certificate = self.secrets_factory.gen_test(
'Certificate', 'cleartext', data={'secret': 'CERTIFICATE DATA'})
certificate['metadata']['name'] = 'example-cert'
certificate_key = self.secrets_factory.gen_test(
'CertificateKey', 'cleartext', data={'secret': 'KEY DATA'})
certificate_key['metadata']['name'] = 'example-key'
document_mapping = {
"_GLOBAL_SUBSTITUTIONS_1_": [{
"dest": {
"path": ".chart.values.tls.certificate"
},
"src": {
"schema": "deckhand/Certificate/v1",
"name": "example-cert",
"path": "."
}
}, {
"dest": {
"path": ".chart.values.tls.key"
},
"src": {
"schema": "deckhand/CertificateKey/v1",
"name": "example-key",
"path": "."
}
}]
}
expected_data = {
'chart': {
'values': {
'tls': {
'certificate': 'CERTIFICATE DATA',
'key': 'KEY DATA'
}
}
}
}
self._test_secret_substitution(
document_mapping, [certificate, certificate_key], expected_data)
def test_secret_substitution_multiple_cleartext(self):
certificate = self.secrets_factory.gen_test(
'Certificate', 'cleartext', data={'secret': 'CERTIFICATE DATA'})
certificate['metadata']['name'] = 'example-cert'
certificate_key = self.secrets_factory.gen_test(
'CertificateKey', 'cleartext', data={'secret': 'KEY DATA'})
certificate_key['metadata']['name'] = 'example-key'
passphrase = self.secrets_factory.gen_test(
'Passphrase', 'cleartext', data={'secret': 'my-secret-password'})
passphrase['metadata']['name'] = 'example-password'
document_mapping = {
"_GLOBAL_DATA_1_": {
'data': {
'chart': {
'values': {
'some_url': (
'http://admin:INSERT_PASSWORD_HERE'
'@service-name:8080/v1')
}
}
}
},
"_GLOBAL_SUBSTITUTIONS_1_": [{
"dest": {
"path": ".chart.values.tls.certificate"
},
"src": {
"schema": "deckhand/Certificate/v1",
"name": "example-cert",
"path": "."
}
}, {
"dest": {
"path": ".chart.values.tls.key"
},
"src": {
"schema": "deckhand/CertificateKey/v1",
"name": "example-key",
"path": "."
}
}, {
"dest": {
"path": ".chart.values.some_url",
"pattern": "INSERT_[A-Z]+_HERE"
},
"src": {
"schema": "deckhand/Passphrase/v1",
"name": "example-password",
"path": "."
}
}]
}
expected_data = {
'chart': {
'values': {
'tls': {
'certificate': 'CERTIFICATE DATA',
'key': 'KEY DATA'
},
'some_url': (
'http://admin:my-secret-password@service-name:8080/v1')
}
}
}
self._test_secret_substitution(
document_mapping, [certificate, certificate_key, passphrase],
expected_data)

31
deckhand/tests/unit/fake_policy.py Normal file
View File

@ -0,0 +1,31 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
policy_data = """
"admin_api": "role:admin"
"deckhand:create_cleartext_documents": "rule:admin_api"
"deckhand:create_encrypted_documents": "rule:admin_api"
"deckhand:list_cleartext_documents": "rule:admin_api"
"deckhand:list_encrypted_documents": "rule:admin_api"
"deckhand:show_revision": "rule:admin_api"
"deckhand:list_revisions": "rule:admin_api"
"deckhand:delete_revisions": "rule:admin_api"
"deckhand:show_revision_diff": "rule:admin_api"
"deckhand:create_tag": "rule:admin_api"
"deckhand:show_tag": "rule:admin_api"
"deckhand:list_tags": "rule:admin_api"
"deckhand:delete_tag": "rule:admin_api"
"deckhand:delete_tags": "rule:admin_api"
"""

74
deckhand/tests/unit/policy_fixture.py Normal file
View File

@ -0,0 +1,74 @@
# Copyright 2012 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
import yaml
import fixtures
from oslo_config import cfg
from oslo_policy import opts as policy_opts
from oslo_policy import policy as oslo_policy
from deckhand import policies
import deckhand.policy
from deckhand.tests.unit import fake_policy
CONF = cfg.CONF
class RealPolicyFixture(fixtures.Fixture):
"""Load the live policy for tests.
A base policy fixture that starts with the assumption that you'd
like to load and enforce the shipped default policy in tests.
"""
def setUp(self):
super(RealPolicyFixture, self).setUp()
self.policy_dir = self.useFixture(fixtures.TempDir())
self.policy_file = os.path.join(self.policy_dir.path,
'policy.yaml')
# Load the fake_policy data and add the missing default rules.
policy_rules = yaml.safe_load(fake_policy.policy_data)
self.add_missing_default_rules(policy_rules)
with open(self.policy_file, 'w') as f:
yaml.safe_dump(policy_rules, f)
policy_opts.set_defaults(CONF)
CONF.set_override('policy_dirs', [], group='oslo_policy')
CONF.set_override('policy_file', self.policy_file, group='oslo_policy')
deckhand.policy.reset()
deckhand.policy.init()
self.addCleanup(deckhand.policy.reset)
def add_missing_default_rules(self, rules):
"""Adds default rules and their values to the given rules dict.
The given rulen dict may have an incomplete set of policy rules.
This method will add the default policy rules and their values to
the dict. It will not override the existing rules.
"""
for rule in policies.list_rules():
if rule.name not in rules:
rules[rule.name] = rule.check_str
def set_rules(self, rules, overwrite=True):
if isinstance(rules, dict):
rules = oslo_policy.Rules.from_dict(rules)
policy = deckhand.policy._ENFORCER
policy.set_rules(rules, overwrite=overwrite)

19
deckhand/tests/unit/test_policy.py
View File

@ -14,12 +14,10 @@ import falcon
import mock
from oslo_policy import policy as common_policy
from deckhand.conf import config
from deckhand.control import base as api_base
from deckhand import policy
import deckhand.policy
from deckhand.tests.unit import base as test_base
CONF = config.CONF
from deckhand.tests.unit import policy_fixture
class PolicyBaseTestCase(test_base.DeckhandTestCase):
@ -33,18 +31,18 @@ class PolicyBaseTestCase(test_base.DeckhandTestCase):
"deckhand:create_cleartext_documents": [['@']],
"deckhand:list_cleartext_documents": [['rule:admin_api']]
}
self.policy_enforcer = common_policy.Enforcer(CONF)
self.policy = self.useFixture(policy_fixture.RealPolicyFixture())
self._set_rules()
def _set_rules(self):
rules = common_policy.Rules.from_dict(self.rules)
self.policy_enforcer.set_rules(rules)
self.addCleanup(self.policy_enforcer.clear)
these_rules = common_policy.Rules.from_dict(self.rules)
deckhand.policy._ENFORCER.set_rules(these_rules)
def _enforce_policy(self, action):
api_args = self._get_args()
@policy.authorize(action)
@deckhand.policy.authorize(action)
def noop(*args, **kwargs):
pass
@ -53,8 +51,7 @@ class PolicyBaseTestCase(test_base.DeckhandTestCase):
def _get_args(self):
# Returns the first two arguments that would be passed to any falcon
# on_{HTTP_VERB} method: (self (which is mocked), falcon Request obj).
falcon_req = api_base.DeckhandRequest(
mock.MagicMock(), policy_enforcer=self.policy_enforcer)
falcon_req = api_base.DeckhandRequest(mock.MagicMock())
return (mock.Mock(), falcon_req)

116
deckhand/utils.py
View File

@ -17,6 +17,8 @@ import string
import jsonpath_ng
from deckhand import errors
def to_camel_case(s):
"""Convert string to camel case."""
@ -30,30 +32,114 @@ def to_snake_case(name):
return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower()
def jsonpath_parse(document, jsonpath):
"""Parse value given JSON path in the document.
def jsonpath_parse(data, jsonpath):
"""Parse value in the data for the given ``jsonpath``.
Retrieve the value corresponding to document[jsonpath] where ``jsonpath``
is a multi-part key. A multi-key is a series of keys and nested keys
concatenated together with ".". For exampple, ``jsonpath`` of
".foo.bar.baz" should mean that ``document`` has the format:
Retrieve the nested entry corresponding to ``data[jsonpath]``. For
example, a ``jsonpath`` of ".foo.bar.baz" means that the data section
should conform to:
.. code-block:: yaml
---
foo:
bar:
baz: <data_to_be_extracted_here>
---
foo:
bar:
baz: <data_to_be_extracted_here>
:param document: Dictionary used for extracting nested entry.
:param jsonpath: A multi-part key that references nested data in a
dictionary.
:returns: Nested entry in ``document`` if present, else None.
:param data: The `data` section of a document.
:param jsonpath: A multi-part key that references a nested path in
``data``.
:returns: Entry that corresponds to ``data[jsonpath]`` if present,
else None.
Example::
src_name = sub['src']['name']
src_path = sub['src']['path']
src_doc = db_api.document_get(schema=src_schema, name=src_name)
src_secret = utils.jsonpath_parse(src_doc['data'], src_path)
# Do something with the extracted secret from the source document.
"""
if jsonpath.startswith('.'):
jsonpath = '$' + jsonpath
p = jsonpath_ng.parse(jsonpath)
matches = p.find(document)
matches = p.find(data)
if matches:
return matches[0].value
def jsonpath_replace(data, value, jsonpath, pattern=None):
"""Update value in ``data`` at the path specified by ``jsonpath``.
If the nested path corresponding to ``jsonpath`` isn't found in ``data``,
the path is created as an empty ``{}`` for each sub-path along the
``jsonpath``.
:param data: The `data` section of a document.
:param value: The new value for ``data[jsonpath]``.
:param jsonpath: A multi-part key that references a nested path in
``data``.
:param pattern: A regular expression pattern.
:returns: Updated value at ``data[jsonpath]``.
:raises: MissingDocumentPattern if ``pattern`` is not None and
``data[jsonpath]`` doesn't exist.
Example::
doc = {
'data': {
'some_url': http://admin:INSERT_PASSWORD_HERE@svc-name:8080/v1
}
}
secret = 'super-duper-secret'
path = '$.some_url'
pattern = 'INSERT_[A-Z]+_HERE'
replaced_data = utils.jsonpath_replace(
doc['data'], secret, path, pattern)
# The returned URL will look like:
# http://admin:super-duper-secret@svc-name:8080/v1
doc['data'].update(replaced_data)
"""
data = data.copy()
if jsonpath.startswith('.'):
jsonpath = '$' + jsonpath
def _do_replace():
p = jsonpath_ng.parse(jsonpath)
p_to_change = p.find(data)
if p_to_change:
_value = value
if pattern:
to_replace = p_to_change[0].value
# value represents the value to inject into to_replace that
# matches the pattern.
try:
_value = re.sub(pattern, value, to_replace)
except TypeError:
_value = None
return p.update(data, _value)
result = _do_replace()
if result:
return result
# A pattern requires us to look up the data located at data[jsonpath]
# and then figure out what re.match(data[jsonpath], pattern) is (in
# pseudocode). But raise an exception in case the path isn't present in the
# data and a pattern has been provided since it is impossible to do the
# look up.
if pattern:
raise errors.MissingDocumentPattern(
data=data, path=jsonpath, pattern=pattern)
# However, Deckhand should be smart enough to create the nested keys in the
# data if they don't exist and a pattern isn't required.
d = data
for path in jsonpath.split('.')[1:]:
if path not in d:
d.setdefault(path, {})
d = d.get(path)
return _do_replace()

2
doc/design.md
View File

@ -290,7 +290,7 @@ layer example above, which includes `global`, `region` and `site` layers, a
document in the `region` layer could insert data from a document in the
`site` layer.
Here is a sample set of documents demonstrating subistution:
Here is a sample set of documents demonstrating substitution:
```yaml
---

29
etc/deckhand/policy.yaml.sample
View File

@ -7,10 +7,6 @@
# revision history, whereby the target revision's documents are re-
# created for
# the new revision.
#
# Conditionally enforced for the endpoints below if the any of the
# documents in
# the request body have a `metadata.storagePolicy` of "cleartext".
# PUT /api/v1.0/bucket/{bucket_name}/documents
# POST /api/v1.0/rollback/{target_revision_id}
#"deckhand:create_cleartext_documents": "rule:admin_api"
@ -22,9 +18,11 @@
# the new
# revision.
#
# Only enforced after ``create_cleartext_documents`` passes.
#
# Conditionally enforced for the endpoints below if the any of the
# documents in
# the request body have a `metadata.storagePolicy` of "encrypted".
# the request body have a ``metadata.storagePolicy`` of "encrypted".
# PUT /api/v1.0/bucket/{bucket_name}/documents
# POST /api/v1.0/rollback/{target_revision_id}
#"deckhand:create_encrypted_documents": "rule:admin_api"
@ -33,31 +31,28 @@
# substitution applied) as well as fully layered and substituted
# concrete
# documents.
#
# Conditionally enforced for the endpoints below if the any of the
# documents in
# the request body have a `metadata.storagePolicy` of "cleartext". If
# policy
# enforcement fails, cleartext documents are omitted.
# GET api/v1.0/revisions/{revision_id}/documents
# GET api/v1.0/revisions/{revision_id}/rendered-documents
#"deckhand:list_cleartext_documents": "rule:admin_api"
# List cleartext documents for a revision (with no layering or
# List encrypted documents for a revision (with no layering or
# substitution applied) as well as fully layered and substituted
# concrete
# documents.
#
# Conditionally enforced for the endpoints below if the any of the
# documents in
# the request body have a `metadata.storagePolicy` of "encrypted". If
# Only enforced after ``list_cleartext_documents`` passes.
#
# Conditionally enforced for the endpoints below if any of the
# documents in the
# request body have a ``metadata.storagePolicy`` of "encrypted". If
# policy
# enforcement fails, encrypted documents are omitted.
# enforcement fails, encrypted documents are exluded from the
# response.
# GET api/v1.0/revisions/{revision_id}/documents
# GET api/v1.0/revisions/{revision_id}/rendered-documents
#"deckhand:list_encrypted_documents": "rule:admin_api"
# Show details for a revision tag.
# Show details for a revision.
# GET /api/v1.0/revisions/{revision_id}
#"deckhand:show_revision": "rule:admin_api"

8
releasenotes/notes/secret-substitution-6eff2c93bf11d82e.yaml Normal file
View File

@ -0,0 +1,8 @@
---
features:
- |
Deckhand now supports secret substitution for documents. The endpoint
``GET revisions/{revision_id}/rendered-documents`` has been added to
Deckhand, which allows the possibility of listing fully substituted
documents. Only documents with ``metadata.substitutions`` field undergo
secret substitution dynamically.