diff --git a/deckhand/common/document.py b/deckhand/common/document.py index 57ef904d..c4d8f996 100644 --- a/deckhand/common/document.py +++ b/deckhand/common/document.py @@ -15,6 +15,7 @@ import collections import re +import hashlib from oslo_serialization import jsonutils as json from oslo_utils import uuidutils import six @@ -171,6 +172,11 @@ class DocumentDict(dict): return [DocumentDict(d) for d in documents] + @classmethod + def redact(cls, input): + return hashlib.sha256(json.dumps(input) + .encode('utf-8')).hexdigest() + def document_dict_representer(dumper, data): return dumper.represent_mapping('tag:yaml.org,2002:map', dict(data)) diff --git a/deckhand/common/utils.py b/deckhand/common/utils.py index 8b6be04f..8e68e110 100644 --- a/deckhand/common/utils.py +++ b/deckhand/common/utils.py @@ -23,6 +23,7 @@ import jsonpath_ng from oslo_log import log as logging import six +from deckhand.common.document import DocumentDict as document_dict from deckhand.conf import config from deckhand import errors @@ -381,3 +382,27 @@ def deepfilter(dct, **filters): return False return True + + +def redact_document(document): + d = _to_document(document) + if d.is_encrypted: + document['data'] = document_dict.redact(d.data) + if d.substitutions: + subs = d.substitutions + for s in subs: + s['src']['path'] = document_dict.redact(s['src']['path']) + s['dest']['path'] = document_dict.redact(s['dest']['path']) + document['metadata']['substitutions'] = subs + return document + + +def redact_documents(documents): + return [redact_document(d) for d in documents] + + +def _to_document(document): + clazz = document_dict + if not isinstance(document, clazz): + document = clazz(document) + return document diff --git a/deckhand/control/revision_documents.py b/deckhand/control/revision_documents.py index 78e5e7a8..db39097c 100644 --- a/deckhand/control/revision_documents.py +++ b/deckhand/control/revision_documents.py @@ -39,7 +39,7 @@ class RevisionDocumentsResource(api_base.BaseResource): @common.sanitize_params([ 'schema', 'metadata.name', 'metadata.layeringDefinition.abstract', 'metadata.layeringDefinition.layer', 'metadata.label', - 'status.bucket', 'order', 'sort', 'limit']) + 'status.bucket', 'order', 'sort', 'limit', 'cleartext-secrets']) def on_get(self, req, resp, sanitized_params, revision_id): """Returns all documents for a `revision_id`. @@ -54,6 +54,7 @@ class RevisionDocumentsResource(api_base.BaseResource): order_by = sanitized_params.pop('order', None) sort_by = sanitized_params.pop('sort', None) limit = sanitized_params.pop('limit', None) + cleartext_secrets = sanitized_params.pop('cleartext-secrets', None) filters = sanitized_params.copy() filters['metadata.storagePolicy'] = ['cleartext'] @@ -68,6 +69,9 @@ class RevisionDocumentsResource(api_base.BaseResource): LOG.exception(six.text_type(e)) raise falcon.HTTPNotFound(description=e.format_message()) + if cleartext_secrets not in [True, 'true', 'True']: + documents = utils.redact_documents(documents) + # Sorts by creation date by default. documents = utils.multisort(documents, sort_by, order_by) if limit is not None: diff --git a/deckhand/policies/document.py b/deckhand/policies/document.py index ee1c47f8..b5422f75 100644 --- a/deckhand/policies/document.py +++ b/deckhand/policies/document.py @@ -84,7 +84,7 @@ Only enforced after ``list_cleartext_documents`` passes. Conditionally enforced for the endpoints below if any of the documents in the request body have a ``metadata.storagePolicy`` of "encrypted". If policy -enforcement fails, encrypted documents are exluded from the response.""", +enforcement fails, encrypted documents are excluded from the response.""", [ { 'method': 'GET', diff --git a/deckhand/tests/integration/gabbits/document-crud-secret.yaml b/deckhand/tests/integration/gabbits/document-crud-secret.yaml index 008c52cb..5fd6be20 100644 --- a/deckhand/tests/integration/gabbits/document-crud-secret.yaml +++ b/deckhand/tests/integration/gabbits/document-crud-secret.yaml @@ -75,6 +75,8 @@ tests: content-type: application/x-yaml response_headers: content-type: application/x-yaml + query_parameters: + cleartext-secrets: 'true' response_multidoc_jsonpaths: $.`len`: 1 # NOTE(felipemonteiro): jsonpath-rw-ext uses a 1 character separator (rather than allowing a string) diff --git a/deckhand/tests/integration/gabbits/document-render-secret-edge-cases.yaml b/deckhand/tests/integration/gabbits/document-render-secret-edge-cases.yaml index 2b5aac0b..c1f63792 100644 --- a/deckhand/tests/integration/gabbits/document-render-secret-edge-cases.yaml +++ b/deckhand/tests/integration/gabbits/document-render-secret-edge-cases.yaml @@ -167,6 +167,7 @@ tests: content-type: application/x-yaml query_parameters: metadata.name: armada-doc + cleartext-secrets: 'true' response_multidoc_jsonpaths: $.`len`: 1 $.[0].data.`split(:, 0, 1)` + "://" + $.[0].data.`split(/, 2, 3)`: $ENVIRON['TEST_BARBICAN_URL'] diff --git a/deckhand/tests/integration/gabbits/document-substitution-secret-generic.yaml b/deckhand/tests/integration/gabbits/document-substitution-secret-generic.yaml index cc6f775a..9e282bba 100644 --- a/deckhand/tests/integration/gabbits/document-substitution-secret-generic.yaml +++ b/deckhand/tests/integration/gabbits/document-substitution-secret-generic.yaml @@ -74,6 +74,7 @@ tests: status: 200 query_parameters: metadata.name: example-armada-cert + cleartext-secrets: 'true' response_multidoc_jsonpaths: $.`len`: 1 # NOTE(felipemonteiro): jsonpath-rw-ext uses a 1 character separator (rather than allowing a string) diff --git a/deckhand/tests/integration/gabbits/document-substitution-secret.yaml b/deckhand/tests/integration/gabbits/document-substitution-secret.yaml index f0779062..e7b81382 100644 --- a/deckhand/tests/integration/gabbits/document-substitution-secret.yaml +++ b/deckhand/tests/integration/gabbits/document-substitution-secret.yaml @@ -180,6 +180,7 @@ tests: - example-passphrase - example-private-key - example-public-key + cleartext-secrets: 'true' response_multidoc_jsonpaths: $.`len`: 7 # NOTE(felipemonteiro): jsonpath-rw-ext uses a 1 character separator (rather than allowing a string) diff --git a/deckhand/tests/unit/control/test_revision_documents_controller.py b/deckhand/tests/unit/control/test_revision_documents_controller.py index 85911efc..f8d0343d 100644 --- a/deckhand/tests/unit/control/test_revision_documents_controller.py +++ b/deckhand/tests/unit/control/test_revision_documents_controller.py @@ -16,6 +16,7 @@ import yaml import mock +from deckhand.common.document import DocumentDict as document_dict from deckhand.engine import secrets_manager from deckhand import factories from deckhand.tests.unit.control import base as test_base @@ -102,6 +103,93 @@ data: self.assertEqual(2, len(retrieved_documents)) self.assertEqual(expected_data_section, retrieved_documents[0]['data']) + def _setup_payload(self): + data = '12345' + sub_src = '.source1' + sub_dest = '.destination2' + secrets_factory = factories.DocumentSecretFactory() + payload = [secrets_factory.gen_test('Certificate', 'encrypted')] + payload[0]['data'] = data + sub1 = {'src': {'schema': 'pegleg/SoftwareVersions/v1', 'name': 'sub1', + 'path': sub_src}, 'dest': {'path': '.destination1'}} + sub2 = {'src': {'schema': 'pegleg/SoftwareVersions/v1', 'name': 'sub2', + 'path': '.source2'}, 'dest': {'path': sub_dest}} + payload[0]['metadata']['substitutions'] = [sub1, sub2] + return payload, data, sub_src, sub_dest + + def test_list_encrypted_revision_documents_redacted(self): + rules = {'deckhand:list_cleartext_documents': '@', + 'deckhand:list_encrypted_documents': '@', + 'deckhand:create_cleartext_documents': '@', + 'deckhand:create_encrypted_documents': '@'} + self.policy.set_rules(rules) + + # Create a document for a bucket. + payload, data, sub_src, sub_dest = self._setup_payload() + + with mock.patch.object(secrets_manager, 'SecretsManager', + autospec=True) as mock_secrets_mgr: + mock_secrets_mgr.create.return_value = payload[0]['data'] + resp = self.app.simulate_put( + '/api/v1.0/buckets/mop/documents', + headers={'Content-Type': 'application/x-yaml'}, + body=yaml.safe_dump_all(payload)) + self.assertEqual(200, resp.status_code) + revision_id = list(yaml.safe_load_all(resp.text))[0]['status'][ + 'revision'] + + # Verify that the created document was redacted. + redacted_data = document_dict.redact(data) + redacted_sub_src = document_dict.redact(sub_src) + redacted_sub_dest = document_dict.redact(sub_dest) + resp = self.app.simulate_get( + '/api/v1.0/revisions/%s/documents' % revision_id, + headers={'Content-Type': 'application/x-yaml'}, + query_string='cleartext-secrets=false') + + self.assertEqual(200, resp.status_code) + self.assertNotEqual(list(yaml.safe_load_all(resp.text)), []) + response_yaml = list(yaml.safe_load_all(resp.text)) + self.assertEqual(redacted_data, response_yaml[0]['data']) + subs = response_yaml[0]['metadata']['substitutions'] + self.assertEqual(redacted_sub_src, subs[0]['src']['path']) + self.assertEqual(redacted_sub_dest, subs[1]['dest']['path']) + + def test_list_encrypted_revision_documents_cleartext_secrets(self): + rules = {'deckhand:list_cleartext_documents': '@', + 'deckhand:list_encrypted_documents': '@', + 'deckhand:create_cleartext_documents': '@', + 'deckhand:create_encrypted_documents': '@'} + self.policy.set_rules(rules) + + # Create a document for a bucket. + payload, data, sub_src, sub_dest = self._setup_payload() + + with mock.patch.object(secrets_manager, 'SecretsManager', + autospec=True) as mock_secrets_mgr: + mock_secrets_mgr.create.return_value = payload[0]['data'] + resp = self.app.simulate_put( + '/api/v1.0/buckets/mop/documents', + headers={'Content-Type': 'application/x-yaml'}, + body=yaml.safe_dump_all(payload)) + self.assertEqual(200, resp.status_code) + revision_id = list(yaml.safe_load_all(resp.text))[0]['status'][ + 'revision'] + + # Verify that the created document was not redacted. + resp = self.app.simulate_get( + '/api/v1.0/revisions/%s/documents' % revision_id, + headers={'Content-Type': 'application/x-yaml'}, + query_string='cleartext-secrets=true') + + self.assertEqual(200, resp.status_code) + self.assertNotEqual(list(yaml.safe_load_all(resp.text)), []) + response_yaml = list(yaml.safe_load_all(resp.text)) + self.assertEqual(data, response_yaml[0]['data']) + subs = response_yaml[0]['metadata']['substitutions'] + self.assertEqual(sub_src, subs[0]['src']['path']) + self.assertEqual(sub_dest, subs[1]['dest']['path']) + class TestRevisionDocumentsControllerNegativeRBAC( test_base.BaseControllerTest): diff --git a/deckhand/tests/unit/fixtures.py b/deckhand/tests/unit/fixtures.py index 41bb9082..45e162bd 100644 --- a/deckhand/tests/unit/fixtures.py +++ b/deckhand/tests/unit/fixtures.py @@ -156,7 +156,7 @@ class RealPolicyFixture(fixtures.Fixture): def enforce_policy_and_remember_actual_rules( action, *a, **k): self.actual_policy_actions.append(action) - _do_enforce_rbac(action, *a, **k) + return _do_enforce_rbac(action, *a, **k) mock_do_enforce_rbac = mock.patch.object( deckhand.policy, '_do_enforce_rbac', autospec=True).start() diff --git a/doc/source/operators/api_ref.rst b/doc/source/operators/api_ref.rst index 4f22c4a7..5c10b325 100644 --- a/doc/source/operators/api_ref.rst +++ b/doc/source/operators/api_ref.rst @@ -88,6 +88,9 @@ Supported query string parameters: descending order. * ``limit`` - int, optional - Controls number of documents returned by this endpoint. +* ``cleartext-secrets`` - boolean, optional - Determines if data and substitutions + paths should be redacted (sha256) if a user has access to encrypted files. + Default is to redact the values. GET ``/revisions/{revision_id}/rendered-documents`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/etc/deckhand/policy.yaml.sample b/etc/deckhand/policy.yaml.sample index f95a0782..43a9dc1e 100644 --- a/etc/deckhand/policy.yaml.sample +++ b/etc/deckhand/policy.yaml.sample @@ -46,7 +46,7 @@ # documents in the # request body have a ``metadata.storagePolicy`` of "encrypted". If # policy -# enforcement fails, encrypted documents are exluded from the +# enforcement fails, encrypted documents are excluded from the # response. # GET api/v1.0/revisions/{revision_id}/documents # GET api/v1.0/revisions/{revision_id}/rendered-documents