Merge "Redacts Raw Documents"
This commit is contained in:
Zuul
Gerrit Code Review
commit
7d697012fc
|
|
@ -15,6 +15,7 @@
|
|||
import collections
|
||||
import re
|
||||
|
||||
import hashlib
|
||||
from oslo_serialization import jsonutils as json
|
||||
from oslo_utils import uuidutils
|
||||
import six
|
||||
|
|
@ -171,6 +172,11 @@ class DocumentDict(dict):
|
|||
|
||||
return [DocumentDict(d) for d in documents]
|
||||
|
||||
@classmethod
|
||||
def redact(cls, input):
|
||||
return hashlib.sha256(json.dumps(input)
|
||||
.encode('utf-8')).hexdigest()
|
||||
|
||||
|
||||
def document_dict_representer(dumper, data):
|
||||
return dumper.represent_mapping('tag:yaml.org,2002:map', dict(data))
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ import jsonpath_ng
|
|||
from oslo_log import log as logging
|
||||
import six
|
||||
|
||||
from deckhand.common.document import DocumentDict as document_dict
|
||||
from deckhand.conf import config
|
||||
from deckhand import errors
|
||||
|
||||
|
|
@ -381,3 +382,27 @@ def deepfilter(dct, **filters):
|
|||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def redact_document(document):
|
||||
d = _to_document(document)
|
||||
if d.is_encrypted:
|
||||
document['data'] = document_dict.redact(d.data)
|
||||
if d.substitutions:
|
||||
subs = d.substitutions
|
||||
for s in subs:
|
||||
s['src']['path'] = document_dict.redact(s['src']['path'])
|
||||
s['dest']['path'] = document_dict.redact(s['dest']['path'])
|
||||
document['metadata']['substitutions'] = subs
|
||||
return document
|
||||
|
||||
|
||||
def redact_documents(documents):
|
||||
return [redact_document(d) for d in documents]
|
||||
|
||||
|
||||
def _to_document(document):
|
||||
clazz = document_dict
|
||||
if not isinstance(document, clazz):
|
||||
document = clazz(document)
|
||||
return document
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ class RevisionDocumentsResource(api_base.BaseResource):
|
|||
@common.sanitize_params([
|
||||
'schema', 'metadata.name', 'metadata.layeringDefinition.abstract',
|
||||
'metadata.layeringDefinition.layer', 'metadata.label',
|
||||
'status.bucket', 'order', 'sort', 'limit'])
|
||||
'status.bucket', 'order', 'sort', 'limit', 'cleartext-secrets'])
|
||||
def on_get(self, req, resp, sanitized_params, revision_id):
|
||||
"""Returns all documents for a `revision_id`.
|
||||
|
||||
|
|
@ -54,6 +54,7 @@ class RevisionDocumentsResource(api_base.BaseResource):
|
|||
order_by = sanitized_params.pop('order', None)
|
||||
sort_by = sanitized_params.pop('sort', None)
|
||||
limit = sanitized_params.pop('limit', None)
|
||||
cleartext_secrets = sanitized_params.pop('cleartext-secrets', None)
|
||||
|
||||
filters = sanitized_params.copy()
|
||||
filters['metadata.storagePolicy'] = ['cleartext']
|
||||
|
|
@ -68,6 +69,9 @@ class RevisionDocumentsResource(api_base.BaseResource):
|
|||
LOG.exception(six.text_type(e))
|
||||
raise falcon.HTTPNotFound(description=e.format_message())
|
||||
|
||||
if cleartext_secrets not in [True, 'true', 'True']:
|
||||
documents = utils.redact_documents(documents)
|
||||
|
||||
# Sorts by creation date by default.
|
||||
documents = utils.multisort(documents, sort_by, order_by)
|
||||
if limit is not None:
|
||||
|
|
|
|||
|
|
@ -84,7 +84,7 @@ Only enforced after ``list_cleartext_documents`` passes.
|
|||
|
||||
Conditionally enforced for the endpoints below if any of the documents in the
|
||||
request body have a ``metadata.storagePolicy`` of "encrypted". If policy
|
||||
enforcement fails, encrypted documents are exluded from the response.""",
|
||||
enforcement fails, encrypted documents are excluded from the response.""",
|
||||
[
|
||||
{
|
||||
'method': 'GET',
|
||||
|
|
|
|||
|
|
@ -75,6 +75,8 @@ tests:
|
|||
content-type: application/x-yaml
|
||||
response_headers:
|
||||
content-type: application/x-yaml
|
||||
query_parameters:
|
||||
cleartext-secrets: 'true'
|
||||
response_multidoc_jsonpaths:
|
||||
$.`len`: 1
|
||||
# NOTE(felipemonteiro): jsonpath-rw-ext uses a 1 character separator (rather than allowing a string)
|
||||
|
|
|
|||
|
|
@ -167,6 +167,7 @@ tests:
|
|||
content-type: application/x-yaml
|
||||
query_parameters:
|
||||
metadata.name: armada-doc
|
||||
cleartext-secrets: 'true'
|
||||
response_multidoc_jsonpaths:
|
||||
$.`len`: 1
|
||||
$.[0].data.`split(:, 0, 1)` + "://" + $.[0].data.`split(/, 2, 3)`: $ENVIRON['TEST_BARBICAN_URL']
|
||||
|
|
|
|||
|
|
@ -74,6 +74,7 @@ tests:
|
|||
status: 200
|
||||
query_parameters:
|
||||
metadata.name: example-armada-cert
|
||||
cleartext-secrets: 'true'
|
||||
response_multidoc_jsonpaths:
|
||||
$.`len`: 1
|
||||
# NOTE(felipemonteiro): jsonpath-rw-ext uses a 1 character separator (rather than allowing a string)
|
||||
|
|
|
|||
|
|
@ -180,6 +180,7 @@ tests:
|
|||
- example-passphrase
|
||||
- example-private-key
|
||||
- example-public-key
|
||||
cleartext-secrets: 'true'
|
||||
response_multidoc_jsonpaths:
|
||||
$.`len`: 7
|
||||
# NOTE(felipemonteiro): jsonpath-rw-ext uses a 1 character separator (rather than allowing a string)
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ import yaml
|
|||
|
||||
import mock
|
||||
|
||||
from deckhand.common.document import DocumentDict as document_dict
|
||||
from deckhand.engine import secrets_manager
|
||||
from deckhand import factories
|
||||
from deckhand.tests.unit.control import base as test_base
|
||||
|
|
@ -102,6 +103,93 @@ data:
|
|||
self.assertEqual(2, len(retrieved_documents))
|
||||
self.assertEqual(expected_data_section, retrieved_documents[0]['data'])
|
||||
|
||||
def _setup_payload(self):
|
||||
data = '12345'
|
||||
sub_src = '.source1'
|
||||
sub_dest = '.destination2'
|
||||
secrets_factory = factories.DocumentSecretFactory()
|
||||
payload = [secrets_factory.gen_test('Certificate', 'encrypted')]
|
||||
payload[0]['data'] = data
|
||||
sub1 = {'src': {'schema': 'pegleg/SoftwareVersions/v1', 'name': 'sub1',
|
||||
'path': sub_src}, 'dest': {'path': '.destination1'}}
|
||||
sub2 = {'src': {'schema': 'pegleg/SoftwareVersions/v1', 'name': 'sub2',
|
||||
'path': '.source2'}, 'dest': {'path': sub_dest}}
|
||||
payload[0]['metadata']['substitutions'] = [sub1, sub2]
|
||||
return payload, data, sub_src, sub_dest
|
||||
|
||||
def test_list_encrypted_revision_documents_redacted(self):
|
||||
rules = {'deckhand:list_cleartext_documents': '@',
|
||||
'deckhand:list_encrypted_documents': '@',
|
||||
'deckhand:create_cleartext_documents': '@',
|
||||
'deckhand:create_encrypted_documents': '@'}
|
||||
self.policy.set_rules(rules)
|
||||
|
||||
# Create a document for a bucket.
|
||||
payload, data, sub_src, sub_dest = self._setup_payload()
|
||||
|
||||
with mock.patch.object(secrets_manager, 'SecretsManager',
|
||||
autospec=True) as mock_secrets_mgr:
|
||||
mock_secrets_mgr.create.return_value = payload[0]['data']
|
||||
resp = self.app.simulate_put(
|
||||
'/api/v1.0/buckets/mop/documents',
|
||||
headers={'Content-Type': 'application/x-yaml'},
|
||||
body=yaml.safe_dump_all(payload))
|
||||
self.assertEqual(200, resp.status_code)
|
||||
revision_id = list(yaml.safe_load_all(resp.text))[0]['status'][
|
||||
'revision']
|
||||
|
||||
# Verify that the created document was redacted.
|
||||
redacted_data = document_dict.redact(data)
|
||||
redacted_sub_src = document_dict.redact(sub_src)
|
||||
redacted_sub_dest = document_dict.redact(sub_dest)
|
||||
resp = self.app.simulate_get(
|
||||
'/api/v1.0/revisions/%s/documents' % revision_id,
|
||||
headers={'Content-Type': 'application/x-yaml'},
|
||||
query_string='cleartext-secrets=false')
|
||||
|
||||
self.assertEqual(200, resp.status_code)
|
||||
self.assertNotEqual(list(yaml.safe_load_all(resp.text)), [])
|
||||
response_yaml = list(yaml.safe_load_all(resp.text))
|
||||
self.assertEqual(redacted_data, response_yaml[0]['data'])
|
||||
subs = response_yaml[0]['metadata']['substitutions']
|
||||
self.assertEqual(redacted_sub_src, subs[0]['src']['path'])
|
||||
self.assertEqual(redacted_sub_dest, subs[1]['dest']['path'])
|
||||
|
||||
def test_list_encrypted_revision_documents_cleartext_secrets(self):
|
||||
rules = {'deckhand:list_cleartext_documents': '@',
|
||||
'deckhand:list_encrypted_documents': '@',
|
||||
'deckhand:create_cleartext_documents': '@',
|
||||
'deckhand:create_encrypted_documents': '@'}
|
||||
self.policy.set_rules(rules)
|
||||
|
||||
# Create a document for a bucket.
|
||||
payload, data, sub_src, sub_dest = self._setup_payload()
|
||||
|
||||
with mock.patch.object(secrets_manager, 'SecretsManager',
|
||||
autospec=True) as mock_secrets_mgr:
|
||||
mock_secrets_mgr.create.return_value = payload[0]['data']
|
||||
resp = self.app.simulate_put(
|
||||
'/api/v1.0/buckets/mop/documents',
|
||||
headers={'Content-Type': 'application/x-yaml'},
|
||||
body=yaml.safe_dump_all(payload))
|
||||
self.assertEqual(200, resp.status_code)
|
||||
revision_id = list(yaml.safe_load_all(resp.text))[0]['status'][
|
||||
'revision']
|
||||
|
||||
# Verify that the created document was not redacted.
|
||||
resp = self.app.simulate_get(
|
||||
'/api/v1.0/revisions/%s/documents' % revision_id,
|
||||
headers={'Content-Type': 'application/x-yaml'},
|
||||
query_string='cleartext-secrets=true')
|
||||
|
||||
self.assertEqual(200, resp.status_code)
|
||||
self.assertNotEqual(list(yaml.safe_load_all(resp.text)), [])
|
||||
response_yaml = list(yaml.safe_load_all(resp.text))
|
||||
self.assertEqual(data, response_yaml[0]['data'])
|
||||
subs = response_yaml[0]['metadata']['substitutions']
|
||||
self.assertEqual(sub_src, subs[0]['src']['path'])
|
||||
self.assertEqual(sub_dest, subs[1]['dest']['path'])
|
||||
|
||||
|
||||
class TestRevisionDocumentsControllerNegativeRBAC(
|
||||
test_base.BaseControllerTest):
|
||||
|
|
|
|||
|
|
@ -156,7 +156,7 @@ class RealPolicyFixture(fixtures.Fixture):
|
|||
def enforce_policy_and_remember_actual_rules(
|
||||
action, *a, **k):
|
||||
self.actual_policy_actions.append(action)
|
||||
_do_enforce_rbac(action, *a, **k)
|
||||
return _do_enforce_rbac(action, *a, **k)
|
||||
|
||||
mock_do_enforce_rbac = mock.patch.object(
|
||||
deckhand.policy, '_do_enforce_rbac', autospec=True).start()
|
||||
|
|
|
|||
|
|
@ -88,6 +88,9 @@ Supported query string parameters:
|
|||
descending order.
|
||||
* ``limit`` - int, optional - Controls number of documents returned by this
|
||||
endpoint.
|
||||
* ``cleartext-secrets`` - boolean, optional - Determines if data and substitutions
|
||||
paths should be redacted (sha256) if a user has access to encrypted files.
|
||||
Default is to redact the values.
|
||||
|
||||
GET ``/revisions/{revision_id}/rendered-documents``
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@
|
|||
# documents in the
|
||||
# request body have a ``metadata.storagePolicy`` of "encrypted". If
|
||||
# policy
|
||||
# enforcement fails, encrypted documents are exluded from the
|
||||
# enforcement fails, encrypted documents are excluded from the
|
||||
# response.
|
||||
# GET api/v1.0/revisions/{revision_id}/documents
|
||||
# GET api/v1.0/revisions/{revision_id}/rendered-documents
|
||||
|
|
|
|||
Loading…
Reference in New Issue