From 11eeb69f4442619303982b22a4116003ec67321a Mon Sep 17 00:00:00 2001 From: Felipe Monteiro Date: Fri, 22 Jun 2018 22:38:44 -0400 Subject: [PATCH] integration tests: Add Barbican validation/assertions This patchset adds Barbican validation/assertions to integration tests by querying the Barbican API server where appropriate and validating that the expected data is returned in order to sanity-check the integration scenarios further. Change-Id: If5d30712b289f09ac9712ee205673be4150cda16 --- deckhand/tests/common/test_gabbi.py | 9 +-- deckhand/tests/deckhand.conf.test | 2 +- .../layering-and-substitution-dag-sample.yaml | 2 +- .../gabbits/document-crud-secret.yaml | 42 +++++++++-- .../document-render-secret-edge-cases.yaml | 72 ++++++++++++++++--- .../gabbits/document-render-secret.yaml | 2 +- .../document-substitution-secret-generic.yaml | 18 ++++- .../gabbits/document-substitution-secret.yaml | 58 +++++++++++++-- .../control/test_validations_controller.py | 6 +- deckhand/tests/unit/db/base.py | 2 +- .../unit/engine/test_document_layering.py | 2 +- ...ment_layering_and_substitution_negative.py | 4 +- .../tests/unit/engine/test_secrets_manager.py | 4 +- 13 files changed, 185 insertions(+), 38 deletions(-) diff --git a/deckhand/tests/common/test_gabbi.py b/deckhand/tests/common/test_gabbi.py index 9dac1237..647fc780 100644 --- a/deckhand/tests/common/test_gabbi.py +++ b/deckhand/tests/common/test_gabbi.py @@ -102,10 +102,11 @@ def load_tests(loader, tests, pattern): loader, host='localhost', url=os.environ.get('DECKHAND_TEST_URL', '127.0.0.1:9000'), - # NOTE(fmontei): When there are multiple handlers listed that accept - # the same content-type, the one that is earliest in the list will be - # used. Thus, we cannot specify multiple content handlers for handling - # list/dictionary responses from the server using different handlers. + # NOTE(felipemonteiro): When there are multiple handlers listed that + # accept the same content-type, the one that is earliest in the list + # will be used. Thus, we cannot specify multiple content handlers for + # handling list/dictionary responses from the server using different + # handlers. content_handlers=[ core.StringResponseHandler, # For parsing text/plain jsonhandler.JSONHandler, # For parsing application/json diff --git a/deckhand/tests/deckhand.conf.test b/deckhand/tests/deckhand.conf.test index 782d14fd..ed82683b 100644 --- a/deckhand/tests/deckhand.conf.test +++ b/deckhand/tests/deckhand.conf.test @@ -13,7 +13,7 @@ policy_file = policy.yaml connection = ${AIRSHIP_DECKHAND_DATABASE_URL} [keystone_authtoken] -# NOTE(fmontei): Values taken from clouds.yaml. Values only used for +# NOTE(felipemonteiro): Values taken from clouds.yaml. Values only used for # integration testing. # # clouds.yaml (snippet): diff --git a/deckhand/tests/functional/gabbits/resources/layering-and-substitution-dag-sample.yaml b/deckhand/tests/functional/gabbits/resources/layering-and-substitution-dag-sample.yaml index 85884070..6118df90 100644 --- a/deckhand/tests/functional/gabbits/resources/layering-and-substitution-dag-sample.yaml +++ b/deckhand/tests/functional/gabbits/resources/layering-and-substitution-dag-sample.yaml @@ -28,7 +28,7 @@ metadata: storagePolicy: cleartext data: my-secret-password --- -# NOTE(fmontei): The documents below are included in reverse order with +# NOTE(felipemonteiro): The documents below are included in reverse order with # respect to their substitution dependency hierarchy in order to verify # that the dependency chain is correctly resolved in the code. schema: armada/Chart/v1 diff --git a/deckhand/tests/integration/gabbits/document-crud-secret.yaml b/deckhand/tests/integration/gabbits/document-crud-secret.yaml index 331f9700..008c52cb 100644 --- a/deckhand/tests/integration/gabbits/document-crud-secret.yaml +++ b/deckhand/tests/integration/gabbits/document-crud-secret.yaml @@ -6,10 +6,7 @@ defaults: request_headers: - content-type: application/x-yaml X-Auth-Token: $ENVIRON['TEST_AUTH_TOKEN'] - response_headers: - content-type: application/x-yaml verbose: true tests: @@ -17,12 +14,18 @@ tests: desc: Begin testing from known state. DELETE: /api/v1.0/revisions status: 204 + request_headers: + content-type: application/x-yaml response_headers: null - name: create_encrypted_passphrase desc: Create passphrase with storagePolicy=encrypted PUT: /api/v1.0/buckets/secret/documents status: 200 + request_headers: + content-type: application/x-yaml + response_headers: + content-type: application/x-yaml data: |- --- schema: deckhand/Passphrase/v1 @@ -37,16 +40,43 @@ tests: ... response_multidoc_jsonpaths: $.`len`: 1 - # NOTE(fmontei): jsonpath-rw-ext uses a 1 character separator (rather than allowing a string) + # NOTE(felipemonteiro): jsonpath-rw-ext uses a 1 character separator (rather than allowing a string) # leading to this nastiness: $.[0].data.`split(:, 0, 1)` + "://" + $.[0].data.`split(/, 2, 3)`: $ENVIRON['TEST_BARBICAN_URL'] + - name: validate_secret_exists_in_barbican + desc: Validate that the secret ref exists in Barbican. + GET: $ENVIRON['TEST_BARBICAN_URL']/v1/secrets/$RESPONSE['$.[0].data.`split(/, 5, -1)`'] + status: 200 + request_headers: + content-type: application/json + response_headers: + content-type: application/json; charset=UTF-8 + response_json_paths: + $.status: ACTIVE + $.name: my-passphrase + + - name: validate_secret_payload_matches_in_barbican + desc: Validate that the secret itself matches in Barbican. + GET: $ENVIRON['TEST_BARBICAN_URL']/v1/secrets/$HISTORY['create_encrypted_passphrase'].$RESPONSE['$.[0].data.`split(/, 5, -1)`']/payload + status: 200 + request_headers: + content-type: application/json + response_headers: + content-type: application/octet-stream; charset=UTF-8 + response_strings: + - not-a-real-password + - name: verify_revision_documents_returns_secret_ref desc: Verify that the documents for the created revision returns the secret ref. - GET: /api/v1.0/revisions/$RESPONSE['$.[0].status.revision']/documents + GET: /api/v1.0/revisions/$HISTORY['create_encrypted_passphrase'].$RESPONSE['$.[0].status.revision']/documents status: 200 + request_headers: + content-type: application/x-yaml + response_headers: + content-type: application/x-yaml response_multidoc_jsonpaths: $.`len`: 1 - # NOTE(fmontei): jsonpath-rw-ext uses a 1 character separator (rather than allowing a string) + # NOTE(felipemonteiro): jsonpath-rw-ext uses a 1 character separator (rather than allowing a string) # leading to this nastiness: $.[0].data.`split(:, 0, 1)` + "://" + $.[0].data.`split(/, 2, 3)`: $ENVIRON['TEST_BARBICAN_URL'] diff --git a/deckhand/tests/integration/gabbits/document-render-secret-edge-cases.yaml b/deckhand/tests/integration/gabbits/document-render-secret-edge-cases.yaml index 72bff7ff..2b5aac0b 100644 --- a/deckhand/tests/integration/gabbits/document-render-secret-edge-cases.yaml +++ b/deckhand/tests/integration/gabbits/document-render-secret-edge-cases.yaml @@ -11,20 +11,29 @@ defaults: request_headers: - content-type: application/x-yaml X-Auth-Token: $ENVIRON['TEST_AUTH_TOKEN'] - response_headers: - content-type: application/x-yaml verbose: true tests: ### Scenario 1 ### - - name: attempt_create_encrypted_passphrase_empty_payload + - name: purge + desc: Begin testing from known state. + DELETE: /api/v1.0/revisions + status: 204 + request_headers: + content-type: application/x-yaml + response_headers: null + + - name: create_encrypted_passphrase_empty_payload_skips desc: | Attempting to create an encrypted passphrase with empty payload should avoid encryption and return the empty payload instead. PUT: /api/v1.0/buckets/secret/documents status: 200 + request_headers: + content-type: application/x-yaml + response_headers: + content-type: application/x-yaml data: |- --- schema: deckhand/LayeringPolicy/v1 @@ -50,10 +59,26 @@ tests: $.[1].metadata.name: my-passphrase $.[1].data: '' + - name: validate_no_secret_exists_in_barbican + desc: Validate that no secret was created in Barbican. + GET: $ENVIRON['TEST_BARBICAN_URL']/v1/secrets + status: 200 + request_headers: + content-type: application/json + response_headers: + content-type: application/json; charset=UTF-8 + response_json_paths: + $.secrets.`len`: 0 + $.secrets: [] + - name: verify_revision_documents_returns_same_empty_payload desc: Verify that the created document wasn't encrypted. - GET: /api/v1.0/revisions/$RESPONSE['$.[0].status.revision']/documents + GET: /api/v1.0/revisions/$HISTORY['create_encrypted_passphrase_empty_payload_skips'].$RESPONSE['$.[0].status.revision']/documents status: 200 + request_headers: + content-type: application/x-yaml + response_headers: + content-type: application/x-yaml query_parameters: metadata.name: my-passphrase response_multidoc_jsonpaths: @@ -65,8 +90,12 @@ tests: Verify that rendering the document returns the same empty payload which requires that the data be read directly from Deckhand's DB rather than Barbican. - GET: /api/v1.0/revisions/$RESPONSE['$.[0].status.revision']/rendered-documents + GET: /api/v1.0/revisions/$HISTORY['create_encrypted_passphrase_empty_payload_skips'].$RESPONSE['$.[0].status.revision']/rendered-documents status: 200 + request_headers: + content-type: application/x-yaml + response_headers: + content-type: application/x-yaml query_parameters: metadata.name: my-passphrase response_multidoc_jsonpaths: @@ -82,6 +111,10 @@ tests: successfully. PUT: /api/v1.0/buckets/secret/documents status: 200 + request_headers: + content-type: application/x-yaml + response_headers: + content-type: application/x-yaml data: |- --- schema: deckhand/LayeringPolicy/v1 @@ -107,14 +140,31 @@ tests: response_multidoc_jsonpaths: $.`len`: 2 $.[1].metadata.name: armada-doc - # NOTE(fmontei): jsonpath-rw-ext uses a 1 character separator (rather than allowing a string) + # NOTE(felipemonteiro): jsonpath-rw-ext uses a 1 character separator (rather than allowing a string) # leading to this nastiness: $.[1].data.`split(:, 0, 1)` + "://" + $.[1].data.`split(/, 2, 3)`: $ENVIRON['TEST_BARBICAN_URL'] + - name: validate_opaque_secret_exists_in_barbican + desc: Validate that the secret was created as opaque in Barbican. + GET: $ENVIRON['TEST_BARBICAN_URL']/v1/secrets/$HISTORY['create_encrypted_passphrase_with_incompatible_payload'].$RESPONSE['$.[1].data.`split(/, 5, -1)`'] + status: 200 + request_headers: + content-type: application/json + response_headers: + content-type: application/json; charset=UTF-8 + response_json_paths: + $.status: ACTIVE + $.name: armada-doc + $.secret_type: opaque + - name: verify_revision_documents_returns_barbican_ref desc: Verify that the encrypted document returns a Barbican ref. - GET: /api/v1.0/revisions/$RESPONSE['$.[0].status.revision']/documents + GET: /api/v1.0/revisions/$HISTORY['create_encrypted_passphrase_with_incompatible_payload'].$RESPONSE['$.[0].status.revision']/documents status: 200 + request_headers: + content-type: application/x-yaml + response_headers: + content-type: application/x-yaml query_parameters: metadata.name: armada-doc response_multidoc_jsonpaths: @@ -126,8 +176,12 @@ tests: Verify that rendering the document returns the original payload which means that Deckhand successfully encoded and decoded the non-compatible payload using base64 encoding. - GET: /api/v1.0/revisions/$RESPONSE['$.[0].status.revision']/rendered-documents + GET: /api/v1.0/revisions/$HISTORY['create_encrypted_passphrase_with_incompatible_payload'].$RESPONSE['$.[0].status.revision']/rendered-documents status: 200 + request_headers: + content-type: application/x-yaml + response_headers: + content-type: application/x-yaml query_parameters: metadata.name: armada-doc response_multidoc_jsonpaths: diff --git a/deckhand/tests/integration/gabbits/document-render-secret.yaml b/deckhand/tests/integration/gabbits/document-render-secret.yaml index 2aaa3bb4..ab6e0596 100644 --- a/deckhand/tests/integration/gabbits/document-render-secret.yaml +++ b/deckhand/tests/integration/gabbits/document-render-secret.yaml @@ -43,7 +43,7 @@ tests: ... response_multidoc_jsonpaths: $.`len`: 2 - # NOTE(fmontei): jsonpath-rw-ext uses a 1 character separator (rather than allowing a string) + # NOTE(felipemonteiro): jsonpath-rw-ext uses a 1 character separator (rather than allowing a string) # leading to this nastiness: $.[1].data.`split(:, 0, 1)` + "://" + $.[1].data.`split(/, 2, 3)`: $ENVIRON['TEST_BARBICAN_URL'] diff --git a/deckhand/tests/integration/gabbits/document-substitution-secret-generic.yaml b/deckhand/tests/integration/gabbits/document-substitution-secret-generic.yaml index 6bbf23e8..cc6f775a 100644 --- a/deckhand/tests/integration/gabbits/document-substitution-secret-generic.yaml +++ b/deckhand/tests/integration/gabbits/document-substitution-secret-generic.yaml @@ -76,13 +76,27 @@ tests: metadata.name: example-armada-cert response_multidoc_jsonpaths: $.`len`: 1 - # NOTE(fmontei): jsonpath-rw-ext uses a 1 character separator (rather than allowing a string) + # NOTE(felipemonteiro): jsonpath-rw-ext uses a 1 character separator (rather than allowing a string) # leading to this nastiness: $.[0].data.`split(:, 0, 1)` + "://" + $.[0].data.`split(/, 2, 3)`: $ENVIRON['TEST_BARBICAN_URL'] + - name: verify_generic_secret_created_in_barbican + desc: Validate that the generic secret gets stored with secret_type passphrase. + GET: $ENVIRON['TEST_BARBICAN_URL']/v1/secrets/$RESPONSE['$.[0].data.`split(/, 5, -1)`'] + status: 200 + request_headers: + content-type: application/json + response_headers: + content-type: application/json; charset=UTF-8 + response_json_paths: + $.status: ACTIVE + $.name: example-armada-cert + # Default type for documents with generic schema. + $.secret_type: passphrase + - name: verify_secret_payload_in_destination_document desc: Verify secret payload is injected in destination document as well as example-armada-cert. - GET: /api/v1.0/revisions/$RESPONSE['$.[0].status.revision']/rendered-documents + GET: /api/v1.0/revisions/$HISTORY['encrypt_generic_document_for_secret_substitution'].$RESPONSE['$.[0].status.revision']/rendered-documents status: 200 query_parameters: metadata.name: diff --git a/deckhand/tests/integration/gabbits/document-substitution-secret.yaml b/deckhand/tests/integration/gabbits/document-substitution-secret.yaml index 82e95fd6..f0779062 100644 --- a/deckhand/tests/integration/gabbits/document-substitution-secret.yaml +++ b/deckhand/tests/integration/gabbits/document-substitution-secret.yaml @@ -9,10 +9,7 @@ defaults: request_headers: - content-type: application/x-yaml X-Auth-Token: $ENVIRON['TEST_AUTH_TOKEN'] - response_headers: - content-type: application/x-yaml verbose: true tests: @@ -20,12 +17,18 @@ tests: desc: Begin testing from known state. DELETE: /api/v1.0/revisions status: 204 + request_headers: + content-type: application/x-yaml response_headers: null - name: create_documents_for_secret_substitution desc: Create documents with substitution source with storagePolicy=encrypted PUT: /api/v1.0/buckets/secret/documents status: 200 + request_headers: + content-type: application/x-yaml + response_headers: + content-type: application/x-yaml data: |- --- schema: deckhand/LayeringPolicy/v1 @@ -164,6 +167,10 @@ tests: desc: Verify that secret ref was created for each secret document type. GET: /api/v1.0/revisions/$RESPONSE['$.[0].status.revision']/documents status: 200 + request_headers: + content-type: application/x-yaml + response_headers: + content-type: application/x-yaml query_parameters: metadata.name: - example-ca @@ -175,7 +182,7 @@ tests: - example-public-key response_multidoc_jsonpaths: $.`len`: 7 - # NOTE(fmontei): jsonpath-rw-ext uses a 1 character separator (rather than allowing a string) + # NOTE(felipemonteiro): jsonpath-rw-ext uses a 1 character separator (rather than allowing a string) # leading to this nastiness: $.[0].data.`split(:, 0, 1)` + "://" + $.[0].data.`split(/, 2, 3)`: $ENVIRON['TEST_BARBICAN_URL'] $.[1].data.`split(:, 0, 1)` + "://" + $.[1].data.`split(/, 2, 3)`: $ENVIRON['TEST_BARBICAN_URL'] @@ -185,13 +192,54 @@ tests: $.[5].data.`split(:, 0, 1)` + "://" + $.[5].data.`split(/, 2, 3)`: $ENVIRON['TEST_BARBICAN_URL'] $.[6].data.`split(:, 0, 1)` + "://" + $.[6].data.`split(/, 2, 3)`: $ENVIRON['TEST_BARBICAN_URL'] + - name: validate_expected_secrets_exist_in_barbican + desc: Validate that all the expected secrets were created in Barbican. + GET: $ENVIRON['TEST_BARBICAN_URL']/v1/secrets + status: 200 + query_parameters: + sort: name + request_headers: + content-type: application/json + response_headers: + content-type: application/json; charset=UTF-8 + response_json_paths: + $.secrets.`len`: 7 + $.secrets[*].status: + - ACTIVE + - ACTIVE + - ACTIVE + - ACTIVE + - ACTIVE + - ACTIVE + - ACTIVE + $.secrets[*].name: + - example-ca + - example-ca-key + - example-cert + - example-cert-key + - example-passphrase + - example-private-key + - example-public-key + $.secrets[*].secret_type: + - certificate + - private + - certificate + - private + - passphrase + - private + - public + - name: verify_secret_payload_in_destination_document desc: | Verify each secret payload is injected into the destination document and that the secret payload is present in each secret document type rather than the Barbican reference. - GET: /api/v1.0/revisions/$RESPONSE['$.[0].status.revision']/rendered-documents + GET: /api/v1.0/revisions/$HISTORY['create_documents_for_secret_substitution'].$RESPONSE['$.[0].status.revision']/rendered-documents status: 200 + request_headers: + content-type: application/x-yaml + response_headers: + content-type: application/x-yaml query_parameters: sort: 'metadata.name' response_multidoc_jsonpaths: diff --git a/deckhand/tests/unit/control/test_validations_controller.py b/deckhand/tests/unit/control/test_validations_controller.py index 5b1cebdf..05c94d4e 100644 --- a/deckhand/tests/unit/control/test_validations_controller.py +++ b/deckhand/tests/unit/control/test_validations_controller.py @@ -84,9 +84,9 @@ class BaseValidationsControllerTest(test_base.BaseControllerTest): """Workaround for testing complex validation scenarios by forcibly passing in `pre_validate=False`. """ - # TODO(fmontei): Remove this workaround by testing these more complex - # scenarios against the rendered-documents endpoint instead (which - # performs post-validation). + # TODO(felipemonteiro): Remove this workaround by testing these more + # complex scenarios against the rendered-documents endpoint instead + # (which performs post-validation). original_document_validation = document_validation.DocumentValidation def monkey_patch(*args, **kwargs): diff --git a/deckhand/tests/unit/db/base.py b/deckhand/tests/unit/db/base.py index 05a2c0b2..17bc3ab1 100644 --- a/deckhand/tests/unit/db/base.py +++ b/deckhand/tests/unit/db/base.py @@ -25,7 +25,7 @@ DOCUMENT_EXPECTED_FIELDS = BASE_EXPECTED_FIELDS + ( REVISION_EXPECTED_FIELDS = ("id", "documents", "tags") -# TODO(fmontei): Move this into a separate module called `fixtures`. +# TODO(felipemonteiro): Move this into a separate module called `fixtures`. class DocumentFixture(object): @staticmethod diff --git a/deckhand/tests/unit/engine/test_document_layering.py b/deckhand/tests/unit/engine/test_document_layering.py index a4e68600..6dd1af87 100644 --- a/deckhand/tests/unit/engine/test_document_layering.py +++ b/deckhand/tests/unit/engine/test_document_layering.py @@ -29,7 +29,7 @@ class TestDocumentLayering(test_base.DeckhandTestCase): def _test_layering(self, documents, site_expected=None, region_expected=None, global_expected=None, validate=False, strict=True, **kwargs): - # TODO(fmontei): Refactor all tests to work with strict=True. + # TODO(felipemonteiro): Refactor all tests to work with strict=True. # Test layering twice: once by passing in the documents in the normal # order and again with the documents in reverse order for good measure, diff --git a/deckhand/tests/unit/engine/test_document_layering_and_substitution_negative.py b/deckhand/tests/unit/engine/test_document_layering_and_substitution_negative.py index b1a2c7e3..2215c015 100644 --- a/deckhand/tests/unit/engine/test_document_layering_and_substitution_negative.py +++ b/deckhand/tests/unit/engine/test_document_layering_and_substitution_negative.py @@ -91,7 +91,7 @@ class TestDocumentLayeringWithSubstitutionNegative( itself. """ - # TODO(fmontei): Move to test_secrets_manager (negative) + # TODO(felipemonteiro): Move to test_secrets_manager (negative) mapping = { "_GLOBAL_DATA_1_": {"data": {"a": {"x": 1, "y": 2}}}, "_SITE_NAME_1_": "site-1", @@ -122,7 +122,7 @@ class TestDocumentLayeringWithSubstitutionNegative( self, mock_log): """Validate that a missing substitution source document fails.""" - # TODO(fmontei): Move to test_secrets_manager (negative) + # TODO(felipemonteiro): Move to test_secrets_manager (negative) mapping = { "_GLOBAL_SUBSTITUTIONS_1_": [{ "dest": { diff --git a/deckhand/tests/unit/engine/test_secrets_manager.py b/deckhand/tests/unit/engine/test_secrets_manager.py index 371fc116..ac733afb 100644 --- a/deckhand/tests/unit/engine/test_secrets_manager.py +++ b/deckhand/tests/unit/engine/test_secrets_manager.py @@ -519,8 +519,8 @@ class TestSecretsSubstitution(test_base.TestDbBase): document_mapping = { "_GLOBAL_SUBSTITUTIONS_1_": [{ "dest": { - # NOTE(fmontei): Usage of special characters like this - # without quotes need not be handled because it is not + # NOTE(felipemonteiro): Usage of special characters like + # this without quotes need not be handled because it is not # valid YAML to include ":" without quotes. "path": ".values.conf.paste.'filter:authtoken'.password" },