integration tests: Add Barbican validation/assertions

This patchset adds Barbican validation/assertions to integration
tests by querying the Barbican API server where appropriate
and validating that the expected data is returned in order
to sanity-check the integration scenarios further.

Change-Id: If5d30712b289f09ac9712ee205673be4150cda16

deckhand/tests/common/test_gabbi.py

@@ -102,10 +102,11 @@ def load_tests(loader, tests, pattern):
         loader,
         host='localhost',
         url=os.environ.get('DECKHAND_TEST_URL', '127.0.0.1:9000'),
-        # NOTE(fmontei): When there are multiple handlers listed that accept
+        # NOTE(felipemonteiro): When there are multiple handlers listed that
-        # the same content-type, the one that is earliest in the list will be
+        # accept the same content-type, the one that is earliest in the list
-        # used. Thus, we cannot specify multiple content handlers for handling
+        # will be used. Thus, we cannot specify multiple content handlers for
-        # list/dictionary responses from the server using different handlers.
+        # handling list/dictionary responses from the server using different
+        # handlers.
         content_handlers=[
             core.StringResponseHandler,  # For parsing text/plain
             jsonhandler.JSONHandler,     # For parsing application/json

deckhand/tests/deckhand.conf.test

@@ -13,7 +13,7 @@ policy_file = policy.yaml
 connection = ${AIRSHIP_DECKHAND_DATABASE_URL}
 
 [keystone_authtoken]
-# NOTE(fmontei): Values taken from clouds.yaml. Values only used for
+# NOTE(felipemonteiro): Values taken from clouds.yaml. Values only used for
 # integration testing.
 #
 # clouds.yaml (snippet):

deckhand/tests/functional/gabbits/resources/layering-and-substitution-dag-sample.yaml

@@ -28,7 +28,7 @@ metadata:
   storagePolicy: cleartext
 data: my-secret-password
 ---
-# NOTE(fmontei): The documents below are included in reverse order with
+# NOTE(felipemonteiro): The documents below are included in reverse order with
 # respect to their substitution dependency hierarchy in order to verify
 # that the dependency chain is correctly resolved in the code.
 schema: armada/Chart/v1

deckhand/tests/integration/gabbits/document-crud-secret.yaml

@@ -6,10 +6,7 @@
 
 defaults:
   request_headers:
-    content-type: application/x-yaml
     X-Auth-Token: $ENVIRON['TEST_AUTH_TOKEN']
-  response_headers:
-    content-type: application/x-yaml
   verbose: true
 
 tests:
@@ -17,12 +14,18 @@ tests:
     desc: Begin testing from known state.
     DELETE: /api/v1.0/revisions
     status: 204
+    request_headers:
+      content-type: application/x-yaml
     response_headers: null
 
   - name: create_encrypted_passphrase
     desc: Create passphrase with storagePolicy=encrypted
     PUT: /api/v1.0/buckets/secret/documents
     status: 200
+    request_headers:
+      content-type: application/x-yaml
+    response_headers:
+      content-type: application/x-yaml
     data: |-
       ---
       schema: deckhand/Passphrase/v1
@@ -37,16 +40,43 @@ tests:
       ...
     response_multidoc_jsonpaths:
       $.`len`: 1
-      # NOTE(fmontei): jsonpath-rw-ext uses a 1 character separator (rather than allowing a string)
+      # NOTE(felipemonteiro): jsonpath-rw-ext uses a 1 character separator (rather than allowing a string)
       # leading to this nastiness:
       $.[0].data.`split(:, 0, 1)` + "://" + $.[0].data.`split(/, 2, 3)`: $ENVIRON['TEST_BARBICAN_URL']
 
+  - name: validate_secret_exists_in_barbican
+    desc: Validate that the secret ref exists in Barbican.
+    GET: $ENVIRON['TEST_BARBICAN_URL']/v1/secrets/$RESPONSE['$.[0].data.`split(/, 5, -1)`']
+    status: 200
+    request_headers:
+      content-type: application/json
+    response_headers:
+      content-type: application/json; charset=UTF-8
+    response_json_paths:
+      $.status: ACTIVE
+      $.name: my-passphrase
+
+  - name: validate_secret_payload_matches_in_barbican
+    desc: Validate that the secret itself matches in Barbican.
+    GET: $ENVIRON['TEST_BARBICAN_URL']/v1/secrets/$HISTORY['create_encrypted_passphrase'].$RESPONSE['$.[0].data.`split(/, 5, -1)`']/payload
+    status: 200
+    request_headers:
+      content-type: application/json
+    response_headers:
+      content-type: application/octet-stream; charset=UTF-8
+    response_strings:
+      - not-a-real-password
+
   - name: verify_revision_documents_returns_secret_ref
     desc: Verify that the documents for the created revision returns the secret ref.
-    GET: /api/v1.0/revisions/$RESPONSE['$.[0].status.revision']/documents
+    GET: /api/v1.0/revisions/$HISTORY['create_encrypted_passphrase'].$RESPONSE['$.[0].status.revision']/documents
     status: 200
+    request_headers:
+      content-type: application/x-yaml
+    response_headers:
+      content-type: application/x-yaml
     response_multidoc_jsonpaths:
       $.`len`: 1
-      # NOTE(fmontei): jsonpath-rw-ext uses a 1 character separator (rather than allowing a string)
+      # NOTE(felipemonteiro): jsonpath-rw-ext uses a 1 character separator (rather than allowing a string)
       # leading to this nastiness:
       $.[0].data.`split(:, 0, 1)` + "://" + $.[0].data.`split(/, 2, 3)`: $ENVIRON['TEST_BARBICAN_URL']

deckhand/tests/integration/gabbits/document-render-secret-edge-cases.yaml

@@ -11,20 +11,29 @@
 
 defaults:
   request_headers:
-    content-type: application/x-yaml
     X-Auth-Token: $ENVIRON['TEST_AUTH_TOKEN']
-  response_headers:
-    content-type: application/x-yaml
   verbose: true
 
 tests:
   ### Scenario 1 ###
-  - name: attempt_create_encrypted_passphrase_empty_payload
+  - name: purge
+    desc: Begin testing from known state.
+    DELETE: /api/v1.0/revisions
+    status: 204
+    request_headers:
+      content-type: application/x-yaml
+    response_headers: null
+
+  - name: create_encrypted_passphrase_empty_payload_skips
     desc: |
       Attempting to create an encrypted passphrase with empty payload should
       avoid encryption and return the empty payload instead.
     PUT: /api/v1.0/buckets/secret/documents
     status: 200
+    request_headers:
+      content-type: application/x-yaml
+    response_headers:
+      content-type: application/x-yaml
     data: |-
       ---
       schema: deckhand/LayeringPolicy/v1
@@ -50,10 +59,26 @@ tests:
       $.[1].metadata.name: my-passphrase
       $.[1].data: ''
 
+  - name: validate_no_secret_exists_in_barbican
+    desc: Validate that no secret was created in Barbican.
+    GET: $ENVIRON['TEST_BARBICAN_URL']/v1/secrets
+    status: 200
+    request_headers:
+      content-type: application/json
+    response_headers:
+      content-type: application/json; charset=UTF-8
+    response_json_paths:
+      $.secrets.`len`: 0
+      $.secrets: []
+
   - name: verify_revision_documents_returns_same_empty_payload
     desc: Verify that the created document wasn't encrypted.
-    GET: /api/v1.0/revisions/$RESPONSE['$.[0].status.revision']/documents
+    GET: /api/v1.0/revisions/$HISTORY['create_encrypted_passphrase_empty_payload_skips'].$RESPONSE['$.[0].status.revision']/documents
     status: 200
+    request_headers:
+      content-type: application/x-yaml
+    response_headers:
+      content-type: application/x-yaml
     query_parameters:
       metadata.name: my-passphrase
     response_multidoc_jsonpaths:
@@ -65,8 +90,12 @@ tests:
       Verify that rendering the document returns the same empty payload
       which requires that the data be read directly from Deckhand's DB
       rather than Barbican.
-    GET: /api/v1.0/revisions/$RESPONSE['$.[0].status.revision']/rendered-documents
+    GET: /api/v1.0/revisions/$HISTORY['create_encrypted_passphrase_empty_payload_skips'].$RESPONSE['$.[0].status.revision']/rendered-documents
     status: 200
+    request_headers:
+      content-type: application/x-yaml
+    response_headers:
+      content-type: application/x-yaml
     query_parameters:
       metadata.name: my-passphrase
     response_multidoc_jsonpaths:
@@ -82,6 +111,10 @@ tests:
       successfully.
     PUT: /api/v1.0/buckets/secret/documents
     status: 200
+    request_headers:
+      content-type: application/x-yaml
+    response_headers:
+      content-type: application/x-yaml
     data: |-
       ---
       schema: deckhand/LayeringPolicy/v1
@@ -107,14 +140,31 @@ tests:
     response_multidoc_jsonpaths:
       $.`len`: 2
       $.[1].metadata.name: armada-doc
-      # NOTE(fmontei): jsonpath-rw-ext uses a 1 character separator (rather than allowing a string)
+      # NOTE(felipemonteiro): jsonpath-rw-ext uses a 1 character separator (rather than allowing a string)
       # leading to this nastiness:
       $.[1].data.`split(:, 0, 1)` + "://" + $.[1].data.`split(/, 2, 3)`: $ENVIRON['TEST_BARBICAN_URL']
 
+  - name: validate_opaque_secret_exists_in_barbican
+    desc: Validate that the secret was created as opaque in Barbican.
+    GET: $ENVIRON['TEST_BARBICAN_URL']/v1/secrets/$HISTORY['create_encrypted_passphrase_with_incompatible_payload'].$RESPONSE['$.[1].data.`split(/, 5, -1)`']
+    status: 200
+    request_headers:
+      content-type: application/json
+    response_headers:
+      content-type: application/json; charset=UTF-8
+    response_json_paths:
+      $.status: ACTIVE
+      $.name: armada-doc
+      $.secret_type: opaque
+
   - name: verify_revision_documents_returns_barbican_ref
     desc: Verify that the encrypted document returns a Barbican ref.
-    GET: /api/v1.0/revisions/$RESPONSE['$.[0].status.revision']/documents
+    GET: /api/v1.0/revisions/$HISTORY['create_encrypted_passphrase_with_incompatible_payload'].$RESPONSE['$.[0].status.revision']/documents
     status: 200
+    request_headers:
+      content-type: application/x-yaml
+    response_headers:
+      content-type: application/x-yaml
     query_parameters:
       metadata.name: armada-doc
     response_multidoc_jsonpaths:
@@ -126,8 +176,12 @@ tests:
       Verify that rendering the document returns the original payload which
       means that Deckhand successfully encoded and decoded the non-compatible
       payload using base64 encoding.
-    GET: /api/v1.0/revisions/$RESPONSE['$.[0].status.revision']/rendered-documents
+    GET: /api/v1.0/revisions/$HISTORY['create_encrypted_passphrase_with_incompatible_payload'].$RESPONSE['$.[0].status.revision']/rendered-documents
     status: 200
+    request_headers:
+      content-type: application/x-yaml
+    response_headers:
+      content-type: application/x-yaml
     query_parameters:
       metadata.name: armada-doc
     response_multidoc_jsonpaths:

deckhand/tests/integration/gabbits/document-render-secret.yaml

@@ -43,7 +43,7 @@ tests:
       ...
     response_multidoc_jsonpaths:
       $.`len`: 2
-      # NOTE(fmontei): jsonpath-rw-ext uses a 1 character separator (rather than allowing a string)
+      # NOTE(felipemonteiro): jsonpath-rw-ext uses a 1 character separator (rather than allowing a string)
       # leading to this nastiness:
       $.[1].data.`split(:, 0, 1)` + "://" + $.[1].data.`split(/, 2, 3)`: $ENVIRON['TEST_BARBICAN_URL']
 

deckhand/tests/integration/gabbits/document-substitution-secret-generic.yaml

@@ -76,13 +76,27 @@ tests:
       metadata.name: example-armada-cert
     response_multidoc_jsonpaths:
       $.`len`: 1
-      # NOTE(fmontei): jsonpath-rw-ext uses a 1 character separator (rather than allowing a string)
+      # NOTE(felipemonteiro): jsonpath-rw-ext uses a 1 character separator (rather than allowing a string)
       # leading to this nastiness:
       $.[0].data.`split(:, 0, 1)` + "://" + $.[0].data.`split(/, 2, 3)`: $ENVIRON['TEST_BARBICAN_URL']
 
+  - name: verify_generic_secret_created_in_barbican
+    desc: Validate that the generic secret gets stored with secret_type passphrase.
+    GET: $ENVIRON['TEST_BARBICAN_URL']/v1/secrets/$RESPONSE['$.[0].data.`split(/, 5, -1)`']
+    status: 200
+    request_headers:
+      content-type: application/json
+    response_headers:
+      content-type: application/json; charset=UTF-8
+    response_json_paths:
+      $.status: ACTIVE
+      $.name: example-armada-cert
+      # Default type for documents with generic schema.
+      $.secret_type: passphrase
+
   - name: verify_secret_payload_in_destination_document
     desc: Verify secret payload is injected in destination document as well as example-armada-cert.
-    GET: /api/v1.0/revisions/$RESPONSE['$.[0].status.revision']/rendered-documents
+    GET: /api/v1.0/revisions/$HISTORY['encrypt_generic_document_for_secret_substitution'].$RESPONSE['$.[0].status.revision']/rendered-documents
     status: 200
     query_parameters:
       metadata.name:

deckhand/tests/integration/gabbits/document-substitution-secret.yaml

@@ -9,10 +9,7 @@
 
 defaults:
   request_headers:
-    content-type: application/x-yaml
     X-Auth-Token: $ENVIRON['TEST_AUTH_TOKEN']
-  response_headers:
-    content-type: application/x-yaml
   verbose: true
 
 tests:
@@ -20,12 +17,18 @@ tests:
     desc: Begin testing from known state.
     DELETE: /api/v1.0/revisions
     status: 204
+    request_headers:
+      content-type: application/x-yaml
     response_headers: null
 
   - name: create_documents_for_secret_substitution
     desc: Create documents with substitution source with storagePolicy=encrypted
     PUT: /api/v1.0/buckets/secret/documents
     status: 200
+    request_headers:
+      content-type: application/x-yaml
+    response_headers:
+      content-type: application/x-yaml
     data: |-
       ---
       schema: deckhand/LayeringPolicy/v1
@@ -164,6 +167,10 @@ tests:
     desc: Verify that secret ref was created for each secret document type.
     GET: /api/v1.0/revisions/$RESPONSE['$.[0].status.revision']/documents
     status: 200
+    request_headers:
+      content-type: application/x-yaml
+    response_headers:
+      content-type: application/x-yaml
     query_parameters:
       metadata.name:
         - example-ca
@@ -175,7 +182,7 @@ tests:
         - example-public-key
     response_multidoc_jsonpaths:
       $.`len`: 7
-      # NOTE(fmontei): jsonpath-rw-ext uses a 1 character separator (rather than allowing a string)
+      # NOTE(felipemonteiro): jsonpath-rw-ext uses a 1 character separator (rather than allowing a string)
       # leading to this nastiness:
       $.[0].data.`split(:, 0, 1)` + "://" + $.[0].data.`split(/, 2, 3)`: $ENVIRON['TEST_BARBICAN_URL']
       $.[1].data.`split(:, 0, 1)` + "://" + $.[1].data.`split(/, 2, 3)`: $ENVIRON['TEST_BARBICAN_URL']
@@ -185,13 +192,54 @@ tests:
       $.[5].data.`split(:, 0, 1)` + "://" + $.[5].data.`split(/, 2, 3)`: $ENVIRON['TEST_BARBICAN_URL']
       $.[6].data.`split(:, 0, 1)` + "://" + $.[6].data.`split(/, 2, 3)`: $ENVIRON['TEST_BARBICAN_URL']
 
+  - name: validate_expected_secrets_exist_in_barbican
+    desc: Validate that all the expected secrets were created in Barbican.
+    GET: $ENVIRON['TEST_BARBICAN_URL']/v1/secrets
+    status: 200
+    query_parameters:
+      sort: name
+    request_headers:
+      content-type: application/json
+    response_headers:
+      content-type: application/json; charset=UTF-8
+    response_json_paths:
+      $.secrets.`len`: 7
+      $.secrets[*].status:
+        - ACTIVE
+        - ACTIVE
+        - ACTIVE
+        - ACTIVE
+        - ACTIVE
+        - ACTIVE
+        - ACTIVE
+      $.secrets[*].name:
+        - example-ca
+        - example-ca-key
+        - example-cert
+        - example-cert-key
+        - example-passphrase
+        - example-private-key
+        - example-public-key
+      $.secrets[*].secret_type:
+        - certificate
+        - private
+        - certificate
+        - private
+        - passphrase
+        - private
+        - public
+
   - name: verify_secret_payload_in_destination_document
     desc: |
       Verify each secret payload is injected into the destination document and that
       the secret payload is present in each secret document type rather than the
       Barbican reference.
-    GET: /api/v1.0/revisions/$RESPONSE['$.[0].status.revision']/rendered-documents
+    GET: /api/v1.0/revisions/$HISTORY['create_documents_for_secret_substitution'].$RESPONSE['$.[0].status.revision']/rendered-documents
     status: 200
+    request_headers:
+      content-type: application/x-yaml
+    response_headers:
+      content-type: application/x-yaml
     query_parameters:
       sort: 'metadata.name'
     response_multidoc_jsonpaths:

deckhand/tests/unit/control/test_validations_controller.py

@@ -84,9 +84,9 @@ class BaseValidationsControllerTest(test_base.BaseControllerTest):
         """Workaround for testing complex validation scenarios by forcibly
         passing in `pre_validate=False`.
         """
-        # TODO(fmontei): Remove this workaround by testing these more complex
+        # TODO(felipemonteiro): Remove this workaround by testing these more
-        # scenarios against the rendered-documents endpoint instead (which
+        # complex scenarios against the rendered-documents endpoint instead
-        # performs post-validation).
+        # (which performs post-validation).
         original_document_validation = document_validation.DocumentValidation
 
         def monkey_patch(*args, **kwargs):

deckhand/tests/unit/db/base.py

@@ -25,7 +25,7 @@ DOCUMENT_EXPECTED_FIELDS = BASE_EXPECTED_FIELDS + (
 REVISION_EXPECTED_FIELDS = ("id", "documents", "tags")
 
 
-# TODO(fmontei): Move this into a separate module called `fixtures`.
+# TODO(felipemonteiro): Move this into a separate module called `fixtures`.
 class DocumentFixture(object):
 
     @staticmethod

deckhand/tests/unit/engine/test_document_layering.py

@@ -29,7 +29,7 @@ class TestDocumentLayering(test_base.DeckhandTestCase):
     def _test_layering(self, documents, site_expected=None,
                        region_expected=None, global_expected=None,
                        validate=False, strict=True, **kwargs):
-        # TODO(fmontei): Refactor all tests to work with strict=True.
+        # TODO(felipemonteiro): Refactor all tests to work with strict=True.
 
         # Test layering twice: once by passing in the documents in the normal
         # order and again with the documents in reverse order for good measure,

deckhand/tests/unit/engine/test_document_layering_and_substitution_negative.py

@@ -91,7 +91,7 @@ class TestDocumentLayeringWithSubstitutionNegative(
         itself.
         """
 
-        # TODO(fmontei): Move to test_secrets_manager (negative)
+        # TODO(felipemonteiro): Move to test_secrets_manager (negative)
         mapping = {
             "_GLOBAL_DATA_1_": {"data": {"a": {"x": 1, "y": 2}}},
             "_SITE_NAME_1_": "site-1",
@@ -122,7 +122,7 @@ class TestDocumentLayeringWithSubstitutionNegative(
             self, mock_log):
         """Validate that a missing substitution source document fails."""
 
-        # TODO(fmontei): Move to test_secrets_manager (negative)
+        # TODO(felipemonteiro): Move to test_secrets_manager (negative)
         mapping = {
             "_GLOBAL_SUBSTITUTIONS_1_": [{
                 "dest": {

deckhand/tests/unit/engine/test_secrets_manager.py

@@ -519,8 +519,8 @@ class TestSecretsSubstitution(test_base.TestDbBase):
         document_mapping = {
             "_GLOBAL_SUBSTITUTIONS_1_": [{
                 "dest": {
-                    # NOTE(fmontei): Usage of special characters like this
+                    # NOTE(felipemonteiro): Usage of special characters like
-                    # without quotes need not be handled because it is not
+                    # this without quotes need not be handled because it is not
                     # valid YAML to include ":" without quotes.
                     "path": ".values.conf.paste.'filter:authtoken'.password"
                 },