diff --git a/deckhand/client/client.py b/deckhand/client/client.py index 093bce1b..24046d30 100644 --- a/deckhand/client/client.py +++ b/deckhand/client/client.py @@ -30,7 +30,6 @@ from deckhand.client import buckets from deckhand.client import exceptions from deckhand.client import revisions from deckhand.client import tags -from deckhand.client import validations class SessionClient(adapter.Adapter): @@ -203,7 +202,6 @@ class Client(object): self.buckets = buckets.BucketManager(self) self.revisions = revisions.RevisionManager(self) self.tags = tags.RevisionTagManager(self) - self.validations = validations.ValidationManager(self) self.client = _construct_http_client( api_version=api_version, diff --git a/deckhand/client/validations.py b/deckhand/client/validations.py deleted file mode 100644 index 54140808..00000000 --- a/deckhand/client/validations.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright 2017 AT&T Intellectual Property. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from deckhand.client import base - - -class Validation(base.Resource): - def __repr__(self): - return ("") - - -class ValidationManager(base.Manager): - """Manage :class:`Validation` resources.""" - resource_class = Validation - - def list(self, revision_id): - """Get list of revision validations.""" - url = '/revisions/%s/validations' % revision_id - return self._list(url) - - def list_entries(self, revision_id, validation_name): - """Get list of entries for a validation.""" - url = '/revisions/%s/validations/%s' % (revision_id, validation_name) - # Call `_get` instead of `_list` because the response from the server - # is a dict of form `{"count": n, "results": []}`. - return self._get(url) - - def get_entry(self, revision_id, validation_name, entry_id): - """Get entry details for a validation.""" - url = '/revisions/%s/validations/%s/entries/%s' % ( - revision_id, validation_name, entry_id) - return self._get(url) - - def create(self, revision_id, validation_name, data): - """Associate a validation with a revision.""" - url = '/revisions/%s/validations/%s' % (revision_id, validation_name) - return self._create(url, data=data) diff --git a/deckhand/control/buckets.py b/deckhand/control/buckets.py index 50d5b313..dc531966 100644 --- a/deckhand/control/buckets.py +++ b/deckhand/control/buckets.py @@ -48,7 +48,7 @@ class BucketsResource(api_base.BaseResource): try: doc_validator = document_validation.DocumentValidation( documents, data_schemas, pre_validate=True) - validations = doc_validator.validate_all() + doc_validator.validate_all() except deckhand_errors.InvalidDocumentFormat as e: with excutils.save_and_reraise_exception(): LOG.exception(e.format_message()) @@ -64,10 +64,6 @@ class BucketsResource(api_base.BaseResource): created_documents = self._create_revision_documents( bucket_name, documents) - if created_documents: - revision_id = created_documents[0]['revision_id'] - self._create_revision_validations(revision_id, validations) - resp.body = self.view_builder.list(created_documents) resp.status = falcon.HTTP_200 @@ -88,8 +84,3 @@ class BucketsResource(api_base.BaseResource): LOG.exception(e.format_message()) return created_documents - - def _create_revision_validations(self, revision_id, validations): - for validation in validations: - db_api.validation_create(revision_id, validation['name'], - validation) diff --git a/deckhand/tests/functional/gabbits/document/document-crud-success-owned-documents.yaml b/deckhand/tests/functional/gabbits/document/document-crud-success-owned-documents.yaml index f3da5e64..d8f1d764 100644 --- a/deckhand/tests/functional/gabbits/document/document-crud-success-owned-documents.yaml +++ b/deckhand/tests/functional/gabbits/document/document-crud-success-owned-documents.yaml @@ -65,26 +65,3 @@ tests: response_multidoc_jsonpaths: $.`len`: 1 $.[0].data: hunter2 - - - name: verify_schema_is_valid - desc: Check schema validation of the added schema - GET: /api/v1.0/revisions/$HISTORY['initialize'].$RESPONSE['$.[0].status.revision']/validations/deckhand-schema-validation - status: 200 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].results[*].status: - - success - - success - - success - - success - - success - - success - - success - - success - - success - - success - - success - - success - - success - - success - - success diff --git a/deckhand/tests/functional/gabbits/document/document-crud-success-unusual-documents.yaml b/deckhand/tests/functional/gabbits/document/document-crud-success-unusual-documents.yaml index bc736817..0aab1fa5 100644 --- a/deckhand/tests/functional/gabbits/document/document-crud-success-unusual-documents.yaml +++ b/deckhand/tests/functional/gabbits/document/document-crud-success-unusual-documents.yaml @@ -61,31 +61,3 @@ tests: response_multidoc_jsonpaths: $.`len`: 1 $.[0].data: 9000 - - - name: verify_document_validation_success_in_list_view - desc: Check document validation success shows in list view - GET: /api/v1.0/revisions/$HISTORY['initialize'].$RESPONSE['$.[0].status.revision']/validations - status: 200 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].count: 1 - $.[0].results[0].status: success - - - name: verify_document_validation_success_in_details_view - desc: Check document validation success shows in detailed view - GET: /api/v1.0/revisions/$HISTORY['initialize'].$RESPONSE['$.[0].status.revision']/validations/deckhand-schema-validation - status: 200 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].count: 9 - $.[0].results[*].status: - # 9 documents are created in total, including DataSchema documents. - - success - - success - - success - - success - - success - - success - - success - - success - - success diff --git a/deckhand/tests/functional/gabbits/schema-validation/schema-validation-success-ucp-sample.yaml b/deckhand/tests/functional/gabbits/schema-validation/schema-validation-success-ucp-sample.yaml deleted file mode 100644 index 88a09d99..00000000 --- a/deckhand/tests/functional/gabbits/schema-validation/schema-validation-success-ucp-sample.yaml +++ /dev/null @@ -1,39 +0,0 @@ -# Test success path for sample Airship documents -# -# 1. Purges existing data to ensure test isolation -# 2. Creates sample Airship documents and schemas -# 3. Checks that the documents pass schema validation - -defaults: - request_headers: - content-type: application/x-yaml - response_headers: - content-type: application/x-yaml - verbose: true - -tests: - - name: purge - desc: Begin testing from known state. - DELETE: /api/v1.0/revisions - status: 204 - response_headers: null - - - name: initialize - desc: Add example schema - PUT: /api/v1.0/buckets/mop/documents - status: 200 - data: <@resources/ucp-sample-documents.yaml - - - name: verify_schema_is_valid - desc: Check schema validation of the added schema - GET: /api/v1.0/revisions/$HISTORY['initialize'].$RESPONSE['$.[0].status.revision']/validations/deckhand-schema-validation - status: 200 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].count: 5 - $.[0].results[*].status: - - success - - success - - success - - success - - success diff --git a/deckhand/tests/functional/gabbits/schema-validation/schema-validation-success-v2.yaml b/deckhand/tests/functional/gabbits/schema-validation/schema-validation-success-v2.yaml deleted file mode 100644 index cc7d6f2a..00000000 --- a/deckhand/tests/functional/gabbits/schema-validation/schema-validation-success-v2.yaml +++ /dev/null @@ -1,214 +0,0 @@ -# Test success path for rollback with a single bucket. -# -# 1. Purges existing data to ensure test isolation -# 2. Creates a v2 DataSchema -# 3. Checks that schema validation for the DataSchema passes -# 4. Puts a valid document (and LayeringPolicy) -# 5. Checks that the document passes schema pre-validation -# 6. Checks that the document passes schema post-validation -# 7. Puts an invalid document -# 8. Checks that the document fails schema pre-validation -# 9. Checks that the document fails schema post-validation by raising expected -# exception -# 10. Checks that the document entry details adhere to expected validation -# format -# 11. Re-puts the same invalid document with substitutions -# 12. Verify that the substitutions were sanitized in the validation output - -defaults: - request_headers: - content-type: application/x-yaml - response_headers: - content-type: application/x-yaml - verbose: true - -tests: - - name: purge - desc: Begin testing from known state. - DELETE: /api/v1.0/revisions - status: 204 - response_headers: null - - - name: create_schema - desc: Add example schema - PUT: /api/v1.0/buckets/mop/documents - status: 200 - data: <@resources/sample-schema-v2.yaml - - - name: verify_schema_is_valid - desc: Check schema validation of the added schema - GET: /api/v1.0/revisions/$HISTORY['create_schema'].$RESPONSE['$.[0].status.revision']/validations/deckhand-schema-validation - status: 200 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].count: 1 - $.[0].results[0].id: 0 - $.[0].results[0].status: success - - - name: verify_schema_validation_in_list_view - desc: Check schema validation success shows in list view - GET: /api/v1.0/revisions/$HISTORY['create_schema'].$RESPONSE['$.[0].status.revision']/validations - status: 200 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].count: 1 - $.[0].results[0].name: deckhand-schema-validation - $.[0].results[0].status: success - - - name: add_valid_document - desc: Add a document that follows the schema - PUT: /api/v1.0/buckets/good/documents - status: 200 - data: |- - --- - schema: deckhand/LayeringPolicy/v1 - metadata: - schema: metadata/Control/v1 - name: layering-policy - data: - layerOrder: - - site - --- - schema: example/Doc/v2 - metadata: - schema: metadata/Document/v1 - name: good - storagePolicy: cleartext - layeringDefinition: - abstract: false - layer: site - data: - a: this-one-is-required - b: 77 - - - name: verify_document_is_valid_pre_validation - desc: Check schema pre-validation of the added document - GET: /api/v1.0/revisions/$HISTORY['add_valid_document'].$RESPONSE['$.[0].status.revision']/validations/deckhand-schema-validation - status: 200 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].count: 2 - $.[0].results[0].id: 0 - $.[0].results[0].status: success - - - name: verify_document_pre_validation_success_in_list_view - desc: Check document pre-validation success shows in list view - GET: /api/v1.0/revisions/$HISTORY['add_valid_document'].$RESPONSE['$.[0].status.revision']/validations - status: 200 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].count: 1 - $.[0].results[*].name: deckhand-schema-validation - $.[0].results[*].status: success - - - name: verify_document_is_valid_post_validation - desc: Check that the document passes post-validation - GET: /api/v1.0/revisions/$HISTORY['add_valid_document'].$RESPONSE['$.[0].status.revision']/rendered-documents - status: 200 - - - name: add_invalid_document - desc: Add a document that does not follow the schema - PUT: /api/v1.0/buckets/bad/documents - status: 200 - data: |- - schema: example/Doc/v2 - metadata: - schema: metadata/Document/v1 - name: bad - storagePolicy: cleartext - layeringDefinition: - abstract: false - layer: site - data: - a: this-one-is-required-and-can-be-different - b: 177 - - - name: verify_invalid_document_is_valid_pre_validation - desc: Check success of schema pre-validation of the added document - GET: /api/v1.0/revisions/$HISTORY['add_invalid_document'].$RESPONSE['$.[0].status.revision']/validations/deckhand-schema-validation - status: 200 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].count: 1 - $.[0].results[*].status: success - - - name: verify_document_pre_validation_failure_in_list_view - desc: Check document pre-validation success shows in list view - GET: /api/v1.0/revisions/$HISTORY['add_invalid_document'].$RESPONSE['$.[0].status.revision']/validations - status: 200 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].count: 1 - $.[0].results[0].name: deckhand-schema-validation - $.[0].results[0].status: success - - - name: verify_document_is_invalid_post_validation - desc: Check that the document fails post-validation - GET: /api/v1.0/revisions/$HISTORY['add_invalid_document'].$RESPONSE['$.[0].status.revision']/rendered-documents - status: 400 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].apiVersion: v1.0 - $.[0].code: 400 Bad Request - $.[0].details.errorCount: 1 - $.[0].details.errorType: InvalidDocumentFormat - $.[0].details.messageList[0].documents: - - layer: site - name: bad - schema: example/Doc/v2 - $.[0].details.messageList[0].error: true - $.[0].details.messageList[0].kind: ValidationMessage - $.[0].details.messageList[0].level: Error - $.[0].details.messageList[0].name: D002 - $.[0].kind: Status - $.[0].message: The provided documents failed schema validation - $.[0].reason: Validation - $.[0].status: Failure - - - name: add_invalid_document_with_substitutions - desc: Add a document that does not follow the schema - PUT: /api/v1.0/buckets/bad/documents - status: 200 - data: |- - --- - schema: example/Doc/v2 - metadata: - schema: metadata/Document/v1 - name: bad - storagePolicy: cleartext - layeringDefinition: - abstract: false - layer: site - substitutions: - - src: - schema: deckhand/Certificate/v1 - name: test-certificate - path: . - dest: - path: .a - data: - a: this-one-is-required-and-can-be-different - b: 177 - --- - schema: deckhand/Certificate/v1 - metadata: - name: test-certificate - schema: metadata/Document/v1 - storagePolicy: cleartext - layeringDefinition: - layer: site - storagePolicy: cleartext - data: this-should-definitely-be-sanitized - - - name: verify_document_post_validation_failure_entry_details_hides_secrets - desc: Check document validation failure hides secrets - GET: /api/v1.0/revisions/$HISTORY['add_invalid_document_with_substitutions'].$RESPONSE['$.[0].status.revision']/rendered-documents - status: 400 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].code: 400 Bad Request - $.[0].details.errorCount: 1 - $.[0].details.errorType: InvalidDocumentFormat - $.[0].details.messageList[0].diagnostic.error_section: - a: 'Sanitized to avoid exposing secret.' - b: 177 diff --git a/deckhand/tests/functional/gabbits/schema-validation/schema-validation-success.yaml b/deckhand/tests/functional/gabbits/schema-validation/schema-validation-success.yaml deleted file mode 100644 index da032d0c..00000000 --- a/deckhand/tests/functional/gabbits/schema-validation/schema-validation-success.yaml +++ /dev/null @@ -1,214 +0,0 @@ -# Test success path for rollback with a single bucket. -# -# 1. Purges existing data to ensure test isolation -# 2. Creates a DataSchema -# 3. Checks that schema validation for the DataSchema passes -# 4. Puts a valid document (and LayeringPolicy) -# 5. Checks that the document passes schema pre-validation -# 6. Checks that the document passes schema post-validation -# 7. Puts an invalid document -# 8. Checks that the document fails schema pre-validation -# 9. Checks that the document fails schema post-validation by raising expected -# exception -# 10. Checks that the document entry details adhere to expected validation -# format -# 11. Re-puts the same invalid document with substitutions -# 12. Verify that the substitutions were sanitized in the validation output - -defaults: - request_headers: - content-type: application/x-yaml - response_headers: - content-type: application/x-yaml - verbose: true - -tests: - - name: purge - desc: Begin testing from known state. - DELETE: /api/v1.0/revisions - status: 204 - response_headers: null - - - name: create_schema - desc: Add example schema - PUT: /api/v1.0/buckets/mop/documents - status: 200 - data: <@resources/sample-schema.yaml - - - name: verify_schema_is_valid - desc: Check schema validation of the added schema - GET: /api/v1.0/revisions/$HISTORY['create_schema'].$RESPONSE['$.[0].status.revision']/validations/deckhand-schema-validation - status: 200 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].count: 1 - $.[0].results[0].id: 0 - $.[0].results[0].status: success - - - name: verify_schema_validation_in_list_view - desc: Check schema validation success shows in list view - GET: /api/v1.0/revisions/$HISTORY['create_schema'].$RESPONSE['$.[0].status.revision']/validations - status: 200 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].count: 1 - $.[0].results[0].name: deckhand-schema-validation - $.[0].results[0].status: success - - - name: add_valid_document - desc: Add a document that follows the schema - PUT: /api/v1.0/buckets/good/documents - status: 200 - data: |- - --- - schema: deckhand/LayeringPolicy/v1 - metadata: - schema: metadata/Control/v1 - name: layering-policy - data: - layerOrder: - - site - --- - schema: example/Doc/v1 - metadata: - schema: metadata/Document/v1 - name: good - storagePolicy: cleartext - layeringDefinition: - abstract: false - layer: site - data: - a: this-one-is-required - b: 77 - - - name: verify_document_is_valid_pre_validation - desc: Check schema pre-validation of the added document - GET: /api/v1.0/revisions/$HISTORY['add_valid_document'].$RESPONSE['$.[0].status.revision']/validations/deckhand-schema-validation - status: 200 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].count: 2 - $.[0].results[0].id: 0 - $.[0].results[0].status: success - - - name: verify_document_pre_validation_success_in_list_view - desc: Check document pre-validation success shows in list view - GET: /api/v1.0/revisions/$HISTORY['add_valid_document'].$RESPONSE['$.[0].status.revision']/validations - status: 200 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].count: 1 - $.[0].results[*].name: deckhand-schema-validation - $.[0].results[*].status: success - - - name: verify_document_is_valid_post_validation - desc: Check that the document passes post-validation - GET: /api/v1.0/revisions/$HISTORY['add_valid_document'].$RESPONSE['$.[0].status.revision']/rendered-documents - status: 200 - - - name: add_invalid_document - desc: Add a document that does not follow the schema - PUT: /api/v1.0/buckets/bad/documents - status: 200 - data: |- - schema: example/Doc/v1 - metadata: - schema: metadata/Document/v1 - name: bad - storagePolicy: cleartext - layeringDefinition: - abstract: false - layer: site - data: - a: this-one-is-required-and-can-be-different - b: 177 - - - name: verify_invalid_document_is_valid_pre_validation - desc: Check success of schema pre-validation of the added document - GET: /api/v1.0/revisions/$HISTORY['add_invalid_document'].$RESPONSE['$.[0].status.revision']/validations/deckhand-schema-validation - status: 200 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].count: 1 - $.[0].results[*].status: success - - - name: verify_document_pre_validation_failure_in_list_view - desc: Check document pre-validation success shows in list view - GET: /api/v1.0/revisions/$HISTORY['add_invalid_document'].$RESPONSE['$.[0].status.revision']/validations - status: 200 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].count: 1 - $.[0].results[0].name: deckhand-schema-validation - $.[0].results[0].status: success - - - name: verify_document_is_invalid_post_validation - desc: Check that the document fails post-validation - GET: /api/v1.0/revisions/$HISTORY['add_invalid_document'].$RESPONSE['$.[0].status.revision']/rendered-documents - status: 400 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].apiVersion: v1.0 - $.[0].code: 400 Bad Request - $.[0].details.errorCount: 1 - $.[0].details.errorType: InvalidDocumentFormat - $.[0].details.messageList[0].documents: - - layer: site - name: bad - schema: example/Doc/v1 - $.[0].details.messageList[0].error: true - $.[0].details.messageList[0].kind: ValidationMessage - $.[0].details.messageList[0].level: Error - $.[0].details.messageList[0].name: D002 - $.[0].kind: Status - $.[0].message: The provided documents failed schema validation - $.[0].reason: Validation - $.[0].status: Failure - - - name: add_invalid_document_with_substitutions - desc: Add a document that does not follow the schema - PUT: /api/v1.0/buckets/bad/documents - status: 200 - data: |- - --- - schema: example/Doc/v1 - metadata: - schema: metadata/Document/v1 - name: bad - storagePolicy: cleartext - layeringDefinition: - abstract: false - layer: site - substitutions: - - src: - schema: deckhand/Certificate/v1 - name: test-certificate - path: . - dest: - path: .a - data: - a: this-one-is-required-and-can-be-different - b: 177 - --- - schema: deckhand/Certificate/v1 - metadata: - name: test-certificate - schema: metadata/Document/v1 - storagePolicy: cleartext - layeringDefinition: - layer: site - storagePolicy: cleartext - data: this-should-definitely-be-sanitized - - - name: verify_document_post_validation_failure_entry_details_hides_secrets - desc: Check document validation failure hides secrets - GET: /api/v1.0/revisions/$HISTORY['add_invalid_document_with_substitutions'].$RESPONSE['$.[0].status.revision']/rendered-documents - status: 400 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].code: 400 Bad Request - $.[0].details.errorCount: 1 - $.[0].details.errorType: InvalidDocumentFormat - $.[0].details.messageList[0].diagnostic.error_section: - a: 'Sanitized to avoid exposing secret.' - b: 177 diff --git a/deckhand/tests/functional/gabbits/schema-validation/schema-validation-with-validation-policy-success.yaml b/deckhand/tests/functional/gabbits/schema-validation/schema-validation-with-validation-policy-success.yaml deleted file mode 100644 index 433e4363..00000000 --- a/deckhand/tests/functional/gabbits/schema-validation/schema-validation-with-validation-policy-success.yaml +++ /dev/null @@ -1,238 +0,0 @@ -# Test success path Validations API in conjunction with Validation Policy. -# -# 1. Purge existing data to ensure test isolation -# 2. Create a Validation Policy. -# 3. Verifies that the revision is success. -# 4. Creates a Validation Policy with two validations, one which requires an -# an external validation result from Promenade. -# 5. Register success result for promenade-schema-validation. -# 6. Validate that the revision is success. -# 7. Create a Validation Policy with two validations, same as before. -# 8. Intentionally do not register promenade-schema-validation. -# 9. Validate that promenade-schema-validation defaults to failure for -# get and list views. -# 10. Create a Validation Policy with one validation -# (deckhand-schema-validation) only. -# 11. Register extra promenade-schema-validation (not included in VP) as -# failure. -# 12. Validate that promenade-schema-validation is ignored for -# get and list views. -# 13. Validate that detail view for promenade-schema-validation indicates it -# is ignored, with detailed message. - -defaults: - request_headers: - content-type: application/x-yaml - response_headers: - content-type: application/x-yaml - verbose: true - -tests: - - name: purge - desc: Begin testing from known state. - DELETE: /api/v1.0/revisions - status: 204 - response_headers: null - - - name: create_validation_policy - desc: Create an example validation policy. - PUT: /api/v1.0/buckets/mop/documents - status: 200 - data: |- - --- - schema: deckhand/ValidationPolicy/v1 - metadata: - schema: metadata/Control/v1 - name: site-deploy-ready - layeringDefinition: - abstract: true - data: - validations: - - name: deckhand-schema-validation - ... - - - name: verify_revision_is_success - desc: | - Validate that the Validation Policy itself passed - deckhand-schema-validation. - GET: /api/v1.0/revisions/$HISTORY['create_validation_policy'].$RESPONSE['$.[0].status.revision']/validations/deckhand-schema-validation - status: 200 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].count: 1 - $.[0].results[0].id: 0 - $.[0].results[0].status: success - - - name: create_vp_two_validations - desc: Add a validation policy with two validations. - PUT: /api/v1.0/buckets/mop/documents - status: 200 - data: |- - --- - schema: deckhand/ValidationPolicy/v1 - metadata: - schema: metadata/Control/v1 - name: site-deploy-ready - layeringDefinition: - abstract: true - data: - validations: - - name: deckhand-schema-validation - - name: promenade-schema-validation - ... - - - name: register_promenade_validation_as_success - desc: Externally register the result for promenade-schema-validation. - POST: /api/v1.0/revisions/$HISTORY['create_vp_two_validations'].$RESPONSE['$.[0].status.revision']/validations/promenade-schema-validation - status: 201 - data: |- - --- - status: success - errors: [] - validator: - name: promenade - version: 1.1.2 - ... - - - name: verify_revision_with_two_validations_list_view_is_success - desc: Verify that both validations are marked as success. - GET: /api/v1.0/revisions/$HISTORY['create_vp_two_validations'].$RESPONSE['$.[0].status.revision']/validations - status: 200 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].count: 2 - # Sort the results by name alphabetically for reliable assertions. - $.[0].results[/name][0].name: deckhand-schema-validation - $.[0].results[/name][0].status: success - $.[0].results[/name][1].name: promenade-schema-validation - $.[0].results[/name][1].status: success - - - name: create_vp_two_validations_missing - desc: Add a validation policy with two validations. - PUT: /api/v1.0/buckets/mop/documents - status: 200 - data: |- - --- - schema: deckhand/ValidationPolicy/v1 - metadata: - schema: metadata/Control/v1 - name: site-deploy-ready - layeringDefinition: - abstract: true - data: - validations: - - name: deckhand-schema-validation - - name: promenade-schema-validation - ... - - # Do not register promenade-schema-validation. - - - name: verify_promenade_schema_validation_is_failure - desc: | - Verify that promenade-schema-validation status is failure because - it should default to failure as it was never externally registered. - GET: /api/v1.0/revisions/$HISTORY['create_vp_two_validations_missing'].$RESPONSE['$.[0].status.revision']/validations/promenade-schema-validation - status: 200 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].count: 1 - $.[0].results[0].id: 0 - $.[0].results[0].status: failure - - - name: verify_revision_with_missing_validation_list_view_is_failure - desc: Verify promenade-schema-validation is failure. - GET: /api/v1.0/revisions/$HISTORY['create_vp_two_validations_missing'].$RESPONSE['$.[0].status.revision']/validations - status: 200 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].count: 2 - # Sort the results by name alphabetically for reliable assertions. - $.[0].results[/name][0].name: deckhand-schema-validation - $.[0].results[/name][0].status: success - $.[0].results[/name][1].name: promenade-schema-validation - $.[0].results[/name][1].status: failure - - - name: create_vp_one_validation_extra - desc: | - Add a validation policy with only one validation but later on register - an extra validation. - PUT: /api/v1.0/buckets/mop/documents - status: 200 - data: |- - --- - schema: deckhand/ValidationPolicy/v1 - metadata: - schema: metadata/Control/v1 - name: site-deploy-ready - layeringDefinition: - abstract: true - data: - validations: - - name: deckhand-schema-validation - ... - - - name: register_extra_promenade_validation_as_failure - desc: | - Externally register the result for extra promenade-schema-validation as - failure. - POST: /api/v1.0/revisions/$HISTORY['create_vp_one_validation_extra'].$RESPONSE['$.[0].status.revision']/validations/promenade-schema-validation - status: 201 - data: |- - --- - status: failure - errors: - - documents: - - schema: promenade/Node/v1 - name: node-document-name - - schema: promenade/Masters/v1 - name: kubernetes-masters - message: Node has master role, but not included in cluster masters list. - validator: - name: promenade - version: 1.1.2 - ... - - - name: verify_promenade_schema_validation_is_ignored - desc: | - Verify that promenade-schema-validation is ignored. - GET: /api/v1.0/revisions/$HISTORY['create_vp_one_validation_extra'].$RESPONSE['$.[0].status.revision']/validations/promenade-schema-validation - status: 200 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].count: 1 - $.[0].results[0].id: 0 - $.[0].results[0].status: ignored [failure] - - - name: verify_revision_with_extra_validation_list_view_is_ignored - desc: | - Verify that the list view shows that promenade-schema-validation is - ignored. - GET: /api/v1.0/revisions/$HISTORY['create_vp_one_validation_extra'].$RESPONSE['$.[0].status.revision']/validations - status: 200 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].count: 2 - # Sort the results by name alphabetically for reliable assertions. - $.[0].results[/name][0].name: deckhand-schema-validation - $.[0].results[/name][0].status: success - $.[0].results[/name][1].name: promenade-schema-validation - $.[0].results[/name][1].status: ignored [failure] - - - name: verify_revision_with_extra_validation_detail_view_is_ignored - desc: | - Verify that details view for promenade-schema-validation indicates it - has been ignored, with a detailed message. - GET: /api/v1.0/revisions/$HISTORY['create_vp_one_validation_extra'].$RESPONSE['$.[0].status.revision']/validations/promenade-schema-validation/entries/0 - status: 200 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].name: promenade-schema-validation - $.[0].status: ignored [failure] - $.[0].createdAt: null - $.[0].expiresAfter: null - $.[0].errors.`len`: 2 - $.[0].errors[/message][1].message: >- - The result for this validation was externally registered but has been - ignored because it is not found in the validations for - ValidationPolicy [deckhand/ValidationPolicy/v1, None] - site-deploy-ready: deckhand-schema-validation. diff --git a/deckhand/tests/unit/control/test_validations_controller.py b/deckhand/tests/unit/control/test_validations_controller.py deleted file mode 100644 index 15bcb7c0..00000000 --- a/deckhand/tests/unit/control/test_validations_controller.py +++ /dev/null @@ -1,1246 +0,0 @@ -# Copyright 2017 AT&T Intellectual Property. All other rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import copy -import yaml - -import mock -from oslo_config import cfg - -from deckhand.control import buckets -from deckhand.engine import document_validation -from deckhand import factories -from deckhand.tests import test_utils -from deckhand.tests.unit.control import base as test_base -from deckhand import types - -CONF = cfg.CONF - - -VALIDATION_FAILURE_RESULT = """ ---- -status: failure -errors: - - documents: - - schema: promenade/Node/v1 - name: node-document-name - - schema: promenade/Masters/v1 - name: kubernetes-masters - message: Node has master role, but not included in cluster masters list. -validator: - name: promenade - version: 1.1.2 -""" - -VALIDATION_SUCCESS_RESULT = """ ---- -status: success -errors: [] -validator: - name: promenade - version: 1.1.2 -""" - - -class BaseValidationsControllerTest(test_base.BaseControllerTest): - - def _create_revision(self, payload=None): - if not payload: - documents_factory = factories.DocumentFactory(1, [1]) - payload = documents_factory.gen_test({}) - data_schema_factory = factories.DataSchemaFactory() - data_schema = data_schema_factory.gen_test( - payload[1]['schema'], data={}) - payload.append(data_schema) - - resp = self.app.simulate_put( - '/api/v1.0/buckets/mop/documents', - headers={'Content-Type': 'application/x-yaml'}, - body=yaml.safe_dump_all(payload)) - self.assertEqual(200, resp.status_code) - revision_id = list(yaml.safe_load_all(resp.text))[0]['status'][ - 'revision'] - return revision_id - - def _create_validation(self, revision_id, validation_name, policy): - resp = self.app.simulate_post( - '/api/v1.0/revisions/%s/validations/%s' % (revision_id, - validation_name), - headers={'Content-Type': 'application/x-yaml'}, body=policy) - return resp - - def _monkey_patch_document_validation(self): - """Workaround for testing complex validation scenarios by forcibly - passing in `pre_validate=False`. - """ - # TODO(felipemonteiro): Remove this workaround by testing these more - # complex scenarios against the rendered-documents endpoint instead - # (which performs post-validation). - original_document_validation = document_validation.DocumentValidation - - def monkey_patch(*args, **kwargs): - return original_document_validation(*args, pre_validate=False) - - mock.patch.object(buckets.document_validation, 'DocumentValidation', - side_effect=monkey_patch, autospec=True).start() - self.addCleanup(mock.patch.stopall) - - -class TestValidationsController(BaseValidationsControllerTest): - """Test suite for validating Validations API.""" - - def test_create_validation(self): - rules = {'deckhand:create_cleartext_documents': '@', - 'deckhand:create_validation': '@'} - self.policy.set_rules(rules) - - revision_id = self._create_revision() - validation_name = test_utils.rand_name('validation') - resp = self._create_validation(revision_id, validation_name, - VALIDATION_FAILURE_RESULT) - - self.assertEqual(201, resp.status_code) - expected_body = { - 'status': 'failure', - 'validator': { - 'name': 'promenade', - 'version': '1.1.2' - } - } - self.assertEqual(expected_body, yaml.safe_load(resp.text)) - - def test_list_validations(self): - rules = {'deckhand:create_cleartext_documents': '@', - 'deckhand:list_validations': '@'} - self.policy.set_rules(rules) - - revision_id = self._create_revision() - - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations' % revision_id, - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - - # Validate that the internal deckhand validation was created already. - body = list(yaml.safe_load_all(resp.text)) - expected = { - 'count': 1, - 'results': [ - { - 'status': 'success', - 'name': types.DECKHAND_SCHEMA_VALIDATION - } - ] - } - self.assertEqual(1, len(body)) - self.assertEqual(expected, body[0]) - - rules = {'deckhand:create_cleartext_documents': '@', - 'deckhand:create_validation': '@', - 'deckhand:list_validations': '@'} - self.policy.set_rules(rules) - - # Validate that, after creating a validation policy by an external - # service, it is listed as well. - validation_name = test_utils.rand_name('validation') - resp = self._create_validation(revision_id, validation_name, - VALIDATION_FAILURE_RESULT) - - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations' % revision_id, - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - - body = yaml.safe_load(resp.text) - expected_body = { - 'count': 2, - 'results': [ - { - 'name': types.DECKHAND_SCHEMA_VALIDATION, - 'status': 'success' - }, - { - 'name': validation_name, - 'status': 'failure' - } - ] - } - body['results'] = sorted(body['results'], key=lambda x: x['name']) - self.assertEqual(expected_body, body) - - def test_list_validation_entries(self): - rules = {'deckhand:create_cleartext_documents': '@', - 'deckhand:create_validation': '@', - 'deckhand:list_validations': '@'} - self.policy.set_rules(rules) - - revision_id = self._create_revision() - - # Validate that 3 entries (1 for each of the 3 documents created) - # exists for: - # /api/v1.0/revisions/1/validations/deckhand-schema-validation - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations/%s' % ( - revision_id, types.DECKHAND_SCHEMA_VALIDATION), - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - body = yaml.safe_load(resp.text) - expected_body = { - 'count': 3, - 'results': [{'id': x, 'status': 'success'} for x in range(3)] - } - self.assertEqual(expected_body, body) - - # Add the result of a validation to a revision. - validation_name = test_utils.rand_name('validation') - resp = self._create_validation(revision_id, validation_name, - VALIDATION_FAILURE_RESULT) - - # Validate that the entry is present. - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations/%s' % (revision_id, - validation_name), - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - - body = yaml.safe_load(resp.text) - expected_body = { - 'count': 1, - 'results': [{'id': 0, 'status': 'failure'}] - } - self.assertEqual(expected_body, body) - - def test_list_validation_entries_after_creating_validation(self): - rules = {'deckhand:create_cleartext_documents': '@', - 'deckhand:create_validation': '@', - 'deckhand:list_validations': '@'} - self.policy.set_rules(rules) - - revision_id = self._create_revision() - - # Add the result of a validation to a revision. - validation_name = test_utils.rand_name('validation') - resp = self._create_validation(revision_id, validation_name, - VALIDATION_FAILURE_RESULT) - - # Validate that the entry is present. - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations/%s' % (revision_id, - validation_name), - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - - body = yaml.safe_load(resp.text) - expected_body = { - 'count': 1, - 'results': [{'id': 0, 'status': 'failure'}] - } - self.assertEqual(expected_body, body) - - # Add the result of another validation to the same revision. - resp = self._create_validation(revision_id, validation_name, - VALIDATION_SUCCESS_RESULT) - - # Validate that 2 entries now exist. - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations/%s' % (revision_id, - validation_name), - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - body = yaml.safe_load(resp.text) - expected_body = { - 'count': 2, - 'results': [ - {'id': 0, 'status': 'failure'}, {'id': 1, 'status': 'success'} - ] - } - self.assertEqual(expected_body, body) - - def test_list_validation_entries_with_multiple_entries(self): - rules = {'deckhand:create_cleartext_documents': '@', - 'deckhand:create_validation': '@', - 'deckhand:list_validations': '@'} - self.policy.set_rules(rules) - - revision_id = self._create_revision() - validation_name = test_utils.rand_name('validation') - self._create_validation(revision_id, validation_name, - VALIDATION_FAILURE_RESULT) - self._create_validation(revision_id, validation_name, - VALIDATION_SUCCESS_RESULT) - - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations/%s' % (revision_id, - validation_name), - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - - body = yaml.safe_load(resp.text) - expected_body = { - 'count': 2, - 'results': [ - {'id': 0, 'status': 'failure'}, {'id': 1, 'status': 'success'} - ] - } - self.assertEqual(expected_body, body) - - def test_show_validation_entry(self): - rules = {'deckhand:create_cleartext_documents': '@', - 'deckhand:create_validation': '@', - 'deckhand:show_validation': '@'} - self.policy.set_rules(rules) - - revision_id = self._create_revision() - validation_name = test_utils.rand_name('validation') - resp = self._create_validation(revision_id, validation_name, - VALIDATION_FAILURE_RESULT) - - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations/%s/entries/0' % ( - revision_id, validation_name), - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - - body = yaml.safe_load(resp.text) - expected_body = { - 'name': validation_name, - 'status': 'failure', - 'createdAt': None, - 'expiresAfter': None, - 'errors': [ - { - 'documents': [ - { - 'name': 'node-document-name', - 'schema': 'promenade/Node/v1' - }, { - 'name': 'kubernetes-masters', - 'schema': 'promenade/Masters/v1' - } - ], - 'message': 'Node has master role, but not included in ' - 'cluster masters list.' - } - ] - } - self.assertEqual(expected_body, body) - - def test_show_nonexistent_validation_entry_returns_404(self): - rules = {'deckhand:create_cleartext_documents': '@', - 'deckhand:create_validation': '@', - 'deckhand:show_validation': '@'} - self.policy.set_rules(rules) - - revision_id = self._create_revision() - validation_name = test_utils.rand_name('validation') - resp = self._create_validation(revision_id, validation_name, - VALIDATION_FAILURE_RESULT) - self.assertEqual(201, resp.status_code) - expected_error = ('The requested validation entry 5 was not found for ' - 'validation name %s and revision ID %d' % ( - validation_name, revision_id)) - - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations/%s/entries/5' % ( - revision_id, validation_name), - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(404, resp.status_code) - self.assertEqual(expected_error, yaml.safe_load(resp.text)['message']) - - def test_list_validations_details(self): - rules = {'deckhand:create_cleartext_documents': '@', - 'deckhand:list_validations': '@'} - self.policy.set_rules(rules) - - revision_id = self._create_revision() - - # Validate that 3 entries (1 for each of the 3 documents created) - # exists for - # /api/v1.0/revisions/1/validations/deckhand-schema-validation - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations/detail' % revision_id, - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - body = yaml.safe_load(resp.text) - expected_body = { - 'results': [{ - 'createdAt': None, - 'errors': [], - 'expiresAfter': None, - 'id': idx, - 'name': 'deckhand-schema-validation', - 'status': 'success' - - } for idx in range(3)], - 'count': 3 - } - self.assertEqual(expected_body, body) - - -class TestValidationsControllerPreValidate(BaseValidationsControllerTest): - """Test suite for validating positive scenarios for pre-validations with - Validations controller. - """ - - def test_pre_validate_flag_skips_registered_dataschema_validations(self): - rules = {'deckhand:create_cleartext_documents': '@', - 'deckhand:list_validations': '@'} - self.policy.set_rules(rules) - - # Create a `DataSchema` against which the test document will be - # validated. - data_schema_factory = factories.DataSchemaFactory() - metadata_name = 'example/foo/v1' - schema_to_use = { - '$schema': 'http://json-schema.org/schema#', - 'type': 'object', - 'properties': { - 'a': { - 'type': 'integer' # Test doc will fail b/c of wrong type. - } - }, - 'required': ['a'] - } - data_schema = data_schema_factory.gen_test( - metadata_name, data=schema_to_use) - - # Create a document that passes validation and another that fails it. - doc_factory = factories.DocumentFactory(1, [1]) - fail_doc = doc_factory.gen_test( - {'_GLOBAL_DATA_1_': {'data': {'a': 'fail'}}}, - global_abstract=False)[-1] - fail_doc['schema'] = 'example/foo/v1' - fail_doc['metadata']['name'] = 'test_doc' - - revision_id = self._create_revision(payload=[data_schema, fail_doc]) - - # Validate that the validation reports success because `fail_doc` - # isn't validated by the `DataSchema`. - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations' % revision_id, - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - body = yaml.safe_load(resp.text) - expected_body = { - 'count': 1, - 'results': [ - {'name': types.DECKHAND_SCHEMA_VALIDATION, 'status': 'success'} - ] - } - self.assertEqual(expected_body, body) - - -class TestValidationsControllerPostValidate(BaseValidationsControllerTest): - """Test suite for validating positive scenarios for post-validations with - Validations controller. - """ - - def setUp(self): - super(TestValidationsControllerPostValidate, self).setUp() - self._monkey_patch_document_validation() - - def test_validation_with_registered_data_schema(self): - rules = {'deckhand:create_cleartext_documents': '@', - 'deckhand:list_validations': '@'} - self.policy.set_rules(rules) - - # Create a `DataSchema` against which the test document will be - # validated. - data_schema_factory = factories.DataSchemaFactory() - metadata_name = 'example/Doc/v1' - schema_to_use = { - '$schema': 'http://json-schema.org/schema#', - 'type': 'object', - 'properties': { - 'a': { - 'type': 'string' - } - }, - 'required': ['a'], - 'additionalProperties': False - } - data_schema = data_schema_factory.gen_test( - metadata_name, data=schema_to_use) - - # Create the test document whose data section adheres to the - # `DataSchema` above. - doc_factory = factories.DocumentFactory(1, [1]) - doc_to_test = doc_factory.gen_test( - {'_GLOBAL_DATA_1_': {'data': {'a': 'whatever'}}}, - global_abstract=False)[-1] - doc_to_test['schema'] = 'example/Doc/v1' - - revision_id = self._create_revision(payload=[doc_to_test, data_schema]) - - # Validate that the validation was created and succeeded. - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations' % revision_id, - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - body = yaml.safe_load(resp.text) - expected_body = { - 'count': 1, - 'results': [ - {'name': types.DECKHAND_SCHEMA_VALIDATION, 'status': 'success'} - ] - } - self.assertEqual(expected_body, body) - - def test_validation_data_schema_different_revision_expect_failure(self): - """Validates that creating a ``DataSchema`` in one revision and then - creating a document in another revision that relies on the previously - created ``DataSchema`` results in an expected failure. - """ - rules = {'deckhand:create_cleartext_documents': '@', - 'deckhand:list_validations': '@', - 'deckhand:list_cleartext_documents': '@', - 'deckhand:list_encrypted_documents': '@'} - self.policy.set_rules(rules) - - # Create a `DataSchema` against which the test document will be - # validated. - data_schema_factory = factories.DataSchemaFactory() - metadata_name = 'example/foo/v1' - schema_to_use = { - '$schema': 'http://json-schema.org/schema#', - 'type': 'object', - 'properties': { - 'a': { - 'type': 'integer' # Test doc will fail b/c of wrong type. - } - }, - 'required': ['a'] - } - data_schema = data_schema_factory.gen_test( - metadata_name, data=schema_to_use) - revision_id = self._create_revision(payload=[data_schema]) - - # Validate that the internal deckhand validation was created. - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations' % revision_id, - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - body = yaml.safe_load(resp.text) - expected_body = { - 'count': 1, - 'results': [ - {'name': types.DECKHAND_SCHEMA_VALIDATION, 'status': 'success'} - ] - } - self.assertEqual(expected_body, body) - - # Create the test document that fails the validation due to the - # schema defined by the `DataSchema` document. - doc_factory = factories.DocumentFactory(1, [1]) - docs_to_test = doc_factory.gen_test( - {'_GLOBAL_DATA_1_': {'data': {'a': 'fail'}}}, - global_abstract=False) - docs_to_test[1]['schema'] = 'example/foo/v1' - docs_to_test[1]['metadata']['name'] = 'test_doc' - - revision_id = self._create_revision( - payload=docs_to_test + [data_schema]) - - # Validate that the validation was created and reports failure. - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations' % revision_id, - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - body = yaml.safe_load(resp.text) - expected_body = { - 'count': 1, - 'results': [ - {'name': types.DECKHAND_SCHEMA_VALIDATION, 'status': 'failure'} - ] - } - self.assertEqual(expected_body, body) - - # Validate that the validation was created and reports failure. - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/rendered-documents' % revision_id, - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(400, resp.status_code) - - def test_validation_data_schema_same_revision_expect_failure(self): - """Validates that creating a ``DataSchema`` alongside a document - that relies on it in the same revision results in an expected failure. - """ - rules = {'deckhand:create_cleartext_documents': '@', - 'deckhand:list_validations': '@'} - self.policy.set_rules(rules) - - # Create a `DataSchema` against which the test document will be - # validated. - data_schema_factory = factories.DataSchemaFactory() - metadata_name = 'example/foo/v1' - schema_to_use = { - '$schema': 'http://json-schema.org/schema#', - 'type': 'object', - 'properties': { - 'a': { - 'type': 'integer' # Test doc will fail b/c of wrong type. - } - }, - 'required': ['a'] - } - data_schema = data_schema_factory.gen_test( - metadata_name, data=schema_to_use) - - # Create the test document that fails the validation due to the - # schema defined by the `DataSchema` document. - doc_factory = factories.DocumentFactory(1, [1]) - doc_to_test = doc_factory.gen_test( - {'_GLOBAL_DATA_1_': {'data': {'a': 'fail'}}}, - global_abstract=False)[-1] - doc_to_test['schema'] = 'example/foo/v1' - doc_to_test['metadata']['name'] = 'test_doc' - - revision_id = self._create_revision(payload=[doc_to_test, data_schema]) - - # Validate that the validation was created and reports failure. - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations' % revision_id, - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - body = yaml.safe_load(resp.text) - expected_body = { - 'count': 1, - 'results': [ - {'name': types.DECKHAND_SCHEMA_VALIDATION, 'status': 'failure'} - ] - } - self.assertEqual(expected_body, body) - - def test_validation_with_registered_data_schema_expect_multi_failure(self): - rules = {'deckhand:create_cleartext_documents': '@', - 'deckhand:list_validations': '@', - 'deckhand:show_validation': '@'} - self.policy.set_rules(rules) - - # Create a `DataSchema` against which the test document will be - # validated. - data_schema_factory = factories.DataSchemaFactory() - metadata_name = 'example/foo/v1' - schema_to_use = { - '$schema': 'http://json-schema.org/schema#', - 'type': 'object', - 'properties': { - 'a': { - 'type': 'integer' # Test doc will fail b/c of wrong type. - }, - 'b': { - 'type': 'string' - } - }, - 'required': ['a', 'b'] - } - data_schema = data_schema_factory.gen_test( - metadata_name, data=schema_to_use) - - # Failure #1: Provide wrong type for property "a". - # Failure #2: Don't include required property "b". - doc_factory = factories.DocumentFactory(1, [1]) - doc_to_test = doc_factory.gen_test( - {'_GLOBAL_DATA_1_': {'data': {'a': 'fail'}}}, - global_abstract=False)[-1] - doc_to_test['schema'] = 'example/foo/v1' - doc_to_test['metadata']['name'] = 'test_doc' - - revision_id = self._create_revision(payload=[doc_to_test, data_schema]) - - # Validate that the validation was created and reports failure. - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations' % revision_id, - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - body = yaml.safe_load(resp.text) - expected_body = { - 'count': 1, - 'results': [ - {'name': types.DECKHAND_SCHEMA_VALIDATION, 'status': 'failure'} - ] - } - self.assertEqual(expected_body, body) - - # Validate that both expected errors are present for validation. - expected_errors = [{ - 'error_section': { - 'data': {'a': 'fail'}, - 'metadata': {'labels': {'global': 'global1'}, - 'storagePolicy': 'cleartext', - 'layeringDefinition': {'abstract': False, - 'layer': 'global'}, - 'name': doc_to_test['metadata']['name'], - 'schema': doc_to_test['metadata']['schema']}, - 'schema': doc_to_test['schema'] - }, - 'name': 'test_doc', - 'layer': 'global', - 'path': '.data', - 'schema': 'example/foo/v1', - 'message': "'b' is a required property", - 'validation_schema': schema_to_use, - 'schema_path': '.required' - }, { - 'error_section': {'a': 'fail'}, - 'name': 'test_doc', - 'layer': 'global', - 'path': '.data.a', - 'schema': 'example/foo/v1', - 'message': "'fail' is not of type 'integer'", - 'validation_schema': schema_to_use, - 'schema_path': '.properties.a.type' - }] - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations/%s/entries/0' % ( - revision_id, types.DECKHAND_SCHEMA_VALIDATION), - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - body = yaml.safe_load(resp.text) - - self.assertEqual('failure', body['status']) - self.assertEqual(sorted(expected_errors, key=lambda x: x['path']), - sorted(body['errors'], key=lambda x: x['path'])) - - def test_validation_with_registered_data_schema_expect_mixed(self): - rules = {'deckhand:create_cleartext_documents': '@', - 'deckhand:list_validations': '@', - 'deckhand:show_validation': '@'} - self.policy.set_rules(rules) - - # Create a `DataSchema` against which the test document will be - # validated. - data_schema_factory = factories.DataSchemaFactory() - metadata_name = 'example/foo/v1' - schema_to_use = { - '$schema': 'http://json-schema.org/schema#', - 'type': 'object', - 'properties': { - 'a': { - 'type': 'integer' # Test doc will fail b/c of wrong type. - } - }, - 'required': ['a'] - } - - expected_errors = [{ - 'error_section': {'a': 'fail'}, - 'name': 'fail_doc', - 'layer': 'global', - 'path': '.data.a', - 'schema': 'example/foo/v1', - 'message': "'fail' is not of type 'integer'", - 'validation_schema': schema_to_use, - 'schema_path': '.properties.a.type' - }] - - data_schema = data_schema_factory.gen_test( - metadata_name, data=schema_to_use) - - # Create a document that passes validation and another that fails it. - doc_factory = factories.DocumentFactory(1, [1]) - fail_doc = doc_factory.gen_test( - {'_GLOBAL_DATA_1_': {'data': {'a': 'fail'}}}, - global_abstract=False)[-1] - fail_doc['schema'] = 'example/foo/v1' - fail_doc['metadata']['name'] = 'fail_doc' - - pass_doc = copy.deepcopy(fail_doc) - pass_doc['metadata']['name'] = 'pass_doc' - pass_doc['data']['a'] = 5 - - revision_id = self._create_revision( - payload=[fail_doc, pass_doc, data_schema]) - - # Validate that the validation reports failure since `fail_doc` - # should've failed validation. - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations' % revision_id, - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - body = yaml.safe_load(resp.text) - expected_body = { - 'count': 1, - 'results': [ - {'name': types.DECKHAND_SCHEMA_VALIDATION, 'status': 'failure'} - ] - } - self.assertEqual(expected_body, body) - - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations/%s' % ( - revision_id, types.DECKHAND_SCHEMA_VALIDATION), - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - body = yaml.safe_load(resp.text) - expected_body = { - 'count': 3, - 'results': [{'id': 0, 'status': 'failure'}, # fail_doc failed. - {'id': 1, 'status': 'success'}, # DataSchema passed. - {'id': 2, 'status': 'success'}] # pass_doc succeeded. - } - self.assertEqual(expected_body, body) - - # Validate that fail_doc validation failed for the expected reason. - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations/%s/entries/0' % ( - revision_id, types.DECKHAND_SCHEMA_VALIDATION), - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - body = yaml.safe_load(resp.text) - - self.assertIn('errors', body) - self.assertEqual(expected_errors, body['errors']) - - def test_document_without_data_section_ingested(self): - """Validate that a document without the data section is ingested - successfully. - """ - rules = {'deckhand:create_cleartext_documents': '@', - 'deckhand:list_validations': '@'} - self.policy.set_rules(rules) - - documents_factory = factories.DocumentFactory(1, [1]) - document = documents_factory.gen_test({}, global_abstract=False)[-1] - del document['data'] - - data_schema_factory = factories.DataSchemaFactory() - data_schema = data_schema_factory.gen_test(document['schema'], {}) - - revision_id = self._create_revision(payload=[document, data_schema]) - - # Validate that the entry is present. - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations/%s' % ( - revision_id, types.DECKHAND_SCHEMA_VALIDATION), - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - - body = yaml.safe_load(resp.text) - expected_body = { - 'count': 2, - 'results': [{'id': 0, 'status': 'success'}, # Document. - {'id': 1, 'status': 'success'}] # DataSchema. - } - self.assertEqual(expected_body, body) - - def test_validation_only_new_data_schema_registered(self): - """Validate whether newly created DataSchemas replace old DataSchemas - when it comes to validation. - """ - rules = {'deckhand:create_cleartext_documents': '@', - 'deckhand:list_validations': '@'} - self.policy.set_rules(rules) - - # Create 2 DataSchemas that will fail if they're used. These shouldn't - # be used for validation. - data_schema_factory = factories.DataSchemaFactory() - metadata_names = ['exampleA/Doc/v1', 'exampleB/Doc/v1'] - schemas_to_use = [{ - '$schema': 'http://json-schema.org/schema#', - 'type': 'object', - 'properties': { - 'a': { - 'type': 'integer' - } - }, - 'required': ['a'], - 'additionalProperties': False - }] * 2 - old_data_schemas = [ - data_schema_factory.gen_test( - metadata_names[i], data=schemas_to_use[i]) - for i in range(2) - ] - # Save the DataSchemas in the first revision. - revision_id = self._create_revision(payload=old_data_schemas) - - # Create 2 DataSchemas that will pass if they're used. These should - # be used for validation. - for schema_to_use in schemas_to_use: - schema_to_use['properties']['a']['type'] = 'string' - new_data_schemas = [ - data_schema_factory.gen_test( - metadata_names[i], data=schemas_to_use[i]) - for i in range(2) - ] - doc_factory = factories.DocumentFactory(1, [1]) - example1_doc = doc_factory.gen_test( - {'_GLOBAL_DATA_1_': {'data': {'a': 'whatever'}}}, - global_abstract=False)[-1] - example1_doc['schema'] = metadata_names[0] - example2_doc = copy.deepcopy(example1_doc) - example2_doc['schema'] = metadata_names[1] - # Save the documents that will be validated alongside the DataSchemas - # that will be used to validate them. - revision_id = self._create_revision( - payload=[example1_doc, example2_doc] + new_data_schemas) - - # Validate that the validation was created and succeeded: This means - # that the new DataSchemas were used, not the old ones. - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations' % revision_id, - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - body = yaml.safe_load(resp.text) - expected_body = { - 'count': 1, - 'results': [ - {'name': types.DECKHAND_SCHEMA_VALIDATION, 'status': 'success'} - ] - } - self.assertEqual(expected_body, body) - - -class TestValidationsControllerWithValidationPolicy( - BaseValidationsControllerTest): - - def setUp(self): - super(TestValidationsControllerWithValidationPolicy, self).setUp() - self._monkey_patch_document_validation() - - def test_validation_with_validation_policy_success(self): - rules = {'deckhand:create_cleartext_documents': '@', - 'deckhand:list_validations': '@'} - self.policy.set_rules(rules) - - # Create a `ValidationPolicy` which is used to check whether a revision - # passed all the validations. - validation_policy = yaml.safe_load(""" ---- -schema: deckhand/ValidationPolicy/v1 -metadata: - schema: metadata/Control/v1 - name: site-deploy-ready - layeringDefinition: - abstract: true - layer: site -data: - validations: - - name: deckhand-schema-validation -... -""") - revision_id = self._create_revision(payload=[validation_policy]) - - # Validate that the validation was created and reports success. - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations' % revision_id, - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - body = yaml.safe_load(resp.text) - expected_body = { - 'count': 1, - 'results': [ - {'name': 'deckhand-schema-validation', 'status': 'success'} - ] - } - self.assertEqual(expected_body, body) - - def test_with_validation_policy_external_validation(self): - """Validate that a ValidationPolicy with an externally registered - validation that is successful passes. - """ - rules = {'deckhand:create_cleartext_documents': '@', - 'deckhand:create_validation': '@', - 'deckhand:list_validations': '@'} - self.policy.set_rules(rules) - - # Create a `ValidationPolicy` which expects two validations. - validation_policy = yaml.safe_load(""" ---- -schema: deckhand/ValidationPolicy/v1 -metadata: - schema: metadata/Control/v1 - name: site-deploy-ready - layeringDefinition: - abstract: true - layer: site -data: - validations: - - name: deckhand-schema-validation - - name: promenade-schema-validation -... -""") - revision_id = self._create_revision(payload=[validation_policy]) - - # Create the external validation for "promenade-schema-validation". - resp = self._create_validation( - revision_id, 'promenade-schema-validation', - VALIDATION_SUCCESS_RESULT) - self.assertEqual(201, resp.status_code) - - # Validate that the validation was created and reports success. - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations' % revision_id, - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - body = yaml.safe_load(resp.text) - expected_body = { - 'count': 2, - 'results': [ - {'name': 'deckhand-schema-validation', 'status': 'success'}, - {'name': 'promenade-schema-validation', 'status': 'success'} - ] - } - body['results'] = sorted(body['results'], key=lambda x: x['name']) - self.assertEqual(expected_body, body) - - def test_with_multiple_validation_policy_external_validation(self): - """Validate that two ValidationPolicy documents, one that references - the internal deckhand-schema-validation, and the other which requires - an externally registered validation, produces a successful validation - result. - """ - rules = {'deckhand:create_cleartext_documents': '@', - 'deckhand:create_validation': '@', - 'deckhand:list_validations': '@'} - self.policy.set_rules(rules) - - # Create two `ValidationPolicy` documents. - validation_policies = yaml.safe_load_all(""" ---- -schema: deckhand/ValidationPolicy/v1 -metadata: - schema: metadata/Control/v1 - name: vp-1 - layeringDefinition: - abstract: true - layer: site -data: - validations: - - name: deckhand-schema-validation ---- -schema: deckhand/ValidationPolicy/v1 -metadata: - schema: metadata/Control/v1 - name: vp-2 - layeringDefinition: - abstract: true - layer: site -data: - validations: - - name: promenade-schema-validation -... -""") - revision_id = self._create_revision(payload=validation_policies) - - # Create the external validation for "promenade-schema-validation". - resp = self._create_validation( - revision_id, 'promenade-schema-validation', - VALIDATION_SUCCESS_RESULT) - self.assertEqual(201, resp.status_code) - - # Validate that the validation was created and reports success. - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations' % revision_id, - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - body = yaml.safe_load(resp.text) - expected_body = { - 'count': 2, - 'results': [ - {'name': 'deckhand-schema-validation', 'status': 'success'}, - {'name': 'promenade-schema-validation', 'status': 'success'} - ] - } - body['results'] = sorted(body['results'], key=lambda x: x['name']) - self.assertEqual(expected_body, body) - - def test_with_validation_policy_missing_external_validation(self): - """Validate that a ValidationPolicy with a missing externally - registered validation that is listed under the validations for the - ValidationPolicy defaults to "failure". - """ - rules = {'deckhand:create_cleartext_documents': '@', - 'deckhand:list_validations': '@', - 'deckhand:show_validation': '@'} - self.policy.set_rules(rules) - - # Create a `ValidationPolicy` which expects two validations but do not - # create the validation for "promenade-schema-validation". - validation_policy = yaml.safe_load(""" ---- -schema: deckhand/ValidationPolicy/v1 -metadata: - schema: metadata/Control/v1 - name: site-deploy-ready - layeringDefinition: - abstract: true - layer: site -data: - validations: - - name: deckhand-schema-validation - - name: promenade-schema-validation -... -""") - revision_id = self._create_revision(payload=[validation_policy]) - - # Validate that the validation was created and that the missing one - # defaults to "failure". - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations' % revision_id, - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - body = yaml.safe_load(resp.text) - expected_body = { - 'count': 2, - 'results': [ - {'name': 'deckhand-schema-validation', 'status': 'success'}, - {'name': 'promenade-schema-validation', 'status': 'failure'} - ] - } - body['results'] = sorted(body['results'], key=lambda x: x['name']) - self.assertEqual(expected_body, body) - - # Validate that 'promenade-schema-validation' is 'failure' even though - # it was never externally registered. - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations/%s' % ( - revision_id, 'promenade-schema-validation'), - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - body = yaml.safe_load(resp.text) - expected_body = { - 'count': 1, - 'results': [{'id': 0, 'status': 'failure'}] - } - self.assertEqual(expected_body, body) - - # Validate information explaining why 'promenade-schema-validation' - # failed is returned. Note that DH should be smart enough to say that - # it was never registered externally, which is why it's 'failure'. - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations/%s/entries/0' % ( - revision_id, 'promenade-schema-validation'), - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - body = yaml.safe_load(resp.text) - - expected_msg = ('The result for this validation was never externally ' - 'registered so its status defaulted to "failure".') - expected_body = { - 'name': 'promenade-schema-validation', - 'status': 'failure', - 'createdAt': None, - 'expiresAfter': None, - 'errors': [{'message': expected_msg}] - } - self.assertEqual(expected_body, body) - - def test_with_validation_policy_extra_external_validation(self): - """Validate that a ValidationPolicy with extra externally registered - validations that aren't listed under the validations for the - ValidationPolicy defaults to "ignored [{original_status}]". - """ - rules = {'deckhand:create_cleartext_documents': '@', - 'deckhand:create_validation': '@', - 'deckhand:list_validations': '@', - 'deckhand:show_validation': '@'} - self.policy.set_rules(rules) - - # Create a `ValidationPolicy` with only 1 validation. - validation_policy = yaml.safe_load(""" ---- -schema: deckhand/ValidationPolicy/v1 -metadata: - schema: metadata/Control/v1 - name: site-deploy-ready - layeringDefinition: - abstract: true - layer: site -data: - validations: - - name: deckhand-schema-validation -... -""") - - def _do_test(validation_result, expected_status): - revision_id = self._create_revision(payload=[validation_policy]) - - # Register an extra validation not in the ValidationPolicy. - resp = self._create_validation( - revision_id, 'promenade-schema-validation', - validation_result) - self.assertEqual(201, resp.status_code) - - # Validate that the extra validation is ignored. - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations' % revision_id, - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - body = yaml.safe_load(resp.text) - expected_body = { - 'count': 2, - 'results': [ - {'name': 'deckhand-schema-validation', - 'status': 'success'}, - {'name': 'promenade-schema-validation', - 'status': 'ignored [%s]' % expected_status} - ] - } - body['results'] = sorted(body['results'], key=lambda x: x['name']) - self.assertEqual(expected_body, body) - - # Validate that 'promenade-schema-validation' is - # 'ignored [expected_status]'. - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations/%s' % ( - revision_id, 'promenade-schema-validation'), - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - body = yaml.safe_load(resp.text) - expected_body = { - 'count': 1, - 'results': [ - {'id': 0, 'status': 'ignored [%s]' % expected_status} - ] - } - self.assertEqual(expected_body, body) - - # Validate information explaining why 'promenade-schema-validation' - # is ignored is returned. - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations/%s/entries/0' % ( - revision_id, 'promenade-schema-validation'), - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - body = yaml.safe_load(resp.text) - - expected_msg = ('The result for this validation was externally ' - 'registered but has been ignored because it is not' - ' found in the validations for ValidationPolicy ' - '[%s, %s] %s: %s.' % ( - validation_policy['schema'], - validation_policy['metadata'][ - 'layeringDefinition']['layer'], - validation_policy['metadata']['name'], - types.DECKHAND_SCHEMA_VALIDATION)) - - expected_errors = [] - if expected_status == 'failure': - expected_errors.extend( - yaml.safe_load(VALIDATION_FAILURE_RESULT)['errors']) - expected_errors.append({'message': expected_msg}) - - expected_body = { - 'name': 'promenade-schema-validation', - 'status': 'ignored [%s]' % expected_status, - 'createdAt': None, - 'expiresAfter': None, - 'errors': expected_errors - } - self.assertEqual(expected_body, body) - - _do_test(VALIDATION_SUCCESS_RESULT, 'success') - _do_test(VALIDATION_FAILURE_RESULT, 'failure') diff --git a/doc/source/users/validation.rst b/doc/source/users/validation.rst index ac47e96b..14eefc16 100644 --- a/doc/source/users/validation.rst +++ b/doc/source/users/validation.rst @@ -41,10 +41,6 @@ Here is a list of internal validations: * ``deckhand-document-schema-validation`` - All concrete documents in the revision successfully pass their JSON schema validations. Will cause this to report an error. -* ``deckhand-policy-validation`` (TODO) - All required policy documents are in-place, - and existing documents conform to those policies. E.g. if a 3rd party - document specifies a ``layer`` that is not present in the layering policy, - that will cause this validation to report an error. Externally Provided Validations ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -84,81 +80,6 @@ Schema validations are controlled by two mechanisms: document's ``data`` section to *additional* validations, validations specified by the ``data`` section of the ``DataSchema`` document. -Policy Validations ------------------- - -*Not yet implemented*. - -Validation Policies -------------------- - -Validation policies are optional. Deckhand will perform all internal and -externally registered schema validations against all documents, with or without -any Validation Policies. - -All ``ValidationPolicy`` documents in Deckhand are externally registered. They -allow services to report success or failure of named validations for a given -revision. The intended purpose is to allow a simple mapping that enables -consuming services to be able to quickly check whether the configuration in -Deckhand is in a valid state for performing a specific action. - -``ValidationPolicy`` documents are not the same as ``DataSchema`` documents. -A ``ValidationPolicy`` document can reference a list of internal Deckhand -validations in addition to externally registered ``DataSchema`` documents. -Whereas a ``DataSchema`` document specifies a new set of validations to check -against relevant documents, a ``ValidationPolicy`` is a bookkeeping device -that merely lists the set of validations in a revision that need to succeed -in order for the revision itself to be valid. - -For example, given Revision 1 which contains a ``ValidationPolicy`` of: - -:: - - --- - schema: deckhand/ValidationPolicy/v1 - metadata: - schema: metadata/Control/v1 - name: later-validation - layeringDefinition: - abstract: False - layer: site - data: - validations: - - name: deckhand-schema-validation - - name: drydock-site-validation - -Deckhand automatically creates ``deckhand-schema-validation`` as soon as the -revision itself is created. Afterward, Drydock can POST its result for -``drydock-site-validation`` using Deckhand's Validations API. Finally, Shipyard -query Deckhand's Validations API which in turn checks whether all validations -contained in the ``ValidationPolicy`` above are successful, before it proceeds -to the next stage in its workflow. - -Missing Validations -^^^^^^^^^^^^^^^^^^^ - -Validations contained in a ``ValidationPolicy`` but which were never created -in Deckhand for a given revision are considered missing. Missing validations -result in the entire validation result reporting "failure". - -If, for example, Drydock never POSTed a result for ``drydock-site-validation`` -then the Deckhand Validations API will return a "failure" result, even if -``deckhand-schema-validation`` reports "success". - -Extra Validations -^^^^^^^^^^^^^^^^^ - -Validations that are registered in Deckhand via the Validations API -but are not included in the ``ValidationPolicy`` (if one exists) for a given -revision are **ignored** (with the original status reported as -"ignored [failure]" or "ignored [success]"). - -For example, given the ``ValidationPolicy`` example above, if Promenade POSTs -``promenade-schema-validation`` with a result of "failure", then the *overall* -validation status for the given revision returned by Deckhand will be *success* -because the "failure" result from Promenade, since it was never registered, -will be ignored. - Validation Stages -----------------