From a2606e75b10d9f47f4df08b8efa79d95fea3b15f Mon Sep 17 00:00:00 2001 From: "Kumar, Nishant (nk613n)" Date: Mon, 7 Oct 2019 10:53:04 -0400 Subject: [PATCH] Remove unused code for policy validation as feature not implemented Policy validation in Deckhand was not implemented completely. Refer link below: https://airshipit.readthedocs.io/projects/deckhand/en/latest/users/validation.html#policy-validations This PS removes some of the code related to the feature which was being used in a code path when a set of documents are uploaded to Deckhand. In standard Airship deployments the number of documents could be quite high and this leads to significant delay (more than 300seconds in some cases). As there are no plans to implement the policy validation feature, it makes sense to remove it from code path which could cause delay and sometimes timeouts while uploading documents. This has been tested on a Baremetal lab: GF and BF. Change-Id: I2ff3f40a7fe37bed5a589fab00d829db726604fe --- deckhand/client/client.py | 2 - deckhand/client/validations.py | 49 - deckhand/control/buckets.py | 11 +- ...document-crud-success-owned-documents.yaml | 23 - ...cument-crud-success-unusual-documents.yaml | 28 - .../schema-validation-success-ucp-sample.yaml | 39 - .../schema-validation-success-v2.yaml | 214 --- .../schema-validation-success.yaml | 214 --- ...dation-with-validation-policy-success.yaml | 238 ---- .../control/test_validations_controller.py | 1246 ----------------- doc/source/users/validation.rst | 79 -- 11 files changed, 1 insertion(+), 2142 deletions(-) delete mode 100644 deckhand/client/validations.py delete mode 100644 deckhand/tests/functional/gabbits/schema-validation/schema-validation-success-ucp-sample.yaml delete mode 100644 deckhand/tests/functional/gabbits/schema-validation/schema-validation-success-v2.yaml delete mode 100644 deckhand/tests/functional/gabbits/schema-validation/schema-validation-success.yaml delete mode 100644 deckhand/tests/functional/gabbits/schema-validation/schema-validation-with-validation-policy-success.yaml delete mode 100644 deckhand/tests/unit/control/test_validations_controller.py diff --git a/deckhand/client/client.py b/deckhand/client/client.py index 093bce1b..24046d30 100644 --- a/deckhand/client/client.py +++ b/deckhand/client/client.py @@ -30,7 +30,6 @@ from deckhand.client import buckets from deckhand.client import exceptions from deckhand.client import revisions from deckhand.client import tags -from deckhand.client import validations class SessionClient(adapter.Adapter): @@ -203,7 +202,6 @@ class Client(object): self.buckets = buckets.BucketManager(self) self.revisions = revisions.RevisionManager(self) self.tags = tags.RevisionTagManager(self) - self.validations = validations.ValidationManager(self) self.client = _construct_http_client( api_version=api_version, diff --git a/deckhand/client/validations.py b/deckhand/client/validations.py deleted file mode 100644 index 54140808..00000000 --- a/deckhand/client/validations.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright 2017 AT&T Intellectual Property. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from deckhand.client import base - - -class Validation(base.Resource): - def __repr__(self): - return ("") - - -class ValidationManager(base.Manager): - """Manage :class:`Validation` resources.""" - resource_class = Validation - - def list(self, revision_id): - """Get list of revision validations.""" - url = '/revisions/%s/validations' % revision_id - return self._list(url) - - def list_entries(self, revision_id, validation_name): - """Get list of entries for a validation.""" - url = '/revisions/%s/validations/%s' % (revision_id, validation_name) - # Call `_get` instead of `_list` because the response from the server - # is a dict of form `{"count": n, "results": []}`. - return self._get(url) - - def get_entry(self, revision_id, validation_name, entry_id): - """Get entry details for a validation.""" - url = '/revisions/%s/validations/%s/entries/%s' % ( - revision_id, validation_name, entry_id) - return self._get(url) - - def create(self, revision_id, validation_name, data): - """Associate a validation with a revision.""" - url = '/revisions/%s/validations/%s' % (revision_id, validation_name) - return self._create(url, data=data) diff --git a/deckhand/control/buckets.py b/deckhand/control/buckets.py index 50d5b313..dc531966 100644 --- a/deckhand/control/buckets.py +++ b/deckhand/control/buckets.py @@ -48,7 +48,7 @@ class BucketsResource(api_base.BaseResource): try: doc_validator = document_validation.DocumentValidation( documents, data_schemas, pre_validate=True) - validations = doc_validator.validate_all() + doc_validator.validate_all() except deckhand_errors.InvalidDocumentFormat as e: with excutils.save_and_reraise_exception(): LOG.exception(e.format_message()) @@ -64,10 +64,6 @@ class BucketsResource(api_base.BaseResource): created_documents = self._create_revision_documents( bucket_name, documents) - if created_documents: - revision_id = created_documents[0]['revision_id'] - self._create_revision_validations(revision_id, validations) - resp.body = self.view_builder.list(created_documents) resp.status = falcon.HTTP_200 @@ -88,8 +84,3 @@ class BucketsResource(api_base.BaseResource): LOG.exception(e.format_message()) return created_documents - - def _create_revision_validations(self, revision_id, validations): - for validation in validations: - db_api.validation_create(revision_id, validation['name'], - validation) diff --git a/deckhand/tests/functional/gabbits/document/document-crud-success-owned-documents.yaml b/deckhand/tests/functional/gabbits/document/document-crud-success-owned-documents.yaml index f3da5e64..d8f1d764 100644 --- a/deckhand/tests/functional/gabbits/document/document-crud-success-owned-documents.yaml +++ b/deckhand/tests/functional/gabbits/document/document-crud-success-owned-documents.yaml @@ -65,26 +65,3 @@ tests: response_multidoc_jsonpaths: $.`len`: 1 $.[0].data: hunter2 - - - name: verify_schema_is_valid - desc: Check schema validation of the added schema - GET: /api/v1.0/revisions/$HISTORY['initialize'].$RESPONSE['$.[0].status.revision']/validations/deckhand-schema-validation - status: 200 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].results[*].status: - - success - - success - - success - - success - - success - - success - - success - - success - - success - - success - - success - - success - - success - - success - - success diff --git a/deckhand/tests/functional/gabbits/document/document-crud-success-unusual-documents.yaml b/deckhand/tests/functional/gabbits/document/document-crud-success-unusual-documents.yaml index bc736817..0aab1fa5 100644 --- a/deckhand/tests/functional/gabbits/document/document-crud-success-unusual-documents.yaml +++ b/deckhand/tests/functional/gabbits/document/document-crud-success-unusual-documents.yaml @@ -61,31 +61,3 @@ tests: response_multidoc_jsonpaths: $.`len`: 1 $.[0].data: 9000 - - - name: verify_document_validation_success_in_list_view - desc: Check document validation success shows in list view - GET: /api/v1.0/revisions/$HISTORY['initialize'].$RESPONSE['$.[0].status.revision']/validations - status: 200 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].count: 1 - $.[0].results[0].status: success - - - name: verify_document_validation_success_in_details_view - desc: Check document validation success shows in detailed view - GET: /api/v1.0/revisions/$HISTORY['initialize'].$RESPONSE['$.[0].status.revision']/validations/deckhand-schema-validation - status: 200 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].count: 9 - $.[0].results[*].status: - # 9 documents are created in total, including DataSchema documents. - - success - - success - - success - - success - - success - - success - - success - - success - - success diff --git a/deckhand/tests/functional/gabbits/schema-validation/schema-validation-success-ucp-sample.yaml b/deckhand/tests/functional/gabbits/schema-validation/schema-validation-success-ucp-sample.yaml deleted file mode 100644 index 88a09d99..00000000 --- a/deckhand/tests/functional/gabbits/schema-validation/schema-validation-success-ucp-sample.yaml +++ /dev/null @@ -1,39 +0,0 @@ -# Test success path for sample Airship documents -# -# 1. Purges existing data to ensure test isolation -# 2. Creates sample Airship documents and schemas -# 3. Checks that the documents pass schema validation - -defaults: - request_headers: - content-type: application/x-yaml - response_headers: - content-type: application/x-yaml - verbose: true - -tests: - - name: purge - desc: Begin testing from known state. - DELETE: /api/v1.0/revisions - status: 204 - response_headers: null - - - name: initialize - desc: Add example schema - PUT: /api/v1.0/buckets/mop/documents - status: 200 - data: <@resources/ucp-sample-documents.yaml - - - name: verify_schema_is_valid - desc: Check schema validation of the added schema - GET: /api/v1.0/revisions/$HISTORY['initialize'].$RESPONSE['$.[0].status.revision']/validations/deckhand-schema-validation - status: 200 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].count: 5 - $.[0].results[*].status: - - success - - success - - success - - success - - success diff --git a/deckhand/tests/functional/gabbits/schema-validation/schema-validation-success-v2.yaml b/deckhand/tests/functional/gabbits/schema-validation/schema-validation-success-v2.yaml deleted file mode 100644 index cc7d6f2a..00000000 --- a/deckhand/tests/functional/gabbits/schema-validation/schema-validation-success-v2.yaml +++ /dev/null @@ -1,214 +0,0 @@ -# Test success path for rollback with a single bucket. -# -# 1. Purges existing data to ensure test isolation -# 2. Creates a v2 DataSchema -# 3. Checks that schema validation for the DataSchema passes -# 4. Puts a valid document (and LayeringPolicy) -# 5. Checks that the document passes schema pre-validation -# 6. Checks that the document passes schema post-validation -# 7. Puts an invalid document -# 8. Checks that the document fails schema pre-validation -# 9. Checks that the document fails schema post-validation by raising expected -# exception -# 10. Checks that the document entry details adhere to expected validation -# format -# 11. Re-puts the same invalid document with substitutions -# 12. Verify that the substitutions were sanitized in the validation output - -defaults: - request_headers: - content-type: application/x-yaml - response_headers: - content-type: application/x-yaml - verbose: true - -tests: - - name: purge - desc: Begin testing from known state. - DELETE: /api/v1.0/revisions - status: 204 - response_headers: null - - - name: create_schema - desc: Add example schema - PUT: /api/v1.0/buckets/mop/documents - status: 200 - data: <@resources/sample-schema-v2.yaml - - - name: verify_schema_is_valid - desc: Check schema validation of the added schema - GET: /api/v1.0/revisions/$HISTORY['create_schema'].$RESPONSE['$.[0].status.revision']/validations/deckhand-schema-validation - status: 200 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].count: 1 - $.[0].results[0].id: 0 - $.[0].results[0].status: success - - - name: verify_schema_validation_in_list_view - desc: Check schema validation success shows in list view - GET: /api/v1.0/revisions/$HISTORY['create_schema'].$RESPONSE['$.[0].status.revision']/validations - status: 200 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].count: 1 - $.[0].results[0].name: deckhand-schema-validation - $.[0].results[0].status: success - - - name: add_valid_document - desc: Add a document that follows the schema - PUT: /api/v1.0/buckets/good/documents - status: 200 - data: |- - --- - schema: deckhand/LayeringPolicy/v1 - metadata: - schema: metadata/Control/v1 - name: layering-policy - data: - layerOrder: - - site - --- - schema: example/Doc/v2 - metadata: - schema: metadata/Document/v1 - name: good - storagePolicy: cleartext - layeringDefinition: - abstract: false - layer: site - data: - a: this-one-is-required - b: 77 - - - name: verify_document_is_valid_pre_validation - desc: Check schema pre-validation of the added document - GET: /api/v1.0/revisions/$HISTORY['add_valid_document'].$RESPONSE['$.[0].status.revision']/validations/deckhand-schema-validation - status: 200 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].count: 2 - $.[0].results[0].id: 0 - $.[0].results[0].status: success - - - name: verify_document_pre_validation_success_in_list_view - desc: Check document pre-validation success shows in list view - GET: /api/v1.0/revisions/$HISTORY['add_valid_document'].$RESPONSE['$.[0].status.revision']/validations - status: 200 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].count: 1 - $.[0].results[*].name: deckhand-schema-validation - $.[0].results[*].status: success - - - name: verify_document_is_valid_post_validation - desc: Check that the document passes post-validation - GET: /api/v1.0/revisions/$HISTORY['add_valid_document'].$RESPONSE['$.[0].status.revision']/rendered-documents - status: 200 - - - name: add_invalid_document - desc: Add a document that does not follow the schema - PUT: /api/v1.0/buckets/bad/documents - status: 200 - data: |- - schema: example/Doc/v2 - metadata: - schema: metadata/Document/v1 - name: bad - storagePolicy: cleartext - layeringDefinition: - abstract: false - layer: site - data: - a: this-one-is-required-and-can-be-different - b: 177 - - - name: verify_invalid_document_is_valid_pre_validation - desc: Check success of schema pre-validation of the added document - GET: /api/v1.0/revisions/$HISTORY['add_invalid_document'].$RESPONSE['$.[0].status.revision']/validations/deckhand-schema-validation - status: 200 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].count: 1 - $.[0].results[*].status: success - - - name: verify_document_pre_validation_failure_in_list_view - desc: Check document pre-validation success shows in list view - GET: /api/v1.0/revisions/$HISTORY['add_invalid_document'].$RESPONSE['$.[0].status.revision']/validations - status: 200 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].count: 1 - $.[0].results[0].name: deckhand-schema-validation - $.[0].results[0].status: success - - - name: verify_document_is_invalid_post_validation - desc: Check that the document fails post-validation - GET: /api/v1.0/revisions/$HISTORY['add_invalid_document'].$RESPONSE['$.[0].status.revision']/rendered-documents - status: 400 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].apiVersion: v1.0 - $.[0].code: 400 Bad Request - $.[0].details.errorCount: 1 - $.[0].details.errorType: InvalidDocumentFormat - $.[0].details.messageList[0].documents: - - layer: site - name: bad - schema: example/Doc/v2 - $.[0].details.messageList[0].error: true - $.[0].details.messageList[0].kind: ValidationMessage - $.[0].details.messageList[0].level: Error - $.[0].details.messageList[0].name: D002 - $.[0].kind: Status - $.[0].message: The provided documents failed schema validation - $.[0].reason: Validation - $.[0].status: Failure - - - name: add_invalid_document_with_substitutions - desc: Add a document that does not follow the schema - PUT: /api/v1.0/buckets/bad/documents - status: 200 - data: |- - --- - schema: example/Doc/v2 - metadata: - schema: metadata/Document/v1 - name: bad - storagePolicy: cleartext - layeringDefinition: - abstract: false - layer: site - substitutions: - - src: - schema: deckhand/Certificate/v1 - name: test-certificate - path: . - dest: - path: .a - data: - a: this-one-is-required-and-can-be-different - b: 177 - --- - schema: deckhand/Certificate/v1 - metadata: - name: test-certificate - schema: metadata/Document/v1 - storagePolicy: cleartext - layeringDefinition: - layer: site - storagePolicy: cleartext - data: this-should-definitely-be-sanitized - - - name: verify_document_post_validation_failure_entry_details_hides_secrets - desc: Check document validation failure hides secrets - GET: /api/v1.0/revisions/$HISTORY['add_invalid_document_with_substitutions'].$RESPONSE['$.[0].status.revision']/rendered-documents - status: 400 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].code: 400 Bad Request - $.[0].details.errorCount: 1 - $.[0].details.errorType: InvalidDocumentFormat - $.[0].details.messageList[0].diagnostic.error_section: - a: 'Sanitized to avoid exposing secret.' - b: 177 diff --git a/deckhand/tests/functional/gabbits/schema-validation/schema-validation-success.yaml b/deckhand/tests/functional/gabbits/schema-validation/schema-validation-success.yaml deleted file mode 100644 index da032d0c..00000000 --- a/deckhand/tests/functional/gabbits/schema-validation/schema-validation-success.yaml +++ /dev/null @@ -1,214 +0,0 @@ -# Test success path for rollback with a single bucket. -# -# 1. Purges existing data to ensure test isolation -# 2. Creates a DataSchema -# 3. Checks that schema validation for the DataSchema passes -# 4. Puts a valid document (and LayeringPolicy) -# 5. Checks that the document passes schema pre-validation -# 6. Checks that the document passes schema post-validation -# 7. Puts an invalid document -# 8. Checks that the document fails schema pre-validation -# 9. Checks that the document fails schema post-validation by raising expected -# exception -# 10. Checks that the document entry details adhere to expected validation -# format -# 11. Re-puts the same invalid document with substitutions -# 12. Verify that the substitutions were sanitized in the validation output - -defaults: - request_headers: - content-type: application/x-yaml - response_headers: - content-type: application/x-yaml - verbose: true - -tests: - - name: purge - desc: Begin testing from known state. - DELETE: /api/v1.0/revisions - status: 204 - response_headers: null - - - name: create_schema - desc: Add example schema - PUT: /api/v1.0/buckets/mop/documents - status: 200 - data: <@resources/sample-schema.yaml - - - name: verify_schema_is_valid - desc: Check schema validation of the added schema - GET: /api/v1.0/revisions/$HISTORY['create_schema'].$RESPONSE['$.[0].status.revision']/validations/deckhand-schema-validation - status: 200 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].count: 1 - $.[0].results[0].id: 0 - $.[0].results[0].status: success - - - name: verify_schema_validation_in_list_view - desc: Check schema validation success shows in list view - GET: /api/v1.0/revisions/$HISTORY['create_schema'].$RESPONSE['$.[0].status.revision']/validations - status: 200 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].count: 1 - $.[0].results[0].name: deckhand-schema-validation - $.[0].results[0].status: success - - - name: add_valid_document - desc: Add a document that follows the schema - PUT: /api/v1.0/buckets/good/documents - status: 200 - data: |- - --- - schema: deckhand/LayeringPolicy/v1 - metadata: - schema: metadata/Control/v1 - name: layering-policy - data: - layerOrder: - - site - --- - schema: example/Doc/v1 - metadata: - schema: metadata/Document/v1 - name: good - storagePolicy: cleartext - layeringDefinition: - abstract: false - layer: site - data: - a: this-one-is-required - b: 77 - - - name: verify_document_is_valid_pre_validation - desc: Check schema pre-validation of the added document - GET: /api/v1.0/revisions/$HISTORY['add_valid_document'].$RESPONSE['$.[0].status.revision']/validations/deckhand-schema-validation - status: 200 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].count: 2 - $.[0].results[0].id: 0 - $.[0].results[0].status: success - - - name: verify_document_pre_validation_success_in_list_view - desc: Check document pre-validation success shows in list view - GET: /api/v1.0/revisions/$HISTORY['add_valid_document'].$RESPONSE['$.[0].status.revision']/validations - status: 200 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].count: 1 - $.[0].results[*].name: deckhand-schema-validation - $.[0].results[*].status: success - - - name: verify_document_is_valid_post_validation - desc: Check that the document passes post-validation - GET: /api/v1.0/revisions/$HISTORY['add_valid_document'].$RESPONSE['$.[0].status.revision']/rendered-documents - status: 200 - - - name: add_invalid_document - desc: Add a document that does not follow the schema - PUT: /api/v1.0/buckets/bad/documents - status: 200 - data: |- - schema: example/Doc/v1 - metadata: - schema: metadata/Document/v1 - name: bad - storagePolicy: cleartext - layeringDefinition: - abstract: false - layer: site - data: - a: this-one-is-required-and-can-be-different - b: 177 - - - name: verify_invalid_document_is_valid_pre_validation - desc: Check success of schema pre-validation of the added document - GET: /api/v1.0/revisions/$HISTORY['add_invalid_document'].$RESPONSE['$.[0].status.revision']/validations/deckhand-schema-validation - status: 200 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].count: 1 - $.[0].results[*].status: success - - - name: verify_document_pre_validation_failure_in_list_view - desc: Check document pre-validation success shows in list view - GET: /api/v1.0/revisions/$HISTORY['add_invalid_document'].$RESPONSE['$.[0].status.revision']/validations - status: 200 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].count: 1 - $.[0].results[0].name: deckhand-schema-validation - $.[0].results[0].status: success - - - name: verify_document_is_invalid_post_validation - desc: Check that the document fails post-validation - GET: /api/v1.0/revisions/$HISTORY['add_invalid_document'].$RESPONSE['$.[0].status.revision']/rendered-documents - status: 400 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].apiVersion: v1.0 - $.[0].code: 400 Bad Request - $.[0].details.errorCount: 1 - $.[0].details.errorType: InvalidDocumentFormat - $.[0].details.messageList[0].documents: - - layer: site - name: bad - schema: example/Doc/v1 - $.[0].details.messageList[0].error: true - $.[0].details.messageList[0].kind: ValidationMessage - $.[0].details.messageList[0].level: Error - $.[0].details.messageList[0].name: D002 - $.[0].kind: Status - $.[0].message: The provided documents failed schema validation - $.[0].reason: Validation - $.[0].status: Failure - - - name: add_invalid_document_with_substitutions - desc: Add a document that does not follow the schema - PUT: /api/v1.0/buckets/bad/documents - status: 200 - data: |- - --- - schema: example/Doc/v1 - metadata: - schema: metadata/Document/v1 - name: bad - storagePolicy: cleartext - layeringDefinition: - abstract: false - layer: site - substitutions: - - src: - schema: deckhand/Certificate/v1 - name: test-certificate - path: . - dest: - path: .a - data: - a: this-one-is-required-and-can-be-different - b: 177 - --- - schema: deckhand/Certificate/v1 - metadata: - name: test-certificate - schema: metadata/Document/v1 - storagePolicy: cleartext - layeringDefinition: - layer: site - storagePolicy: cleartext - data: this-should-definitely-be-sanitized - - - name: verify_document_post_validation_failure_entry_details_hides_secrets - desc: Check document validation failure hides secrets - GET: /api/v1.0/revisions/$HISTORY['add_invalid_document_with_substitutions'].$RESPONSE['$.[0].status.revision']/rendered-documents - status: 400 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].code: 400 Bad Request - $.[0].details.errorCount: 1 - $.[0].details.errorType: InvalidDocumentFormat - $.[0].details.messageList[0].diagnostic.error_section: - a: 'Sanitized to avoid exposing secret.' - b: 177 diff --git a/deckhand/tests/functional/gabbits/schema-validation/schema-validation-with-validation-policy-success.yaml b/deckhand/tests/functional/gabbits/schema-validation/schema-validation-with-validation-policy-success.yaml deleted file mode 100644 index 433e4363..00000000 --- a/deckhand/tests/functional/gabbits/schema-validation/schema-validation-with-validation-policy-success.yaml +++ /dev/null @@ -1,238 +0,0 @@ -# Test success path Validations API in conjunction with Validation Policy. -# -# 1. Purge existing data to ensure test isolation -# 2. Create a Validation Policy. -# 3. Verifies that the revision is success. -# 4. Creates a Validation Policy with two validations, one which requires an -# an external validation result from Promenade. -# 5. Register success result for promenade-schema-validation. -# 6. Validate that the revision is success. -# 7. Create a Validation Policy with two validations, same as before. -# 8. Intentionally do not register promenade-schema-validation. -# 9. Validate that promenade-schema-validation defaults to failure for -# get and list views. -# 10. Create a Validation Policy with one validation -# (deckhand-schema-validation) only. -# 11. Register extra promenade-schema-validation (not included in VP) as -# failure. -# 12. Validate that promenade-schema-validation is ignored for -# get and list views. -# 13. Validate that detail view for promenade-schema-validation indicates it -# is ignored, with detailed message. - -defaults: - request_headers: - content-type: application/x-yaml - response_headers: - content-type: application/x-yaml - verbose: true - -tests: - - name: purge - desc: Begin testing from known state. - DELETE: /api/v1.0/revisions - status: 204 - response_headers: null - - - name: create_validation_policy - desc: Create an example validation policy. - PUT: /api/v1.0/buckets/mop/documents - status: 200 - data: |- - --- - schema: deckhand/ValidationPolicy/v1 - metadata: - schema: metadata/Control/v1 - name: site-deploy-ready - layeringDefinition: - abstract: true - data: - validations: - - name: deckhand-schema-validation - ... - - - name: verify_revision_is_success - desc: | - Validate that the Validation Policy itself passed - deckhand-schema-validation. - GET: /api/v1.0/revisions/$HISTORY['create_validation_policy'].$RESPONSE['$.[0].status.revision']/validations/deckhand-schema-validation - status: 200 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].count: 1 - $.[0].results[0].id: 0 - $.[0].results[0].status: success - - - name: create_vp_two_validations - desc: Add a validation policy with two validations. - PUT: /api/v1.0/buckets/mop/documents - status: 200 - data: |- - --- - schema: deckhand/ValidationPolicy/v1 - metadata: - schema: metadata/Control/v1 - name: site-deploy-ready - layeringDefinition: - abstract: true - data: - validations: - - name: deckhand-schema-validation - - name: promenade-schema-validation - ... - - - name: register_promenade_validation_as_success - desc: Externally register the result for promenade-schema-validation. - POST: /api/v1.0/revisions/$HISTORY['create_vp_two_validations'].$RESPONSE['$.[0].status.revision']/validations/promenade-schema-validation - status: 201 - data: |- - --- - status: success - errors: [] - validator: - name: promenade - version: 1.1.2 - ... - - - name: verify_revision_with_two_validations_list_view_is_success - desc: Verify that both validations are marked as success. - GET: /api/v1.0/revisions/$HISTORY['create_vp_two_validations'].$RESPONSE['$.[0].status.revision']/validations - status: 200 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].count: 2 - # Sort the results by name alphabetically for reliable assertions. - $.[0].results[/name][0].name: deckhand-schema-validation - $.[0].results[/name][0].status: success - $.[0].results[/name][1].name: promenade-schema-validation - $.[0].results[/name][1].status: success - - - name: create_vp_two_validations_missing - desc: Add a validation policy with two validations. - PUT: /api/v1.0/buckets/mop/documents - status: 200 - data: |- - --- - schema: deckhand/ValidationPolicy/v1 - metadata: - schema: metadata/Control/v1 - name: site-deploy-ready - layeringDefinition: - abstract: true - data: - validations: - - name: deckhand-schema-validation - - name: promenade-schema-validation - ... - - # Do not register promenade-schema-validation. - - - name: verify_promenade_schema_validation_is_failure - desc: | - Verify that promenade-schema-validation status is failure because - it should default to failure as it was never externally registered. - GET: /api/v1.0/revisions/$HISTORY['create_vp_two_validations_missing'].$RESPONSE['$.[0].status.revision']/validations/promenade-schema-validation - status: 200 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].count: 1 - $.[0].results[0].id: 0 - $.[0].results[0].status: failure - - - name: verify_revision_with_missing_validation_list_view_is_failure - desc: Verify promenade-schema-validation is failure. - GET: /api/v1.0/revisions/$HISTORY['create_vp_two_validations_missing'].$RESPONSE['$.[0].status.revision']/validations - status: 200 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].count: 2 - # Sort the results by name alphabetically for reliable assertions. - $.[0].results[/name][0].name: deckhand-schema-validation - $.[0].results[/name][0].status: success - $.[0].results[/name][1].name: promenade-schema-validation - $.[0].results[/name][1].status: failure - - - name: create_vp_one_validation_extra - desc: | - Add a validation policy with only one validation but later on register - an extra validation. - PUT: /api/v1.0/buckets/mop/documents - status: 200 - data: |- - --- - schema: deckhand/ValidationPolicy/v1 - metadata: - schema: metadata/Control/v1 - name: site-deploy-ready - layeringDefinition: - abstract: true - data: - validations: - - name: deckhand-schema-validation - ... - - - name: register_extra_promenade_validation_as_failure - desc: | - Externally register the result for extra promenade-schema-validation as - failure. - POST: /api/v1.0/revisions/$HISTORY['create_vp_one_validation_extra'].$RESPONSE['$.[0].status.revision']/validations/promenade-schema-validation - status: 201 - data: |- - --- - status: failure - errors: - - documents: - - schema: promenade/Node/v1 - name: node-document-name - - schema: promenade/Masters/v1 - name: kubernetes-masters - message: Node has master role, but not included in cluster masters list. - validator: - name: promenade - version: 1.1.2 - ... - - - name: verify_promenade_schema_validation_is_ignored - desc: | - Verify that promenade-schema-validation is ignored. - GET: /api/v1.0/revisions/$HISTORY['create_vp_one_validation_extra'].$RESPONSE['$.[0].status.revision']/validations/promenade-schema-validation - status: 200 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].count: 1 - $.[0].results[0].id: 0 - $.[0].results[0].status: ignored [failure] - - - name: verify_revision_with_extra_validation_list_view_is_ignored - desc: | - Verify that the list view shows that promenade-schema-validation is - ignored. - GET: /api/v1.0/revisions/$HISTORY['create_vp_one_validation_extra'].$RESPONSE['$.[0].status.revision']/validations - status: 200 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].count: 2 - # Sort the results by name alphabetically for reliable assertions. - $.[0].results[/name][0].name: deckhand-schema-validation - $.[0].results[/name][0].status: success - $.[0].results[/name][1].name: promenade-schema-validation - $.[0].results[/name][1].status: ignored [failure] - - - name: verify_revision_with_extra_validation_detail_view_is_ignored - desc: | - Verify that details view for promenade-schema-validation indicates it - has been ignored, with a detailed message. - GET: /api/v1.0/revisions/$HISTORY['create_vp_one_validation_extra'].$RESPONSE['$.[0].status.revision']/validations/promenade-schema-validation/entries/0 - status: 200 - response_multidoc_jsonpaths: - $.`len`: 1 - $.[0].name: promenade-schema-validation - $.[0].status: ignored [failure] - $.[0].createdAt: null - $.[0].expiresAfter: null - $.[0].errors.`len`: 2 - $.[0].errors[/message][1].message: >- - The result for this validation was externally registered but has been - ignored because it is not found in the validations for - ValidationPolicy [deckhand/ValidationPolicy/v1, None] - site-deploy-ready: deckhand-schema-validation. diff --git a/deckhand/tests/unit/control/test_validations_controller.py b/deckhand/tests/unit/control/test_validations_controller.py deleted file mode 100644 index 15bcb7c0..00000000 --- a/deckhand/tests/unit/control/test_validations_controller.py +++ /dev/null @@ -1,1246 +0,0 @@ -# Copyright 2017 AT&T Intellectual Property. All other rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import copy -import yaml - -import mock -from oslo_config import cfg - -from deckhand.control import buckets -from deckhand.engine import document_validation -from deckhand import factories -from deckhand.tests import test_utils -from deckhand.tests.unit.control import base as test_base -from deckhand import types - -CONF = cfg.CONF - - -VALIDATION_FAILURE_RESULT = """ ---- -status: failure -errors: - - documents: - - schema: promenade/Node/v1 - name: node-document-name - - schema: promenade/Masters/v1 - name: kubernetes-masters - message: Node has master role, but not included in cluster masters list. -validator: - name: promenade - version: 1.1.2 -""" - -VALIDATION_SUCCESS_RESULT = """ ---- -status: success -errors: [] -validator: - name: promenade - version: 1.1.2 -""" - - -class BaseValidationsControllerTest(test_base.BaseControllerTest): - - def _create_revision(self, payload=None): - if not payload: - documents_factory = factories.DocumentFactory(1, [1]) - payload = documents_factory.gen_test({}) - data_schema_factory = factories.DataSchemaFactory() - data_schema = data_schema_factory.gen_test( - payload[1]['schema'], data={}) - payload.append(data_schema) - - resp = self.app.simulate_put( - '/api/v1.0/buckets/mop/documents', - headers={'Content-Type': 'application/x-yaml'}, - body=yaml.safe_dump_all(payload)) - self.assertEqual(200, resp.status_code) - revision_id = list(yaml.safe_load_all(resp.text))[0]['status'][ - 'revision'] - return revision_id - - def _create_validation(self, revision_id, validation_name, policy): - resp = self.app.simulate_post( - '/api/v1.0/revisions/%s/validations/%s' % (revision_id, - validation_name), - headers={'Content-Type': 'application/x-yaml'}, body=policy) - return resp - - def _monkey_patch_document_validation(self): - """Workaround for testing complex validation scenarios by forcibly - passing in `pre_validate=False`. - """ - # TODO(felipemonteiro): Remove this workaround by testing these more - # complex scenarios against the rendered-documents endpoint instead - # (which performs post-validation). - original_document_validation = document_validation.DocumentValidation - - def monkey_patch(*args, **kwargs): - return original_document_validation(*args, pre_validate=False) - - mock.patch.object(buckets.document_validation, 'DocumentValidation', - side_effect=monkey_patch, autospec=True).start() - self.addCleanup(mock.patch.stopall) - - -class TestValidationsController(BaseValidationsControllerTest): - """Test suite for validating Validations API.""" - - def test_create_validation(self): - rules = {'deckhand:create_cleartext_documents': '@', - 'deckhand:create_validation': '@'} - self.policy.set_rules(rules) - - revision_id = self._create_revision() - validation_name = test_utils.rand_name('validation') - resp = self._create_validation(revision_id, validation_name, - VALIDATION_FAILURE_RESULT) - - self.assertEqual(201, resp.status_code) - expected_body = { - 'status': 'failure', - 'validator': { - 'name': 'promenade', - 'version': '1.1.2' - } - } - self.assertEqual(expected_body, yaml.safe_load(resp.text)) - - def test_list_validations(self): - rules = {'deckhand:create_cleartext_documents': '@', - 'deckhand:list_validations': '@'} - self.policy.set_rules(rules) - - revision_id = self._create_revision() - - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations' % revision_id, - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - - # Validate that the internal deckhand validation was created already. - body = list(yaml.safe_load_all(resp.text)) - expected = { - 'count': 1, - 'results': [ - { - 'status': 'success', - 'name': types.DECKHAND_SCHEMA_VALIDATION - } - ] - } - self.assertEqual(1, len(body)) - self.assertEqual(expected, body[0]) - - rules = {'deckhand:create_cleartext_documents': '@', - 'deckhand:create_validation': '@', - 'deckhand:list_validations': '@'} - self.policy.set_rules(rules) - - # Validate that, after creating a validation policy by an external - # service, it is listed as well. - validation_name = test_utils.rand_name('validation') - resp = self._create_validation(revision_id, validation_name, - VALIDATION_FAILURE_RESULT) - - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations' % revision_id, - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - - body = yaml.safe_load(resp.text) - expected_body = { - 'count': 2, - 'results': [ - { - 'name': types.DECKHAND_SCHEMA_VALIDATION, - 'status': 'success' - }, - { - 'name': validation_name, - 'status': 'failure' - } - ] - } - body['results'] = sorted(body['results'], key=lambda x: x['name']) - self.assertEqual(expected_body, body) - - def test_list_validation_entries(self): - rules = {'deckhand:create_cleartext_documents': '@', - 'deckhand:create_validation': '@', - 'deckhand:list_validations': '@'} - self.policy.set_rules(rules) - - revision_id = self._create_revision() - - # Validate that 3 entries (1 for each of the 3 documents created) - # exists for: - # /api/v1.0/revisions/1/validations/deckhand-schema-validation - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations/%s' % ( - revision_id, types.DECKHAND_SCHEMA_VALIDATION), - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - body = yaml.safe_load(resp.text) - expected_body = { - 'count': 3, - 'results': [{'id': x, 'status': 'success'} for x in range(3)] - } - self.assertEqual(expected_body, body) - - # Add the result of a validation to a revision. - validation_name = test_utils.rand_name('validation') - resp = self._create_validation(revision_id, validation_name, - VALIDATION_FAILURE_RESULT) - - # Validate that the entry is present. - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations/%s' % (revision_id, - validation_name), - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - - body = yaml.safe_load(resp.text) - expected_body = { - 'count': 1, - 'results': [{'id': 0, 'status': 'failure'}] - } - self.assertEqual(expected_body, body) - - def test_list_validation_entries_after_creating_validation(self): - rules = {'deckhand:create_cleartext_documents': '@', - 'deckhand:create_validation': '@', - 'deckhand:list_validations': '@'} - self.policy.set_rules(rules) - - revision_id = self._create_revision() - - # Add the result of a validation to a revision. - validation_name = test_utils.rand_name('validation') - resp = self._create_validation(revision_id, validation_name, - VALIDATION_FAILURE_RESULT) - - # Validate that the entry is present. - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations/%s' % (revision_id, - validation_name), - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - - body = yaml.safe_load(resp.text) - expected_body = { - 'count': 1, - 'results': [{'id': 0, 'status': 'failure'}] - } - self.assertEqual(expected_body, body) - - # Add the result of another validation to the same revision. - resp = self._create_validation(revision_id, validation_name, - VALIDATION_SUCCESS_RESULT) - - # Validate that 2 entries now exist. - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations/%s' % (revision_id, - validation_name), - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - body = yaml.safe_load(resp.text) - expected_body = { - 'count': 2, - 'results': [ - {'id': 0, 'status': 'failure'}, {'id': 1, 'status': 'success'} - ] - } - self.assertEqual(expected_body, body) - - def test_list_validation_entries_with_multiple_entries(self): - rules = {'deckhand:create_cleartext_documents': '@', - 'deckhand:create_validation': '@', - 'deckhand:list_validations': '@'} - self.policy.set_rules(rules) - - revision_id = self._create_revision() - validation_name = test_utils.rand_name('validation') - self._create_validation(revision_id, validation_name, - VALIDATION_FAILURE_RESULT) - self._create_validation(revision_id, validation_name, - VALIDATION_SUCCESS_RESULT) - - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations/%s' % (revision_id, - validation_name), - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - - body = yaml.safe_load(resp.text) - expected_body = { - 'count': 2, - 'results': [ - {'id': 0, 'status': 'failure'}, {'id': 1, 'status': 'success'} - ] - } - self.assertEqual(expected_body, body) - - def test_show_validation_entry(self): - rules = {'deckhand:create_cleartext_documents': '@', - 'deckhand:create_validation': '@', - 'deckhand:show_validation': '@'} - self.policy.set_rules(rules) - - revision_id = self._create_revision() - validation_name = test_utils.rand_name('validation') - resp = self._create_validation(revision_id, validation_name, - VALIDATION_FAILURE_RESULT) - - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations/%s/entries/0' % ( - revision_id, validation_name), - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - - body = yaml.safe_load(resp.text) - expected_body = { - 'name': validation_name, - 'status': 'failure', - 'createdAt': None, - 'expiresAfter': None, - 'errors': [ - { - 'documents': [ - { - 'name': 'node-document-name', - 'schema': 'promenade/Node/v1' - }, { - 'name': 'kubernetes-masters', - 'schema': 'promenade/Masters/v1' - } - ], - 'message': 'Node has master role, but not included in ' - 'cluster masters list.' - } - ] - } - self.assertEqual(expected_body, body) - - def test_show_nonexistent_validation_entry_returns_404(self): - rules = {'deckhand:create_cleartext_documents': '@', - 'deckhand:create_validation': '@', - 'deckhand:show_validation': '@'} - self.policy.set_rules(rules) - - revision_id = self._create_revision() - validation_name = test_utils.rand_name('validation') - resp = self._create_validation(revision_id, validation_name, - VALIDATION_FAILURE_RESULT) - self.assertEqual(201, resp.status_code) - expected_error = ('The requested validation entry 5 was not found for ' - 'validation name %s and revision ID %d' % ( - validation_name, revision_id)) - - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations/%s/entries/5' % ( - revision_id, validation_name), - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(404, resp.status_code) - self.assertEqual(expected_error, yaml.safe_load(resp.text)['message']) - - def test_list_validations_details(self): - rules = {'deckhand:create_cleartext_documents': '@', - 'deckhand:list_validations': '@'} - self.policy.set_rules(rules) - - revision_id = self._create_revision() - - # Validate that 3 entries (1 for each of the 3 documents created) - # exists for - # /api/v1.0/revisions/1/validations/deckhand-schema-validation - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations/detail' % revision_id, - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - body = yaml.safe_load(resp.text) - expected_body = { - 'results': [{ - 'createdAt': None, - 'errors': [], - 'expiresAfter': None, - 'id': idx, - 'name': 'deckhand-schema-validation', - 'status': 'success' - - } for idx in range(3)], - 'count': 3 - } - self.assertEqual(expected_body, body) - - -class TestValidationsControllerPreValidate(BaseValidationsControllerTest): - """Test suite for validating positive scenarios for pre-validations with - Validations controller. - """ - - def test_pre_validate_flag_skips_registered_dataschema_validations(self): - rules = {'deckhand:create_cleartext_documents': '@', - 'deckhand:list_validations': '@'} - self.policy.set_rules(rules) - - # Create a `DataSchema` against which the test document will be - # validated. - data_schema_factory = factories.DataSchemaFactory() - metadata_name = 'example/foo/v1' - schema_to_use = { - '$schema': 'http://json-schema.org/schema#', - 'type': 'object', - 'properties': { - 'a': { - 'type': 'integer' # Test doc will fail b/c of wrong type. - } - }, - 'required': ['a'] - } - data_schema = data_schema_factory.gen_test( - metadata_name, data=schema_to_use) - - # Create a document that passes validation and another that fails it. - doc_factory = factories.DocumentFactory(1, [1]) - fail_doc = doc_factory.gen_test( - {'_GLOBAL_DATA_1_': {'data': {'a': 'fail'}}}, - global_abstract=False)[-1] - fail_doc['schema'] = 'example/foo/v1' - fail_doc['metadata']['name'] = 'test_doc' - - revision_id = self._create_revision(payload=[data_schema, fail_doc]) - - # Validate that the validation reports success because `fail_doc` - # isn't validated by the `DataSchema`. - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations' % revision_id, - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - body = yaml.safe_load(resp.text) - expected_body = { - 'count': 1, - 'results': [ - {'name': types.DECKHAND_SCHEMA_VALIDATION, 'status': 'success'} - ] - } - self.assertEqual(expected_body, body) - - -class TestValidationsControllerPostValidate(BaseValidationsControllerTest): - """Test suite for validating positive scenarios for post-validations with - Validations controller. - """ - - def setUp(self): - super(TestValidationsControllerPostValidate, self).setUp() - self._monkey_patch_document_validation() - - def test_validation_with_registered_data_schema(self): - rules = {'deckhand:create_cleartext_documents': '@', - 'deckhand:list_validations': '@'} - self.policy.set_rules(rules) - - # Create a `DataSchema` against which the test document will be - # validated. - data_schema_factory = factories.DataSchemaFactory() - metadata_name = 'example/Doc/v1' - schema_to_use = { - '$schema': 'http://json-schema.org/schema#', - 'type': 'object', - 'properties': { - 'a': { - 'type': 'string' - } - }, - 'required': ['a'], - 'additionalProperties': False - } - data_schema = data_schema_factory.gen_test( - metadata_name, data=schema_to_use) - - # Create the test document whose data section adheres to the - # `DataSchema` above. - doc_factory = factories.DocumentFactory(1, [1]) - doc_to_test = doc_factory.gen_test( - {'_GLOBAL_DATA_1_': {'data': {'a': 'whatever'}}}, - global_abstract=False)[-1] - doc_to_test['schema'] = 'example/Doc/v1' - - revision_id = self._create_revision(payload=[doc_to_test, data_schema]) - - # Validate that the validation was created and succeeded. - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations' % revision_id, - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - body = yaml.safe_load(resp.text) - expected_body = { - 'count': 1, - 'results': [ - {'name': types.DECKHAND_SCHEMA_VALIDATION, 'status': 'success'} - ] - } - self.assertEqual(expected_body, body) - - def test_validation_data_schema_different_revision_expect_failure(self): - """Validates that creating a ``DataSchema`` in one revision and then - creating a document in another revision that relies on the previously - created ``DataSchema`` results in an expected failure. - """ - rules = {'deckhand:create_cleartext_documents': '@', - 'deckhand:list_validations': '@', - 'deckhand:list_cleartext_documents': '@', - 'deckhand:list_encrypted_documents': '@'} - self.policy.set_rules(rules) - - # Create a `DataSchema` against which the test document will be - # validated. - data_schema_factory = factories.DataSchemaFactory() - metadata_name = 'example/foo/v1' - schema_to_use = { - '$schema': 'http://json-schema.org/schema#', - 'type': 'object', - 'properties': { - 'a': { - 'type': 'integer' # Test doc will fail b/c of wrong type. - } - }, - 'required': ['a'] - } - data_schema = data_schema_factory.gen_test( - metadata_name, data=schema_to_use) - revision_id = self._create_revision(payload=[data_schema]) - - # Validate that the internal deckhand validation was created. - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations' % revision_id, - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - body = yaml.safe_load(resp.text) - expected_body = { - 'count': 1, - 'results': [ - {'name': types.DECKHAND_SCHEMA_VALIDATION, 'status': 'success'} - ] - } - self.assertEqual(expected_body, body) - - # Create the test document that fails the validation due to the - # schema defined by the `DataSchema` document. - doc_factory = factories.DocumentFactory(1, [1]) - docs_to_test = doc_factory.gen_test( - {'_GLOBAL_DATA_1_': {'data': {'a': 'fail'}}}, - global_abstract=False) - docs_to_test[1]['schema'] = 'example/foo/v1' - docs_to_test[1]['metadata']['name'] = 'test_doc' - - revision_id = self._create_revision( - payload=docs_to_test + [data_schema]) - - # Validate that the validation was created and reports failure. - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations' % revision_id, - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - body = yaml.safe_load(resp.text) - expected_body = { - 'count': 1, - 'results': [ - {'name': types.DECKHAND_SCHEMA_VALIDATION, 'status': 'failure'} - ] - } - self.assertEqual(expected_body, body) - - # Validate that the validation was created and reports failure. - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/rendered-documents' % revision_id, - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(400, resp.status_code) - - def test_validation_data_schema_same_revision_expect_failure(self): - """Validates that creating a ``DataSchema`` alongside a document - that relies on it in the same revision results in an expected failure. - """ - rules = {'deckhand:create_cleartext_documents': '@', - 'deckhand:list_validations': '@'} - self.policy.set_rules(rules) - - # Create a `DataSchema` against which the test document will be - # validated. - data_schema_factory = factories.DataSchemaFactory() - metadata_name = 'example/foo/v1' - schema_to_use = { - '$schema': 'http://json-schema.org/schema#', - 'type': 'object', - 'properties': { - 'a': { - 'type': 'integer' # Test doc will fail b/c of wrong type. - } - }, - 'required': ['a'] - } - data_schema = data_schema_factory.gen_test( - metadata_name, data=schema_to_use) - - # Create the test document that fails the validation due to the - # schema defined by the `DataSchema` document. - doc_factory = factories.DocumentFactory(1, [1]) - doc_to_test = doc_factory.gen_test( - {'_GLOBAL_DATA_1_': {'data': {'a': 'fail'}}}, - global_abstract=False)[-1] - doc_to_test['schema'] = 'example/foo/v1' - doc_to_test['metadata']['name'] = 'test_doc' - - revision_id = self._create_revision(payload=[doc_to_test, data_schema]) - - # Validate that the validation was created and reports failure. - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations' % revision_id, - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - body = yaml.safe_load(resp.text) - expected_body = { - 'count': 1, - 'results': [ - {'name': types.DECKHAND_SCHEMA_VALIDATION, 'status': 'failure'} - ] - } - self.assertEqual(expected_body, body) - - def test_validation_with_registered_data_schema_expect_multi_failure(self): - rules = {'deckhand:create_cleartext_documents': '@', - 'deckhand:list_validations': '@', - 'deckhand:show_validation': '@'} - self.policy.set_rules(rules) - - # Create a `DataSchema` against which the test document will be - # validated. - data_schema_factory = factories.DataSchemaFactory() - metadata_name = 'example/foo/v1' - schema_to_use = { - '$schema': 'http://json-schema.org/schema#', - 'type': 'object', - 'properties': { - 'a': { - 'type': 'integer' # Test doc will fail b/c of wrong type. - }, - 'b': { - 'type': 'string' - } - }, - 'required': ['a', 'b'] - } - data_schema = data_schema_factory.gen_test( - metadata_name, data=schema_to_use) - - # Failure #1: Provide wrong type for property "a". - # Failure #2: Don't include required property "b". - doc_factory = factories.DocumentFactory(1, [1]) - doc_to_test = doc_factory.gen_test( - {'_GLOBAL_DATA_1_': {'data': {'a': 'fail'}}}, - global_abstract=False)[-1] - doc_to_test['schema'] = 'example/foo/v1' - doc_to_test['metadata']['name'] = 'test_doc' - - revision_id = self._create_revision(payload=[doc_to_test, data_schema]) - - # Validate that the validation was created and reports failure. - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations' % revision_id, - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - body = yaml.safe_load(resp.text) - expected_body = { - 'count': 1, - 'results': [ - {'name': types.DECKHAND_SCHEMA_VALIDATION, 'status': 'failure'} - ] - } - self.assertEqual(expected_body, body) - - # Validate that both expected errors are present for validation. - expected_errors = [{ - 'error_section': { - 'data': {'a': 'fail'}, - 'metadata': {'labels': {'global': 'global1'}, - 'storagePolicy': 'cleartext', - 'layeringDefinition': {'abstract': False, - 'layer': 'global'}, - 'name': doc_to_test['metadata']['name'], - 'schema': doc_to_test['metadata']['schema']}, - 'schema': doc_to_test['schema'] - }, - 'name': 'test_doc', - 'layer': 'global', - 'path': '.data', - 'schema': 'example/foo/v1', - 'message': "'b' is a required property", - 'validation_schema': schema_to_use, - 'schema_path': '.required' - }, { - 'error_section': {'a': 'fail'}, - 'name': 'test_doc', - 'layer': 'global', - 'path': '.data.a', - 'schema': 'example/foo/v1', - 'message': "'fail' is not of type 'integer'", - 'validation_schema': schema_to_use, - 'schema_path': '.properties.a.type' - }] - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations/%s/entries/0' % ( - revision_id, types.DECKHAND_SCHEMA_VALIDATION), - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - body = yaml.safe_load(resp.text) - - self.assertEqual('failure', body['status']) - self.assertEqual(sorted(expected_errors, key=lambda x: x['path']), - sorted(body['errors'], key=lambda x: x['path'])) - - def test_validation_with_registered_data_schema_expect_mixed(self): - rules = {'deckhand:create_cleartext_documents': '@', - 'deckhand:list_validations': '@', - 'deckhand:show_validation': '@'} - self.policy.set_rules(rules) - - # Create a `DataSchema` against which the test document will be - # validated. - data_schema_factory = factories.DataSchemaFactory() - metadata_name = 'example/foo/v1' - schema_to_use = { - '$schema': 'http://json-schema.org/schema#', - 'type': 'object', - 'properties': { - 'a': { - 'type': 'integer' # Test doc will fail b/c of wrong type. - } - }, - 'required': ['a'] - } - - expected_errors = [{ - 'error_section': {'a': 'fail'}, - 'name': 'fail_doc', - 'layer': 'global', - 'path': '.data.a', - 'schema': 'example/foo/v1', - 'message': "'fail' is not of type 'integer'", - 'validation_schema': schema_to_use, - 'schema_path': '.properties.a.type' - }] - - data_schema = data_schema_factory.gen_test( - metadata_name, data=schema_to_use) - - # Create a document that passes validation and another that fails it. - doc_factory = factories.DocumentFactory(1, [1]) - fail_doc = doc_factory.gen_test( - {'_GLOBAL_DATA_1_': {'data': {'a': 'fail'}}}, - global_abstract=False)[-1] - fail_doc['schema'] = 'example/foo/v1' - fail_doc['metadata']['name'] = 'fail_doc' - - pass_doc = copy.deepcopy(fail_doc) - pass_doc['metadata']['name'] = 'pass_doc' - pass_doc['data']['a'] = 5 - - revision_id = self._create_revision( - payload=[fail_doc, pass_doc, data_schema]) - - # Validate that the validation reports failure since `fail_doc` - # should've failed validation. - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations' % revision_id, - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - body = yaml.safe_load(resp.text) - expected_body = { - 'count': 1, - 'results': [ - {'name': types.DECKHAND_SCHEMA_VALIDATION, 'status': 'failure'} - ] - } - self.assertEqual(expected_body, body) - - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations/%s' % ( - revision_id, types.DECKHAND_SCHEMA_VALIDATION), - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - body = yaml.safe_load(resp.text) - expected_body = { - 'count': 3, - 'results': [{'id': 0, 'status': 'failure'}, # fail_doc failed. - {'id': 1, 'status': 'success'}, # DataSchema passed. - {'id': 2, 'status': 'success'}] # pass_doc succeeded. - } - self.assertEqual(expected_body, body) - - # Validate that fail_doc validation failed for the expected reason. - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations/%s/entries/0' % ( - revision_id, types.DECKHAND_SCHEMA_VALIDATION), - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - body = yaml.safe_load(resp.text) - - self.assertIn('errors', body) - self.assertEqual(expected_errors, body['errors']) - - def test_document_without_data_section_ingested(self): - """Validate that a document without the data section is ingested - successfully. - """ - rules = {'deckhand:create_cleartext_documents': '@', - 'deckhand:list_validations': '@'} - self.policy.set_rules(rules) - - documents_factory = factories.DocumentFactory(1, [1]) - document = documents_factory.gen_test({}, global_abstract=False)[-1] - del document['data'] - - data_schema_factory = factories.DataSchemaFactory() - data_schema = data_schema_factory.gen_test(document['schema'], {}) - - revision_id = self._create_revision(payload=[document, data_schema]) - - # Validate that the entry is present. - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations/%s' % ( - revision_id, types.DECKHAND_SCHEMA_VALIDATION), - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - - body = yaml.safe_load(resp.text) - expected_body = { - 'count': 2, - 'results': [{'id': 0, 'status': 'success'}, # Document. - {'id': 1, 'status': 'success'}] # DataSchema. - } - self.assertEqual(expected_body, body) - - def test_validation_only_new_data_schema_registered(self): - """Validate whether newly created DataSchemas replace old DataSchemas - when it comes to validation. - """ - rules = {'deckhand:create_cleartext_documents': '@', - 'deckhand:list_validations': '@'} - self.policy.set_rules(rules) - - # Create 2 DataSchemas that will fail if they're used. These shouldn't - # be used for validation. - data_schema_factory = factories.DataSchemaFactory() - metadata_names = ['exampleA/Doc/v1', 'exampleB/Doc/v1'] - schemas_to_use = [{ - '$schema': 'http://json-schema.org/schema#', - 'type': 'object', - 'properties': { - 'a': { - 'type': 'integer' - } - }, - 'required': ['a'], - 'additionalProperties': False - }] * 2 - old_data_schemas = [ - data_schema_factory.gen_test( - metadata_names[i], data=schemas_to_use[i]) - for i in range(2) - ] - # Save the DataSchemas in the first revision. - revision_id = self._create_revision(payload=old_data_schemas) - - # Create 2 DataSchemas that will pass if they're used. These should - # be used for validation. - for schema_to_use in schemas_to_use: - schema_to_use['properties']['a']['type'] = 'string' - new_data_schemas = [ - data_schema_factory.gen_test( - metadata_names[i], data=schemas_to_use[i]) - for i in range(2) - ] - doc_factory = factories.DocumentFactory(1, [1]) - example1_doc = doc_factory.gen_test( - {'_GLOBAL_DATA_1_': {'data': {'a': 'whatever'}}}, - global_abstract=False)[-1] - example1_doc['schema'] = metadata_names[0] - example2_doc = copy.deepcopy(example1_doc) - example2_doc['schema'] = metadata_names[1] - # Save the documents that will be validated alongside the DataSchemas - # that will be used to validate them. - revision_id = self._create_revision( - payload=[example1_doc, example2_doc] + new_data_schemas) - - # Validate that the validation was created and succeeded: This means - # that the new DataSchemas were used, not the old ones. - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations' % revision_id, - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - body = yaml.safe_load(resp.text) - expected_body = { - 'count': 1, - 'results': [ - {'name': types.DECKHAND_SCHEMA_VALIDATION, 'status': 'success'} - ] - } - self.assertEqual(expected_body, body) - - -class TestValidationsControllerWithValidationPolicy( - BaseValidationsControllerTest): - - def setUp(self): - super(TestValidationsControllerWithValidationPolicy, self).setUp() - self._monkey_patch_document_validation() - - def test_validation_with_validation_policy_success(self): - rules = {'deckhand:create_cleartext_documents': '@', - 'deckhand:list_validations': '@'} - self.policy.set_rules(rules) - - # Create a `ValidationPolicy` which is used to check whether a revision - # passed all the validations. - validation_policy = yaml.safe_load(""" ---- -schema: deckhand/ValidationPolicy/v1 -metadata: - schema: metadata/Control/v1 - name: site-deploy-ready - layeringDefinition: - abstract: true - layer: site -data: - validations: - - name: deckhand-schema-validation -... -""") - revision_id = self._create_revision(payload=[validation_policy]) - - # Validate that the validation was created and reports success. - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations' % revision_id, - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - body = yaml.safe_load(resp.text) - expected_body = { - 'count': 1, - 'results': [ - {'name': 'deckhand-schema-validation', 'status': 'success'} - ] - } - self.assertEqual(expected_body, body) - - def test_with_validation_policy_external_validation(self): - """Validate that a ValidationPolicy with an externally registered - validation that is successful passes. - """ - rules = {'deckhand:create_cleartext_documents': '@', - 'deckhand:create_validation': '@', - 'deckhand:list_validations': '@'} - self.policy.set_rules(rules) - - # Create a `ValidationPolicy` which expects two validations. - validation_policy = yaml.safe_load(""" ---- -schema: deckhand/ValidationPolicy/v1 -metadata: - schema: metadata/Control/v1 - name: site-deploy-ready - layeringDefinition: - abstract: true - layer: site -data: - validations: - - name: deckhand-schema-validation - - name: promenade-schema-validation -... -""") - revision_id = self._create_revision(payload=[validation_policy]) - - # Create the external validation for "promenade-schema-validation". - resp = self._create_validation( - revision_id, 'promenade-schema-validation', - VALIDATION_SUCCESS_RESULT) - self.assertEqual(201, resp.status_code) - - # Validate that the validation was created and reports success. - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations' % revision_id, - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - body = yaml.safe_load(resp.text) - expected_body = { - 'count': 2, - 'results': [ - {'name': 'deckhand-schema-validation', 'status': 'success'}, - {'name': 'promenade-schema-validation', 'status': 'success'} - ] - } - body['results'] = sorted(body['results'], key=lambda x: x['name']) - self.assertEqual(expected_body, body) - - def test_with_multiple_validation_policy_external_validation(self): - """Validate that two ValidationPolicy documents, one that references - the internal deckhand-schema-validation, and the other which requires - an externally registered validation, produces a successful validation - result. - """ - rules = {'deckhand:create_cleartext_documents': '@', - 'deckhand:create_validation': '@', - 'deckhand:list_validations': '@'} - self.policy.set_rules(rules) - - # Create two `ValidationPolicy` documents. - validation_policies = yaml.safe_load_all(""" ---- -schema: deckhand/ValidationPolicy/v1 -metadata: - schema: metadata/Control/v1 - name: vp-1 - layeringDefinition: - abstract: true - layer: site -data: - validations: - - name: deckhand-schema-validation ---- -schema: deckhand/ValidationPolicy/v1 -metadata: - schema: metadata/Control/v1 - name: vp-2 - layeringDefinition: - abstract: true - layer: site -data: - validations: - - name: promenade-schema-validation -... -""") - revision_id = self._create_revision(payload=validation_policies) - - # Create the external validation for "promenade-schema-validation". - resp = self._create_validation( - revision_id, 'promenade-schema-validation', - VALIDATION_SUCCESS_RESULT) - self.assertEqual(201, resp.status_code) - - # Validate that the validation was created and reports success. - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations' % revision_id, - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - body = yaml.safe_load(resp.text) - expected_body = { - 'count': 2, - 'results': [ - {'name': 'deckhand-schema-validation', 'status': 'success'}, - {'name': 'promenade-schema-validation', 'status': 'success'} - ] - } - body['results'] = sorted(body['results'], key=lambda x: x['name']) - self.assertEqual(expected_body, body) - - def test_with_validation_policy_missing_external_validation(self): - """Validate that a ValidationPolicy with a missing externally - registered validation that is listed under the validations for the - ValidationPolicy defaults to "failure". - """ - rules = {'deckhand:create_cleartext_documents': '@', - 'deckhand:list_validations': '@', - 'deckhand:show_validation': '@'} - self.policy.set_rules(rules) - - # Create a `ValidationPolicy` which expects two validations but do not - # create the validation for "promenade-schema-validation". - validation_policy = yaml.safe_load(""" ---- -schema: deckhand/ValidationPolicy/v1 -metadata: - schema: metadata/Control/v1 - name: site-deploy-ready - layeringDefinition: - abstract: true - layer: site -data: - validations: - - name: deckhand-schema-validation - - name: promenade-schema-validation -... -""") - revision_id = self._create_revision(payload=[validation_policy]) - - # Validate that the validation was created and that the missing one - # defaults to "failure". - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations' % revision_id, - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - body = yaml.safe_load(resp.text) - expected_body = { - 'count': 2, - 'results': [ - {'name': 'deckhand-schema-validation', 'status': 'success'}, - {'name': 'promenade-schema-validation', 'status': 'failure'} - ] - } - body['results'] = sorted(body['results'], key=lambda x: x['name']) - self.assertEqual(expected_body, body) - - # Validate that 'promenade-schema-validation' is 'failure' even though - # it was never externally registered. - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations/%s' % ( - revision_id, 'promenade-schema-validation'), - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - body = yaml.safe_load(resp.text) - expected_body = { - 'count': 1, - 'results': [{'id': 0, 'status': 'failure'}] - } - self.assertEqual(expected_body, body) - - # Validate information explaining why 'promenade-schema-validation' - # failed is returned. Note that DH should be smart enough to say that - # it was never registered externally, which is why it's 'failure'. - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations/%s/entries/0' % ( - revision_id, 'promenade-schema-validation'), - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - body = yaml.safe_load(resp.text) - - expected_msg = ('The result for this validation was never externally ' - 'registered so its status defaulted to "failure".') - expected_body = { - 'name': 'promenade-schema-validation', - 'status': 'failure', - 'createdAt': None, - 'expiresAfter': None, - 'errors': [{'message': expected_msg}] - } - self.assertEqual(expected_body, body) - - def test_with_validation_policy_extra_external_validation(self): - """Validate that a ValidationPolicy with extra externally registered - validations that aren't listed under the validations for the - ValidationPolicy defaults to "ignored [{original_status}]". - """ - rules = {'deckhand:create_cleartext_documents': '@', - 'deckhand:create_validation': '@', - 'deckhand:list_validations': '@', - 'deckhand:show_validation': '@'} - self.policy.set_rules(rules) - - # Create a `ValidationPolicy` with only 1 validation. - validation_policy = yaml.safe_load(""" ---- -schema: deckhand/ValidationPolicy/v1 -metadata: - schema: metadata/Control/v1 - name: site-deploy-ready - layeringDefinition: - abstract: true - layer: site -data: - validations: - - name: deckhand-schema-validation -... -""") - - def _do_test(validation_result, expected_status): - revision_id = self._create_revision(payload=[validation_policy]) - - # Register an extra validation not in the ValidationPolicy. - resp = self._create_validation( - revision_id, 'promenade-schema-validation', - validation_result) - self.assertEqual(201, resp.status_code) - - # Validate that the extra validation is ignored. - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations' % revision_id, - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - body = yaml.safe_load(resp.text) - expected_body = { - 'count': 2, - 'results': [ - {'name': 'deckhand-schema-validation', - 'status': 'success'}, - {'name': 'promenade-schema-validation', - 'status': 'ignored [%s]' % expected_status} - ] - } - body['results'] = sorted(body['results'], key=lambda x: x['name']) - self.assertEqual(expected_body, body) - - # Validate that 'promenade-schema-validation' is - # 'ignored [expected_status]'. - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations/%s' % ( - revision_id, 'promenade-schema-validation'), - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - body = yaml.safe_load(resp.text) - expected_body = { - 'count': 1, - 'results': [ - {'id': 0, 'status': 'ignored [%s]' % expected_status} - ] - } - self.assertEqual(expected_body, body) - - # Validate information explaining why 'promenade-schema-validation' - # is ignored is returned. - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations/%s/entries/0' % ( - revision_id, 'promenade-schema-validation'), - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - body = yaml.safe_load(resp.text) - - expected_msg = ('The result for this validation was externally ' - 'registered but has been ignored because it is not' - ' found in the validations for ValidationPolicy ' - '[%s, %s] %s: %s.' % ( - validation_policy['schema'], - validation_policy['metadata'][ - 'layeringDefinition']['layer'], - validation_policy['metadata']['name'], - types.DECKHAND_SCHEMA_VALIDATION)) - - expected_errors = [] - if expected_status == 'failure': - expected_errors.extend( - yaml.safe_load(VALIDATION_FAILURE_RESULT)['errors']) - expected_errors.append({'message': expected_msg}) - - expected_body = { - 'name': 'promenade-schema-validation', - 'status': 'ignored [%s]' % expected_status, - 'createdAt': None, - 'expiresAfter': None, - 'errors': expected_errors - } - self.assertEqual(expected_body, body) - - _do_test(VALIDATION_SUCCESS_RESULT, 'success') - _do_test(VALIDATION_FAILURE_RESULT, 'failure') diff --git a/doc/source/users/validation.rst b/doc/source/users/validation.rst index ac47e96b..14eefc16 100644 --- a/doc/source/users/validation.rst +++ b/doc/source/users/validation.rst @@ -41,10 +41,6 @@ Here is a list of internal validations: * ``deckhand-document-schema-validation`` - All concrete documents in the revision successfully pass their JSON schema validations. Will cause this to report an error. -* ``deckhand-policy-validation`` (TODO) - All required policy documents are in-place, - and existing documents conform to those policies. E.g. if a 3rd party - document specifies a ``layer`` that is not present in the layering policy, - that will cause this validation to report an error. Externally Provided Validations ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -84,81 +80,6 @@ Schema validations are controlled by two mechanisms: document's ``data`` section to *additional* validations, validations specified by the ``data`` section of the ``DataSchema`` document. -Policy Validations ------------------- - -*Not yet implemented*. - -Validation Policies -------------------- - -Validation policies are optional. Deckhand will perform all internal and -externally registered schema validations against all documents, with or without -any Validation Policies. - -All ``ValidationPolicy`` documents in Deckhand are externally registered. They -allow services to report success or failure of named validations for a given -revision. The intended purpose is to allow a simple mapping that enables -consuming services to be able to quickly check whether the configuration in -Deckhand is in a valid state for performing a specific action. - -``ValidationPolicy`` documents are not the same as ``DataSchema`` documents. -A ``ValidationPolicy`` document can reference a list of internal Deckhand -validations in addition to externally registered ``DataSchema`` documents. -Whereas a ``DataSchema`` document specifies a new set of validations to check -against relevant documents, a ``ValidationPolicy`` is a bookkeeping device -that merely lists the set of validations in a revision that need to succeed -in order for the revision itself to be valid. - -For example, given Revision 1 which contains a ``ValidationPolicy`` of: - -:: - - --- - schema: deckhand/ValidationPolicy/v1 - metadata: - schema: metadata/Control/v1 - name: later-validation - layeringDefinition: - abstract: False - layer: site - data: - validations: - - name: deckhand-schema-validation - - name: drydock-site-validation - -Deckhand automatically creates ``deckhand-schema-validation`` as soon as the -revision itself is created. Afterward, Drydock can POST its result for -``drydock-site-validation`` using Deckhand's Validations API. Finally, Shipyard -query Deckhand's Validations API which in turn checks whether all validations -contained in the ``ValidationPolicy`` above are successful, before it proceeds -to the next stage in its workflow. - -Missing Validations -^^^^^^^^^^^^^^^^^^^ - -Validations contained in a ``ValidationPolicy`` but which were never created -in Deckhand for a given revision are considered missing. Missing validations -result in the entire validation result reporting "failure". - -If, for example, Drydock never POSTed a result for ``drydock-site-validation`` -then the Deckhand Validations API will return a "failure" result, even if -``deckhand-schema-validation`` reports "success". - -Extra Validations -^^^^^^^^^^^^^^^^^ - -Validations that are registered in Deckhand via the Validations API -but are not included in the ``ValidationPolicy`` (if one exists) for a given -revision are **ignored** (with the original status reported as -"ignored [failure]" or "ignored [success]"). - -For example, given the ``ValidationPolicy`` example above, if Promenade POSTs -``promenade-schema-validation`` with a result of "failure", then the *overall* -validation status for the given revision returned by Deckhand will be *success* -because the "failure" result from Promenade, since it was never registered, -will be ignored. - Validation Stages -----------------