From e0fc59e89baf027cd92a2fc2cac96e786a6f2235 Mon Sep 17 00:00:00 2001 From: Felipe Monteiro Date: Sat, 20 Jan 2018 01:24:14 +0000 Subject: [PATCH] Deckhand schemas as YAML files Use YAML formatting for built-in Deckhand schemas used for validations to align with other UCP services. The second most important intention behind this PS is to allow pre_validate flag to cascade correctly between the layering and document_validation modules. If pre_validate is true, then: * the base_schema validates ALL documents * ALL built-in schemas validate the appropriate document given a schema match * NO externally registered DataSchema documents are used for validation Else (if pre_validate is false): * the base_schema validates ALL documents * ALL built-in schemas validate the appropriate document given a schema match * ALL externally registered DataSchema documents are used for validation given a schema match A more minor change is setting pre_validate flags in all modules to True for consistency. The idea is to facilitate the way other projects that import Deckhand in directly interface with Deckhand. Change-Id: I859f61989ec15bede1c104b86625d116064f056d --- deckhand/control/revision_documents.py | 2 +- deckhand/engine/document_validation.py | 443 +++++++++--------- deckhand/engine/layering.py | 13 +- deckhand/engine/schema/__init__.py | 0 deckhand/engine/schema/base_schema.py | 47 -- deckhand/engine/schema/v1_0/__init__.py | 31 -- .../v1_0/certificate_authority_key_schema.py | 66 --- .../v1_0/certificate_authority_schema.py | 66 --- .../schema/v1_0/certificate_key_schema.py | 65 --- .../engine/schema/v1_0/certificate_schema.py | 65 --- .../engine/schema/v1_0/data_schema_schema.py | 76 --- .../engine/schema/v1_0/document_schema.py | 123 ----- .../schema/v1_0/layering_policy_schema.py | 74 --- .../engine/schema/v1_0/passphrase_schema.py | 65 --- .../engine/schema/v1_0/private_key_schema.py | 64 --- .../engine/schema/v1_0/public_key_schema.py | 64 --- .../schema/v1_0/validation_policy_schema.py | 80 ---- deckhand/engine/schemas/base_schema.yaml | 119 +++++ .../certificate_authority_key_schema.yaml | 65 +++ .../schemas/certificate_authority_schema.yaml | 65 +++ .../schemas/certificate_key_schema.yaml | 65 +++ .../engine/schemas/certificate_schema.yaml | 65 +++ .../engine/schemas/dataschema_schema.yaml | 33 ++ .../schemas/layering_policy_schema.yaml | 35 ++ .../engine/schemas/passphrase_schema.yaml | 65 +++ .../engine/schemas/private_key_schema.yaml | 65 +++ .../engine/schemas/public_key_schema.yaml | 65 +++ .../schemas/validation_policy_schema.yaml | 76 +++ deckhand/errors.py | 5 +- deckhand/factories.py | 50 +- .../control/test_validations_controller.py | 85 ++-- .../engine/test_document_layering_negative.py | 27 ++ .../unit/engine/test_document_validation.py | 11 +- .../test_document_validation_negative.py | 165 +++---- .../unit/resources/sample_certificate.yaml | 3 + .../sample_certificate_authority.yaml | 16 + .../sample_certificate_authority_key.yaml | 16 + .../resources/sample_certificate_key.yaml | 3 + .../unit/resources/sample_data_schema.yaml | 3 + .../resources/sample_layering_policy.yaml | 3 + .../unit/resources/sample_passphrase.yaml | 1 + .../unit/resources/sample_private_key.yaml | 10 + .../unit/resources/sample_public_key.yaml | 10 + .../resources/sample_validation_policy.yaml | 3 + doc/source/document_types.rst | 4 +- doc/source/validation.rst | 190 +++++++- 46 files changed, 1376 insertions(+), 1291 deletions(-) delete mode 100644 deckhand/engine/schema/__init__.py delete mode 100644 deckhand/engine/schema/base_schema.py delete mode 100644 deckhand/engine/schema/v1_0/__init__.py delete mode 100644 deckhand/engine/schema/v1_0/certificate_authority_key_schema.py delete mode 100644 deckhand/engine/schema/v1_0/certificate_authority_schema.py delete mode 100644 deckhand/engine/schema/v1_0/certificate_key_schema.py delete mode 100644 deckhand/engine/schema/v1_0/certificate_schema.py delete mode 100644 deckhand/engine/schema/v1_0/data_schema_schema.py delete mode 100644 deckhand/engine/schema/v1_0/document_schema.py delete mode 100644 deckhand/engine/schema/v1_0/layering_policy_schema.py delete mode 100644 deckhand/engine/schema/v1_0/passphrase_schema.py delete mode 100644 deckhand/engine/schema/v1_0/private_key_schema.py delete mode 100644 deckhand/engine/schema/v1_0/public_key_schema.py delete mode 100644 deckhand/engine/schema/v1_0/validation_policy_schema.py create mode 100644 deckhand/engine/schemas/base_schema.yaml create mode 100644 deckhand/engine/schemas/certificate_authority_key_schema.yaml create mode 100644 deckhand/engine/schemas/certificate_authority_schema.yaml create mode 100644 deckhand/engine/schemas/certificate_key_schema.yaml create mode 100644 deckhand/engine/schemas/certificate_schema.yaml create mode 100644 deckhand/engine/schemas/dataschema_schema.yaml create mode 100644 deckhand/engine/schemas/layering_policy_schema.yaml create mode 100644 deckhand/engine/schemas/passphrase_schema.yaml create mode 100644 deckhand/engine/schemas/private_key_schema.yaml create mode 100644 deckhand/engine/schemas/public_key_schema.yaml create mode 100644 deckhand/engine/schemas/validation_policy_schema.yaml create mode 100644 deckhand/tests/unit/resources/sample_certificate_authority.yaml create mode 100644 deckhand/tests/unit/resources/sample_certificate_authority_key.yaml create mode 100644 deckhand/tests/unit/resources/sample_private_key.yaml create mode 100644 deckhand/tests/unit/resources/sample_public_key.yaml diff --git a/deckhand/control/revision_documents.py b/deckhand/control/revision_documents.py index 6dcf0c55..7c2c1ff2 100644 --- a/deckhand/control/revision_documents.py +++ b/deckhand/control/revision_documents.py @@ -186,7 +186,7 @@ class RenderedDocumentsResource(api_base.BaseResource): data_schemas = db_api.revision_documents_get( schema=types.DATA_SCHEMA_SCHEMA, deleted=False) doc_validator = document_validation.DocumentValidation( - rendered_documents, data_schemas) + rendered_documents, data_schemas, pre_validate=False) try: validations = doc_validator.validate_all() except errors.InvalidDocumentFormat as e: diff --git a/deckhand/engine/document_validation.py b/deckhand/engine/document_validation.py index d4a11707..5cda2d08 100644 --- a/deckhand/engine/document_validation.py +++ b/deckhand/engine/document_validation.py @@ -13,15 +13,17 @@ # limitations under the License. import abc +import copy +import os +import pkg_resources import re +import yaml import jsonschema from oslo_log import log as logging import six from deckhand.engine import document_wrapper -from deckhand.engine.schema import base_schema -from deckhand.engine.schema import v1_0 from deckhand.engine.secrets_manager import SecretsSubstitution from deckhand import errors from deckhand import types @@ -29,6 +31,48 @@ from deckhand import utils LOG = logging.getLogger(__name__) +_DEFAULT_SCHEMAS = {} +_SUPPORTED_SCHEMA_VERSIONS = ('v1',) + + +def _get_schema_parts(document, schema_key='schema'): + # TODO(fmontei): Remove this function once documents have been standardized + # around macroversions or microversions. + schema_parts = utils.jsonpath_parse(document, schema_key).split('/') + schema_prefix = '/'.join(schema_parts[:2]) + schema_version = schema_parts[2] + if schema_version.endswith('.0'): + schema_version = schema_version[:-2] + return schema_prefix, schema_version + + +def _get_schema_dir(): + return pkg_resources.resource_filename('deckhand.engine', 'schemas') + + +def _build_schema_map(): + """Populates ``_DEFAULT_SCHEMAS`` with built-in Deckhand schemas.""" + global _DEFAULT_SCHEMAS + _DEFAULT_SCHEMAS = {k: {} for k in _SUPPORTED_SCHEMA_VERSIONS} + schema_dir = _get_schema_dir() + for schema_file in os.listdir(schema_dir): + if not schema_file.endswith('.yaml'): + continue + with open(os.path.join(schema_dir, schema_file)) as f: + for schema in yaml.safe_load_all(f): + schema_name = schema['metadata']['name'] + version = schema_name.split('/')[-1] + _DEFAULT_SCHEMAS.setdefault(version, {}) + if schema_file in _DEFAULT_SCHEMAS[version]: + raise RuntimeError("Duplicate DataSchema document [%s] %s " + "detected." % (schema['schema'], + schema_name)) + _DEFAULT_SCHEMAS[version].setdefault( + '/'.join(schema_name.split('/')[:2]), schema['data']) + + +_build_schema_map() + @six.add_metaclass(abc.ABCMeta) class BaseValidator(object): @@ -41,6 +85,10 @@ class BaseValidator(object): _supported_versions = ('v1',) _schema_re = re.compile(r'^[a-zA-Z]+\/[a-zA-Z]+\/v\d+(.0)?$') + def __init__(self): + global _DEFAULT_SCHEMAS + self._schema_map = _DEFAULT_SCHEMAS + @abc.abstractmethod def matches(self, document): """Whether this Validator should be used to validate ``document``. @@ -59,11 +107,15 @@ class GenericValidator(BaseValidator): or abstract, or what version its schema is. """ + def __init__(self): + super(GenericValidator, self).__init__() + self.base_schema = self._schema_map['v1']['deckhand/Base'] + def matches(self, document): # Applies to all schemas, so unconditionally returns True. return True - def validate(self, document): + def validate(self, document, **kwargs): """Validate ``document``against basic schema validation. Sanity-checks each document for mandatory keys like "metadata" and @@ -82,8 +134,8 @@ class GenericValidator(BaseValidator): """ try: - jsonschema.Draft4Validator.check_schema(base_schema.schema) - schema_validator = jsonschema.Draft4Validator(base_schema.schema) + jsonschema.Draft4Validator.check_schema(self.base_schema) + schema_validator = jsonschema.Draft4Validator(self.base_schema) error_messages = [ e.message for e in schema_validator.iter_errors(document)] except Exception as e: @@ -96,34 +148,110 @@ class GenericValidator(BaseValidator): 'Failed sanity-check validation for document [%s] %s. ' 'Details: %s', document.get('schema', 'N/A'), document.metadata.get('name'), error_messages) - raise errors.InvalidDocumentFormat(details=error_messages) + raise errors.InvalidDocumentFormat( + document_schema=document.schema, + document_name=document.name, + errors=', '.join(error_messages)) -class SchemaValidator(BaseValidator): - """Validator for validating built-in document kinds.""" +class DataSchemaValidator(GenericValidator): + """Validator for validating ``DataSchema`` documents.""" - _schema_map = { - 'v1': { - 'deckhand/CertificateAuthorityKey': - v1_0.certificate_authority_key_schema, - 'deckhand/CertificateAuthority': v1_0.certificate_authority_schema, - 'deckhand/CertificateKey': v1_0.certificate_key_schema, - 'deckhand/Certificate': v1_0.certificate_schema, - 'deckhand/DataSchema': v1_0.data_schema_schema, - 'deckhand/LayeringPolicy': v1_0.layering_policy_schema, - 'deckhand/Passphrase': v1_0.passphrase_schema, - 'deckhand/PrivateKey': v1_0.private_key_schema, - 'deckhand/PublicKey': v1_0.public_key_schema, - 'deckhand/ValidationPolicy': v1_0.validation_policy_schema, + def __init__(self, data_schemas): + super(DataSchemaValidator, self).__init__() + global _DEFAULT_SCHEMAS + + self._default_schema_map = _DEFAULT_SCHEMAS + self._external_data_schemas = [d.data for d in data_schemas] + self._schema_map = self._build_schema_map(data_schemas) + + def _build_schema_map(self, data_schemas): + schema_map = copy.deepcopy(self._default_schema_map) + + for data_schema in data_schemas: + # Ensure that each `DataSchema` document has required properties + # before they themselves can be used to validate other documents. + if 'name' not in data_schema.metadata: + continue + if self._schema_re.match(data_schema.name) is None: + continue + if 'data' not in data_schema: + continue + schema_prefix, schema_version = _get_schema_parts(data_schema, + 'metadata.name') + schema_map[schema_version].setdefault(schema_prefix, + data_schema.data) + + return schema_map + + def matches(self, document): + if document.is_abstract: + LOG.info('Skipping schema validation for abstract document [%s]: ' + '%s.', document.schema, document.name) + return False + schema_prefix, schema_version = _get_schema_parts(document) + return schema_prefix in self._schema_map.get(schema_version, {}) + + def _generate_validation_error_output(self, schema, document, error, + root_path): + """Returns a formatted output with necessary details for debugging why + a validation failed. + + The response is a dictionary with the following keys: + + * validation_schema: The schema body that was used to validate the + document. + * schema_path: The JSON path in the schema where the failure + originated. + * name: The document name. + * schema: The document schema. + * path: The JSON path in the document where the failure originated. + * error_section: The "section" in the document above which the error + originated (i.e. the dict in which ``path`` is found). + * message: The error message returned by the ``jsonschema`` validator. + + :returns: Dictionary in the above format. + """ + error_path = '.'.join([str(x) for x in error.path]) + if error_path: + path_to_error_in_document = '.'.join([root_path, error_path]) + else: + path_to_error_in_document = root_path + path_to_error_in_schema = '.' + '.'.join( + [str(x) for x in error.schema_path]) + + parent_path_to_error_in_document = '.'.join( + path_to_error_in_document.split('.')[:-1]) or '.' + try: + # NOTE(fmontei): Because validation is performed on fully rendered + # documents, it is necessary to omit the parts of the data section + # where substitution may have occurred to avoid exposing any + # secrets. While this may make debugging a few validation failures + # more difficult, it is a necessary evil. + sanitized_document = ( + SecretsSubstitution.sanitize_potential_secrets(document)) + parent_error_section = utils.jsonpath_parse( + sanitized_document, parent_path_to_error_in_document) + except Exception: + parent_error_section = ( + 'Failed to find parent section above where error occurred.') + + error_output = { + 'validation_schema': schema, + 'schema_path': path_to_error_in_schema, + 'name': document.name, + 'schema': document.schema, + 'path': path_to_error_in_document, + 'error_section': parent_error_section, + # TODO(fmontei): Also sanitize any secrets contained in the message + # as well. + 'message': error.message } - } - # Represents a generic document schema. - _fallback_schema = v1_0.document_schema + return error_output def _get_schemas(self, document): - """Retrieve the relevant schemas based on the document's - ``schema``. + """Retrieve the relevant schemas based on the document's ``schema``. :param dict doc: The document used for finding the correct schema to validate it based on its ``schema``. @@ -134,32 +262,25 @@ class SchemaValidator(BaseValidator): """ schema_prefix, schema_version = _get_schema_parts(document) matching_schemas = [] + relevant_schemas = self._schema_map.get(schema_version, {}) - for candidae_schema_prefix, schema in relevant_schemas.items(): - if candidae_schema_prefix == schema_prefix: + for candidate_schema_prefix, schema in relevant_schemas.items(): + if candidate_schema_prefix == schema_prefix: if schema not in matching_schemas: matching_schemas.append(schema) return matching_schemas - def matches(self, document): - if document.is_abstract: - LOG.info('Skipping schema validation for abstract document [%s]: ' - '%s.', document.schema, document.name) - return False - return True - - def validate(self, document, validate_section='', - use_fallback_schema=True): + def validate(self, document, pre_validate=True): """Validate ``document`` against built-in ``schema``-specific schemas. Does not apply to abstract documents. - :param dict document: Document to validate. - :param str validate_section: Document section to validate. If empty - string, validates entire ``document``. - :param bool use_fallback_schema: Whether to use the "fallback" schema - if no matching schemas are found by :method:``matches``. - + :param document: Document to validate. + :type document: DocumentDict + :param pre_validate: Whether to pre-validate documents using built-in + schema validation. Skips over externally registered ``DataSchema`` + documents to avoid false positives. Default is True. + :type pre_validate: bool :raises RuntimeError: If the Deckhand schema itself is invalid. :returns: Tuple of (error message, parent path for failing property) following schema validation failure. @@ -167,19 +288,30 @@ class SchemaValidator(BaseValidator): """ schemas_to_use = self._get_schemas(document) - if not schemas_to_use and use_fallback_schema: - LOG.debug('Document schema %s not recognized. Using "fallback" ' - 'schema.', document.schema) - schemas_to_use = [SchemaValidator._fallback_schema] + if not schemas_to_use: + LOG.debug('Document schema %s not recognized by %s. No further ' + 'validation required.', document.schema, + self.__class__.__name__) - for schema_to_use in schemas_to_use: - schema = schema_to_use.schema - if validate_section: - to_validate = document.get(validate_section, None) - root_path = '.' + validate_section + '.' - else: - to_validate = document + for schema in schemas_to_use: + is_builtin_schema = schema not in self._external_data_schemas + # NOTE(fmontei): The purpose of this `continue` is to not + # PRE-validate documents against externally registered + # `DataSchema` documents, in order to avoid raising spurious + # errors. These spurious errors arise from `DataSchema` documents + # really only applying post-rendering, when documents have all + # the substitutions they need to pass externally registered + # `DataSchema` validations. + if not is_builtin_schema and pre_validate: + continue + + if is_builtin_schema: root_path = '.' + to_validate = document + else: + root_path = '.data' + to_validate = document.get('data', {}) + try: jsonschema.Draft4Validator.check_schema(schema) schema_validator = jsonschema.Draft4Validator(schema) @@ -195,80 +327,52 @@ class SchemaValidator(BaseValidator): 'Failed schema validation for document [%s] %s. ' 'Details: %s.', document.schema, document.name, error.message) - yield _generate_validation_error_output( - schema_to_use, document, error, root_path) - - -class DataSchemaValidator(SchemaValidator): - """Validator for validating ``DataSchema`` documents.""" - - def __init__(self, data_schemas): - super(DataSchemaValidator, self).__init__() - self._schema_map = self._build_schema_map(data_schemas) - - def _build_schema_map(self, data_schemas): - schema_map = {k: {} for k in self._supported_versions} - - for data_schema in data_schemas: - # Ensure that each `DataSchema` document has required properties - # before they themselves can be used to validate other documents. - if 'name' not in data_schema.metadata: - continue - if self._schema_re.match(data_schema.name) is None: - continue - if 'data' not in data_schema: - continue - schema_prefix, schema_version = _get_schema_parts(data_schema, - 'metadata.name') - - class Schema(object): - schema = data_schema.data - - schema_map[schema_version].setdefault(schema_prefix, Schema()) - - return schema_map - - def matches(self, document): - if document.is_abstract: - LOG.info('Skipping schema validation for abstract document [%s]: ' - '%s.', document.schema, document.name) - return False - schema_prefix, schema_version = _get_schema_parts(document) - return schema_prefix in self._schema_map.get(schema_version, {}) - - def validate(self, document): - return super(DataSchemaValidator, self).validate( - document, validate_section='data', use_fallback_schema=False) + yield self._generate_validation_error_output( + schema, document, error, root_path) class DocumentValidation(object): def __init__(self, documents, existing_data_schemas=None, - pre_validate=False): + pre_validate=True): """Class for document validation logic for documents. This class is responsible for validating documents according to their schema. + If ``pre_validate`` is true, then: + + * the base_schema validates ALL documents + * ALL built-in schemas validate the appropriate + document given a schema match + * NO externally registered DataSchema documents + are used for validation + + Else: + + * the base_schema validates ALL documents + * ALL built-in schemas validate the appropriate + document given a schema match + * ALL externally registered DataSchema documents + are used for validation given a schema match + :param documents: Documents to be validated. :type documents: List[dict] :param existing_data_schemas: ``DataSchema`` documents created in prior - revisions to be used the "data" section of each document in - ``documents``. Additional ``DataSchema`` documents in ``documents`` - are combined with these. + revisions to be used to validate the "data" section of each + document in ``documents``. Additional ``DataSchema`` documents in + ``documents`` are combined with these. :type existing_data_schemas: dict or List[dict] - :param pre_validate: Only runs validations from ``GenericValidator`` - and ``SchemaValidator`` against the documents if True. Otherwise - runs them all. This is useful to avoid spurious errors arising - from missing properties that may only exist post-substitution. - Default is False. + :param pre_validate: Whether to pre-validate documents using built-in + schema validation. Skips over externally registered ``DataSchema`` + documents to avoid false positives. Default is True. + :type pre_validate: bool """ - self.documents = [] - existing_data_schemas = existing_data_schemas or [] - data_schemas = [document_wrapper.DocumentDict(d) - for d in existing_data_schemas] - _data_schema_map = {d.name: d for d in data_schemas} + self._documents = [] + self._external_data_schemas = [document_wrapper.DocumentDict(d) + for d in existing_data_schemas or []] + data_schema_map = {d.name: d for d in self._external_data_schemas} raw_properties = ('data', 'metadata', 'schema') @@ -283,30 +387,30 @@ class DocumentValidation(object): document = document_wrapper.DocumentDict(raw_document) if document.schema.startswith(types.DATA_SCHEMA_SCHEMA): - data_schemas.append(document) + self._external_data_schemas.append(document) # If a newer version of the same DataSchema was passed in, # only use the new one and discard the old one. - if document.name in _data_schema_map: - data_schemas.remove(_data_schema_map.pop(document.name)) + if document.name in data_schema_map: + self._external_data_schemas.remove( + data_schema_map.pop(document.name)) - self.documents.append(document) + self._documents.append(document) # NOTE(fmontei): The order of the validators is important. The # ``GenericValidator`` must come first. self._validators = [ GenericValidator(), - SchemaValidator(), - DataSchemaValidator(data_schemas) + DataSchemaValidator(self._external_data_schemas) ] self._pre_validate = pre_validate def _get_supported_schema_list(self): schema_list = [] - for validator in self._validators[1:]: # Skip over `GenericValidator`. - for schema_version, schema_map in validator._schema_map.items(): - for schema_prefix in schema_map: - schema_list.append(schema_prefix + '/' + schema_version) + validator = self._validators[-1] + for schema_version, schema_map in validator._schema_map.items(): + for schema_name in schema_map: + schema_list.append(schema_name + '/' + schema_version) return schema_list def _format_validation_results(self, results): @@ -349,13 +453,10 @@ class DocumentValidation(object): supported_schema_list)) LOG.info(message) - validators = self._validators - if self._pre_validate is True: - validators = self._validators[:-1] - - for validator in validators: + for validator in self._validators: if validator.matches(document): - error_outputs = validator.validate(document) + error_outputs = validator.validate( + document, pre_validate=self._pre_validate) if error_outputs: for error_output in error_outputs: result['errors'].append(error_output) @@ -368,7 +469,7 @@ class DocumentValidation(object): return result def validate_all(self): - """Pre-validate that all documents are correctly formatted. + """Validate that all documents are correctly formatted. All concrete documents in the revision must successfully pass their JSON schema validations. The result of the validation is stored under @@ -377,28 +478,15 @@ class DocumentValidation(object): All abstract documents must themselves be sanity-checked. - Validation is broken up into 3 stages: + Validation is broken up into 2 "main" stages: 1) Validate that each document contains the basic bulding blocks needed: i.e. ``schema`` and ``metadata`` using a "base" schema. Failing this validation is deemed a critical failure, resulting in an exception. - .. note:: - - The ``data`` section, while mandatory, will not result in - critical failure. This is because a document can rely - on yet another document for ``data`` substitution. But - the validation for the document will be tagged as - ``failure``. - - 2) Validate each specific document type (e.g. validation policy) - using a more detailed schema. Failing this validation is deemed - non-critical, resulting in the error being recorded along with - any other non-critical exceptions, which are returned together - later. - - 3) Execute ``DataSchema`` validations if applicable. + 2) Execute ``DataSchema`` validations if applicable. Includes all + built-in ``DataSchema`` documents by default. :returns: A list of validations (one for each document validated). :rtype: List[dict] @@ -410,72 +498,9 @@ class DocumentValidation(object): validation_results = [] - for document in self.documents: + for document in self._documents: result = self._validate_one(document) validation_results.append(result) validations = self._format_validation_results(validation_results) return validations - - -def _get_schema_parts(document, schema_key='schema'): - schema_parts = utils.jsonpath_parse(document, schema_key).split('/') - schema_prefix = '/'.join(schema_parts[:2]) - schema_version = schema_parts[2] - if schema_version.endswith('.0'): - schema_version = schema_version[:-2] - return schema_prefix, schema_version - - -def _generate_validation_error_output(schema, document, error, root_path): - """Returns a formatted output with necessary details for debugging why - a validation failed. - - The response is a dictionary with the following keys: - - * validation_schema: The schema body that was used to validate the - document. - * schema_path: The JSON path in the schema where the failure originated. - * name: The document name. - * schema: The document schema. - * path: The JSON path in the document where the failure originated. - * error_section: The "section" in the document above which the error - originated (i.e. the dict in which ``path`` is found). - * message: The error message returned by the ``jsonschema`` validator. - - :returns: Dictionary in the above format. - """ - path_to_error_in_document = root_path + '.'.join( - [str(x) for x in error.path]) - path_to_error_in_schema = '.' + '.'.join( - [str(x) for x in error.schema_path]) - - parent_path_to_error_in_document = '.'.join( - path_to_error_in_document.split('.')[:-1]) or '.' - try: - # NOTE(fmontei): Because validation is performed on fully rendered - # documents, it is necessary to omit the parts of the data section - # where substitution may have occurred to avoid exposing any - # secrets. While this may make debugging a few validation failures - # more difficult, it is a necessary evil. - sanitized_document = SecretsSubstitution.sanitize_potential_secrets( - document) - parent_error_section = utils.jsonpath_parse( - sanitized_document, parent_path_to_error_in_document) - except Exception: - parent_error_section = ( - 'Failed to find parent section above where error occurred.') - - error_output = { - 'validation_schema': schema.schema, - 'schema_path': path_to_error_in_schema, - 'name': document.name, - 'schema': document.schema, - 'path': path_to_error_in_document, - 'error_section': parent_error_section, - # TODO(fmontei): Also sanitize any secrets contained in the message - # as well. - 'message': error.message - } - - return error_output diff --git a/deckhand/engine/layering.py b/deckhand/engine/layering.py index 971c8cb6..603481eb 100644 --- a/deckhand/engine/layering.py +++ b/deckhand/engine/layering.py @@ -245,7 +245,7 @@ class DocumentLayering(object): return result - def _validate_documents(self, documents): + def _pre_validate_documents(self, documents): LOG.debug('%s performing document pre-validation.', self.__class__.__name__) validator = document_validation.DocumentValidation( @@ -262,8 +262,9 @@ class DocumentLayering(object): 'Document [%s] %s failed with pre-validation error: %s.', *error) raise errors.InvalidDocumentFormat( - details='The following pre-validation errors occurred ' - '(schema, name, error): %s.' % val_errors) + document_schema=', '.join(v[0] for v in val_errors), + document_name=', '.join(v[1] for v in val_errors), + errors=', '.join(v[2] for v in val_errors)) def __init__(self, documents, substitution_sources=None, validate=True, fail_on_missing_sub_src=True): @@ -279,7 +280,8 @@ class DocumentLayering(object): sources for substitution. Should only include concrete documents. :type substitution_sources: List[dict] :param validate: Whether to pre-validate documents using built-in - schema validation. Default is True. + schema validation. Skips over externally registered ``DataSchema`` + documents to avoid false positives. Default is True. :type validate: bool :param fail_on_missing_sub_src: Whether to fail on a missing substitution source. Default is True. @@ -299,8 +301,9 @@ class DocumentLayering(object): self._documents_by_labels = {} self._layering_policy = None + # TODO(fmontei): Add a hook for post-validation too. if validate: - self._validate_documents(documents) + self._pre_validate_documents(documents) layering_policies = list( filter(lambda x: x.get('schema').startswith( diff --git a/deckhand/engine/schema/__init__.py b/deckhand/engine/schema/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/deckhand/engine/schema/base_schema.py b/deckhand/engine/schema/base_schema.py deleted file mode 100644 index 39295006..00000000 --- a/deckhand/engine/schema/base_schema.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2017 AT&T Intellectual Property. All other rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -schema = { - 'type': 'object', - 'properties': { - 'schema': { - 'type': 'string', - # Currently supported versions include v1/v1.0 only. - 'pattern': '^[A-Za-z]+\/[A-Za-z]+\/v\d+(.0)?$' - }, - 'metadata': { - 'type': 'object', - 'properties': { - 'schema': {'type': 'string'}, - 'name': {'type': 'string'} - }, - 'additionalProperties': True, - 'required': ['schema', 'name'] - }, - 'data': {'type': ['null', 'string', 'integer', 'array', 'object']} - }, - 'additionalProperties': False, - 'required': ['schema', 'metadata'] -} -"""Base JSON schema against which all Deckhand documents are validated. - -.. literalinclude:: ../../deckhand/engine/schema/base_schema.py - :language: python - :lines: 15-36 - -This schema is used to sanity-check all documents that are passed to Deckhand. -Failure to pass this schema results in a critical error. -""" - -__all__ = ['schema'] diff --git a/deckhand/engine/schema/v1_0/__init__.py b/deckhand/engine/schema/v1_0/__init__.py deleted file mode 100644 index 2948f9c8..00000000 --- a/deckhand/engine/schema/v1_0/__init__.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2017 AT&T Intellectual Property. All other rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from deckhand.engine.schema.v1_0 import certificate_authority_key_schema -from deckhand.engine.schema.v1_0 import certificate_authority_schema -from deckhand.engine.schema.v1_0 import certificate_key_schema -from deckhand.engine.schema.v1_0 import certificate_schema -from deckhand.engine.schema.v1_0 import data_schema_schema -from deckhand.engine.schema.v1_0 import document_schema -from deckhand.engine.schema.v1_0 import layering_policy_schema -from deckhand.engine.schema.v1_0 import passphrase_schema -from deckhand.engine.schema.v1_0 import private_key_schema -from deckhand.engine.schema.v1_0 import public_key_schema -from deckhand.engine.schema.v1_0 import validation_policy_schema - -__all__ = ['certificate_key_schema', 'certificate_schema', - 'certificate_authority_key_schema', 'certificate_authority_schema', - 'private_key_schema', 'public_key_schema', - 'data_schema_schema', 'document_schema', 'layering_policy_schema', - 'passphrase_schema', 'validation_policy_schema'] diff --git a/deckhand/engine/schema/v1_0/certificate_authority_key_schema.py b/deckhand/engine/schema/v1_0/certificate_authority_key_schema.py deleted file mode 100644 index fee6508b..00000000 --- a/deckhand/engine/schema/v1_0/certificate_authority_key_schema.py +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright 2017 AT&T Intellectual Property. All other rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -schema = { - 'type': 'object', - 'properties': { - 'schema': { - 'type': 'string', - 'pattern': ( - '^(deckhand/CertificateAuthorityKey/v[1]{1}(\.[0]{1}){0,1})$') - }, - 'metadata': { - 'type': 'object', - 'properties': { - 'schema': { - 'type': 'string', - 'pattern': '^(metadata/Document/v[1]{1}(\.[0]{1}){0,1})$', - }, - 'name': {'type': 'string'}, - # Not strictly needed for secrets. - 'layeringDefinition': { - 'type': 'object', - 'properties': { - 'layer': {'type': 'string'} - } - }, - 'storagePolicy': { - 'type': 'string', - 'enum': ['encrypted', 'cleartext'] - } - }, - 'additionalProperties': False, - 'required': ['schema', 'name', 'storagePolicy'] - }, - 'data': {'type': 'string'} - }, - 'additionalProperties': False, - 'required': ['schema', 'metadata', 'data'] -} -"""JSON schema against which all documents with -``deckhand/CertificateAuthorityKey/v1`` ``schema`` are validated. - -.. literalinclude:: - ../../deckhand/engine/schema/v1_0/certificate_authority_key_schema.py - :language: python - :lines: 15-49 - -This schema is used to sanity-check all CertificateAuthorityKey documents that -are passed to Deckhand. This schema is only enforced after validation for -:py:data:`~deckhand.engine.schema.base_schema` has passed. Failure to pass this -schema will result in an error entry being created for the validation with name -``deckhand-schema-validation`` corresponding to the created revision. -""" - -__all__ = ['schema'] diff --git a/deckhand/engine/schema/v1_0/certificate_authority_schema.py b/deckhand/engine/schema/v1_0/certificate_authority_schema.py deleted file mode 100644 index faf318e0..00000000 --- a/deckhand/engine/schema/v1_0/certificate_authority_schema.py +++ /dev/null @@ -1,66 +0,0 @@ -# Copyright 2017 AT&T Intellectual Property. All other rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -schema = { - 'type': 'object', - 'properties': { - 'schema': { - 'type': 'string', - 'pattern': ( - '^(deckhand/CertificateAuthority/v[1]{1}(\.[0]{1}){0,1})$') - }, - 'metadata': { - 'type': 'object', - 'properties': { - 'schema': { - 'type': 'string', - 'pattern': '^(metadata/Document/v[1]{1}(\.[0]{1}){0,1})$', - }, - 'name': {'type': 'string'}, - # Not strictly needed for secrets. - 'layeringDefinition': { - 'type': 'object', - 'properties': { - 'layer': {'type': 'string'} - } - }, - 'storagePolicy': { - 'type': 'string', - 'enum': ['encrypted', 'cleartext'] - } - }, - 'additionalProperties': False, - 'required': ['schema', 'name', 'storagePolicy'] - }, - 'data': {'type': 'string'} - }, - 'additionalProperties': False, - 'required': ['schema', 'metadata', 'data'] -} -"""JSON schema against which all documents with -``deckhand/CertificateAuthority/v1`` ``schema`` are validated. - -.. literalinclude:: - ../../deckhand/engine/schema/v1_0/certificate_authority_schema.py - :language: python - :lines: 15-50 - -This schema is used to sanity-check all CertificateAuthority documents that are -passed to Deckhand. This schema is only enforced after validation for -:py:data:`~deckhand.engine.schema.base_schema` has passed. Failure to pass -this schema will result in an error entry being created for the validation -with name ``deckhand-schema-validation`` corresponding to the created revision. -""" - -__all__ = ['schema'] diff --git a/deckhand/engine/schema/v1_0/certificate_key_schema.py b/deckhand/engine/schema/v1_0/certificate_key_schema.py deleted file mode 100644 index 76d6c9d0..00000000 --- a/deckhand/engine/schema/v1_0/certificate_key_schema.py +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright 2017 AT&T Intellectual Property. All other rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -schema = { - 'type': 'object', - 'properties': { - 'schema': { - 'type': 'string', - 'pattern': '^deckhand/CertificateKey/v\d+(.0)?$' - }, - 'metadata': { - 'type': 'object', - 'properties': { - 'schema': { - 'type': 'string', - 'pattern': '^metadata/Document/v\d+(.0)?$', - }, - 'name': {'type': 'string'}, - # Not strictly needed for secrets. - 'layeringDefinition': { - 'type': 'object', - 'properties': { - 'layer': {'type': 'string'}, - 'abstract': {'type': 'boolean'} - } - }, - 'storagePolicy': { - 'type': 'string', - 'enum': ['encrypted', 'cleartext'] - } - }, - 'additionalProperties': False, - 'required': ['schema', 'name', 'storagePolicy'] - }, - 'data': {'type': 'string'} - }, - 'additionalProperties': False, - 'required': ['schema', 'metadata', 'data'] -} -"""JSON schema against which all documents with ``deckhand/CertificateKey/v1`` -``schema`` are validated. - -.. literalinclude:: ../../deckhand/engine/schema/v1_0/certificate_key_schema.py - :language: python - :lines: 15-49 - -This schema is used to sanity-check all CertificateKey documents that are -passed to Deckhand. This schema is only enforced after validation for -:py:data:`~deckhand.engine.schema.base_schema` has passed. Failure to pass -this schema will result in an error entry being created for the validation -with name ``deckhand-schema-validation`` corresponding to the created revision. -""" - -__all__ = ['schema'] diff --git a/deckhand/engine/schema/v1_0/certificate_schema.py b/deckhand/engine/schema/v1_0/certificate_schema.py deleted file mode 100644 index 54c87a19..00000000 --- a/deckhand/engine/schema/v1_0/certificate_schema.py +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright 2017 AT&T Intellectual Property. All other rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -schema = { - 'type': 'object', - 'properties': { - 'schema': { - 'type': 'string', - 'pattern': '^deckhand/Certificate/v\d+(.0)?$' - }, - 'metadata': { - 'type': 'object', - 'properties': { - 'schema': { - 'type': 'string', - 'pattern': '^metadata/Document/v\d+(.0)?$', - }, - 'name': {'type': 'string'}, - # Not strictly needed for secrets. - 'layeringDefinition': { - 'type': 'object', - 'properties': { - 'layer': {'type': 'string'}, - 'abstract': {'type': 'boolean'} - } - }, - 'storagePolicy': { - 'type': 'string', - 'enum': ['encrypted', 'cleartext'] - } - }, - 'additionalProperties': False, - 'required': ['schema', 'name', 'storagePolicy'] - }, - 'data': {'type': 'string'} - }, - 'additionalProperties': False, - 'required': ['schema', 'metadata', 'data'] -} -"""JSON schema against which all documents with ``deckhand/Certificate/v1`` -``schema`` are validated. - -.. literalinclude:: ../../deckhand/engine/schema/v1_0/certificate_schema.py - :language: python - :lines: 15-49 - -This schema is used to sanity-check all Certificate documents that are -passed to Deckhand. This schema is only enforced after validation for -:py:data:`~deckhand.engine.schema.base_schema` has passed. Failure to pass -this schema will result in an error entry being created for the validation -with name ``deckhand-schema-validation`` corresponding to the created revision. -""" - -__all__ = ['schema'] diff --git a/deckhand/engine/schema/v1_0/data_schema_schema.py b/deckhand/engine/schema/v1_0/data_schema_schema.py deleted file mode 100644 index 0549adde..00000000 --- a/deckhand/engine/schema/v1_0/data_schema_schema.py +++ /dev/null @@ -1,76 +0,0 @@ -# Copyright 2017 AT&T Intellectual Property. All other rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This specifies the official JSON schema meta-schema. DataSchema documents -# are used by various services to register new schemas that Deckhand can use -# for validation. -schema = { - 'type': 'object', - 'properties': { - 'schema': { - 'type': 'string', - 'pattern': '^deckhand/DataSchema/v\d+(.0)?$' - }, - 'metadata': { - 'type': 'object', - 'properties': { - 'schema': { - 'type': 'string', - 'pattern': '^metadata/Control/v\d+(.0)?$' - }, - 'name': { - 'type': 'string', - 'pattern': '^[A-Za-z]+\/[A-Za-z]+\/v\d+(.0)?$' - }, - # Labels are optional. - 'labels': { - 'type': 'object' - }, - 'storagePolicy': { - 'type': 'string', - 'enum': ['encrypted', 'cleartext'] - } - }, - 'additionalProperties': True, # Can include layeringDefinition. - 'required': ['schema', 'name'] - }, - 'data': { - 'type': 'object', - 'properties': { - '$schema': { - 'type': ['string', 'integer', 'array', 'object'] - } - }, - 'additionalProperties': True, - 'required': ['$schema'] - } - }, - 'additionalProperties': False, - 'required': ['schema', 'metadata', 'data'] -} -"""JSON schema against which all documents with ``deckhand/DataSchema/v1`` -``schema`` are validated. - -.. literalinclude:: ../../deckhand/engine/schema/v1_0/data_schema_schema.py - :language: python - :lines: 15-61 - -This schema is used to sanity-check all DataSchema documents that are -passed to Deckhand. This schema is only enforced after validation for -:py:data:`~deckhand.engine.schema.base_schema` has passed. Failure to pass -this schema will result in an error entry being created for the validation -with name ``deckhand-schema-validation`` corresponding to the created revision. -""" - -__all__ = ['schema'] diff --git a/deckhand/engine/schema/v1_0/document_schema.py b/deckhand/engine/schema/v1_0/document_schema.py deleted file mode 100644 index d5591a4f..00000000 --- a/deckhand/engine/schema/v1_0/document_schema.py +++ /dev/null @@ -1,123 +0,0 @@ -# Copyright 2017 AT&T Intellectual Property. All other rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -substitution_schema = { - 'type': 'object', - 'properties': { - 'dest': { - 'type': 'object', - 'properties': { - 'path': {'type': 'string'}, - 'pattern': {'type': 'string'} - }, - 'additionalProperties': False, - 'required': ['path'] - }, - 'src': { - 'type': 'object', - 'properties': { - 'schema': { - 'type': 'string', - 'pattern': '^[A-Za-z]+/[A-Za-z]+/v\d+(.0)?$' - }, - 'name': {'type': 'string'}, - 'path': {'type': 'string'} - }, - 'additionalProperties': False, - 'required': ['schema', 'name', 'path'] - } - }, - 'additionalProperties': False, - 'required': ['dest', 'src'] -} - -schema = { - 'type': 'object', - 'properties': { - 'schema': { - 'type': 'string', - 'pattern': '^[A-Za-z]+/[A-Za-z]+/v\d+(.0)?$' - }, - 'metadata': { - 'type': 'object', - 'properties': { - 'schema': { - 'type': 'string', - 'pattern': '^metadata/Document/v\d+(.0)?$' - }, - 'name': {'type': 'string'}, - 'labels': {'type': 'object'}, - 'layeringDefinition': { - 'type': 'object', - 'properties': { - 'layer': {'type': 'string'}, - 'abstract': {'type': 'boolean'}, - # "parentSelector" is optional. - 'parentSelector': {'type': 'object'}, - # "actions" is optional. - 'actions': { - 'type': 'array', - 'items': { - 'type': 'object', - 'properties': { - 'method': {'enum': ['replace', 'delete', - 'merge']}, - 'path': {'type': 'string'} - }, - 'additionalProperties': False, - 'required': ['method', 'path'] - } - } - }, - 'additionalProperties': False, - 'required': ['layer'] - }, - # "substitutions" is optional. - 'substitutions': { - 'type': 'array', - 'items': substitution_schema - }, - 'storagePolicy': { - 'type': 'string', - 'enum': ['encrypted', 'cleartext'] - } - }, - 'additionalProperties': False, - 'required': ['schema', 'name', 'layeringDefinition'] - }, - 'data': { - 'type': ['string', 'integer', 'array', 'object'] - } - }, - 'additionalProperties': False, - 'required': ['schema', 'metadata', 'data'] -} -"""JSON schema against which all documents with ``metadata/Document/v1`` -``metadata.schema`` are validated. - -.. literalinclude:: ../../deckhand/engine/schema/v1_0/document_schema.py - :language: python - :lines: 15-102 - -This schema is used to sanity-check all "metadata/Document" documents that are -passed to Deckhand. This validation comes into play when a new schema is -registered under the ``data`` section of a ``deckhand/DataSchema/v1`` document. - -This schema is only enforced after validation for -:py:data:`~deckhand.engine.schema.base_schema` has passed. Failure to pass -this schema will result in an error entry being created for the validation -with name ``deckhand-schema-validation`` corresponding to the created revision. -""" - -__all__ = ['schema'] diff --git a/deckhand/engine/schema/v1_0/layering_policy_schema.py b/deckhand/engine/schema/v1_0/layering_policy_schema.py deleted file mode 100644 index 277f63f9..00000000 --- a/deckhand/engine/schema/v1_0/layering_policy_schema.py +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright 2017 AT&T Intellectual Property. All other rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -schema = { - 'type': 'object', - 'properties': { - 'schema': { - 'type': 'string', - 'pattern': '^deckhand/LayeringPolicy/v\d+(.0)?$' - }, - 'metadata': { - 'type': 'object', - 'properties': { - 'schema': { - 'type': 'string', - 'pattern': '^metadata/Control/v\d+(.0)?$' - }, - 'name': {'type': 'string'}, - 'storagePolicy': { - 'type': 'string', - 'enum': ['encrypted', 'cleartext'] - }, - 'layeringDefinition': { - 'type': 'object', - 'properties': { - 'abstract': {'type': 'boolean'} - }, - 'additionalProperties': False - } - }, - 'additionalProperties': False, - 'required': ['schema', 'name'] - }, - 'data': { - 'type': 'object', - 'properties': { - 'layerOrder': { - 'type': 'array', - 'items': {'type': 'string'} - } - }, - 'additionalProperties': True, - 'required': ['layerOrder'] - } - }, - 'additionalProperties': False, - 'required': ['schema', 'metadata', 'data'] -} -"""JSON schema against which all documents with ``deckhand/LayeringPolicy/v1`` -``schema`` are validated. - -.. literalinclude:: ../../deckhand/engine/schema/v1_0/layering_policy_schema.py - :language: python - :lines: 15-52 - -This schema is used to sanity-check all LayeringPolicy documents that are -passed to Deckhand. This schema is only enforced after validation for -:py:data:`~deckhand.engine.schema.base_schema` has passed. Failure to pass -this schema will result in an error entry being created for the validation -with name ``deckhand-schema-validation`` corresponding to the created revision. -""" - -__all__ = ['schema'] diff --git a/deckhand/engine/schema/v1_0/passphrase_schema.py b/deckhand/engine/schema/v1_0/passphrase_schema.py deleted file mode 100644 index 02ddb928..00000000 --- a/deckhand/engine/schema/v1_0/passphrase_schema.py +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright 2017 AT&T Intellectual Property. All other rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -schema = { - 'type': 'object', - 'properties': { - 'schema': { - 'type': 'string', - 'pattern': '^deckhand/Passphrase/v\d+(.0)?$' - }, - 'metadata': { - 'type': 'object', - 'properties': { - 'schema': { - 'type': 'string', - 'pattern': '^metadata/Document/v\d+(.0)?$', - }, - 'name': {'type': 'string'}, - # Not strictly needed. - 'layeringDefinition': { - 'type': 'object', - 'properties': { - 'layer': {'type': 'string'}, - 'abstract': {'type': 'boolean'} - } - }, - 'storagePolicy': { - 'type': 'string', - 'enum': ['encrypted', 'cleartext'] - } - }, - 'additionalProperties': False, - 'required': ['schema', 'name', 'storagePolicy'] - }, - 'data': {'type': 'string'} - }, - 'additionalProperties': False, - 'required': ['schema', 'metadata', 'data'] -} -"""JSON schema against which all documents with ``deckhand/Passphrase/v1`` -``schema`` are validated. - -.. literalinclude:: ../../deckhand/engine/schema/v1_0/passphrase_schema.py - :language: python - :lines: 15-49 - -This schema is used to sanity-check all Passphrase documents that are -passed to Deckhand. This schema is only enforced after validation for -:py:data:`~deckhand.engine.schema.base_schema` has passed. Failure to pass -this schema will result in an error entry being created for the validation -with name ``deckhand-schema-validation`` corresponding to the created revision. -""" - -__all__ = ['schema'] diff --git a/deckhand/engine/schema/v1_0/private_key_schema.py b/deckhand/engine/schema/v1_0/private_key_schema.py deleted file mode 100644 index c34efe78..00000000 --- a/deckhand/engine/schema/v1_0/private_key_schema.py +++ /dev/null @@ -1,64 +0,0 @@ -# Copyright 2017 AT&T Intellectual Property. All other rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -schema = { - 'type': 'object', - 'properties': { - 'schema': { - 'type': 'string', - 'pattern': '^(deckhand/PrivateKey/v[1]{1}(\.[0]{1}){0,1})$' - }, - 'metadata': { - 'type': 'object', - 'properties': { - 'schema': { - 'type': 'string', - 'pattern': '^(metadata/Document/v[1]{1}(\.[0]{1}){0,1})$', - }, - 'name': {'type': 'string'}, - # Not strictly needed for secrets. - 'layeringDefinition': { - 'type': 'object', - 'properties': { - 'layer': {'type': 'string'} - } - }, - 'storagePolicy': { - 'type': 'string', - 'enum': ['encrypted', 'cleartext'] - } - }, - 'additionalProperties': False, - 'required': ['schema', 'name', 'storagePolicy'] - }, - 'data': {'type': 'string'} - }, - 'additionalProperties': False, - 'required': ['schema', 'metadata', 'data'] -} -"""JSON schema against which all documents with ``deckhand/PrivateKey/v1`` -``schema`` are validated. - -.. literalinclude:: ../../deckhand/engine/schema/v1_0/private_key_schema.py - :language: python - :lines: 15-49 - -This schema is used to sanity-check all PrivateKey documents that are -passed to Deckhand. This schema is only enforced after validation for -:py:data:`~deckhand.engine.schema.base_schema` has passed. Failure to pass -this schema will result in an error entry being created for the validation -with name ``deckhand-schema-validation`` corresponding to the created revision. -""" - -__all__ = ['schema'] diff --git a/deckhand/engine/schema/v1_0/public_key_schema.py b/deckhand/engine/schema/v1_0/public_key_schema.py deleted file mode 100644 index e0a2ff2a..00000000 --- a/deckhand/engine/schema/v1_0/public_key_schema.py +++ /dev/null @@ -1,64 +0,0 @@ -# Copyright 2017 AT&T Intellectual Property. All other rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -schema = { - 'type': 'object', - 'properties': { - 'schema': { - 'type': 'string', - 'pattern': '^(deckhand/PublicKey/v[1]{1}(\.[0]{1}){0,1})$' - }, - 'metadata': { - 'type': 'object', - 'properties': { - 'schema': { - 'type': 'string', - 'pattern': '^(metadata/Document/v[1]{1}(\.[0]{1}){0,1})$', - }, - 'name': {'type': 'string'}, - # Not strictly needed for secrets. - 'layeringDefinition': { - 'type': 'object', - 'properties': { - 'layer': {'type': 'string'} - } - }, - 'storagePolicy': { - 'type': 'string', - 'enum': ['encrypted', 'cleartext'] - } - }, - 'additionalProperties': False, - 'required': ['schema', 'name', 'storagePolicy'] - }, - 'data': {'type': 'string'} - }, - 'additionalProperties': False, - 'required': ['schema', 'metadata', 'data'] -} -"""JSON schema against which all documents with ``deckhand/PublicKey/v1`` -``schema`` are validated. - -.. literalinclude:: ../../deckhand/engine/schema/v1_0/public_key_schema.py - :language: python - :lines: 15-49 - -This schema is used to sanity-check all PublicKey documents that are -passed to Deckhand. This schema is only enforced after validation for -:py:data:`~deckhand.engine.schema.base_schema` has passed. Failure to pass -this schema will result in an error entry being created for the validation -with name ``deckhand-schema-validation`` corresponding to the created revision. -""" - -__all__ = ['schema'] diff --git a/deckhand/engine/schema/v1_0/validation_policy_schema.py b/deckhand/engine/schema/v1_0/validation_policy_schema.py deleted file mode 100644 index 08fef701..00000000 --- a/deckhand/engine/schema/v1_0/validation_policy_schema.py +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright 2017 AT&T Intellectual Property. All other rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -schema = { - 'type': 'object', - 'properties': { - 'schema': { - 'type': 'string', - 'pattern': '^deckhand/ValidationPolicy/v\d+(.0)?$' - }, - 'metadata': { - 'type': 'object', - 'properties': { - 'schema': { - 'type': 'string', - 'pattern': '^metadata/Control/v\d+(.0)?$' - }, - 'name': {'type': 'string'}, - 'storagePolicy': { - 'type': 'string', - 'enum': ['encrypted', 'cleartext'] - } - }, - 'additionalProperties': False, - 'required': ['schema', 'name'] - }, - 'data': { - 'type': 'object', - 'properties': { - 'validations': { - 'type': 'array', - 'items': { - 'type': 'object', - 'properties': { - 'name': { - 'type': 'string', - 'pattern': '^.*-(validation|verification)$' - }, - # 'expiresAfter' is optional. - 'expiresAfter': {'type': 'string'} - }, - 'additionalProperties': False, - 'required': ['name'] - } - } - }, - 'additionalProperties': True, - 'required': ['validations'] - } - }, - 'additionalProperties': False, - 'required': ['schema', 'metadata', 'data'] -} -"""JSON schema against which all documents with -``deckhand/ValidationPolicy/v1`` ``schema`` are validated. - -.. literalinclude:: - ../../deckhand/engine/schema/v1_0/validation_policy_schema.py - :language: python - :lines: 15-64 - -This schema is used to sanity-check all ValidationPolicy documents that are -passed to Deckhand. This schema is only enforced after validation for -:py:data:`~deckhand.engine.schema.base_schema` has passed. Failure to pass -this schema will result in an error entry being created for the validation -with name ``deckhand-schema-validation`` corresponding to the created revision. -""" - -__all__ = ['schema'] diff --git a/deckhand/engine/schemas/base_schema.yaml b/deckhand/engine/schemas/base_schema.yaml new file mode 100644 index 00000000..6b72294c --- /dev/null +++ b/deckhand/engine/schemas/base_schema.yaml @@ -0,0 +1,119 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +schema: deckhand/DataSchema/v1 +metadata: + name: deckhand/Base/v1 + schema: metadata/Control/v1 +data: + $schema: http://json-schema.org/schema# + properties: + schema: + type: string + pattern: ^[A-Za-z]+/[A-Za-z]+/v\d+(.0)?$ + metadata: + type: object + properties: + schema: + anyOf: + - type: string + pattern: ^metadata/Document/v\d+(.0)?$ + - type: string + pattern: ^metadata/Control/v\d+(.0)?$ + name: + type: string + labels: + type: object + layeringDefinition: + type: object + properties: + layer: + type: string + abstract: + type: boolean + parentSelector: + type: object + actions: + type: array + items: + type: object + properties: + method: + enum: + - replace + - delete + - merge + path: + type: string + additionalProperties: false + required: + - method + - path + additionalProperties: false + substitutions: + type: array + items: + type: object + properties: + dest: + type: object + properties: + path: + type: string + pattern: + type: string + additionalProperties: false + required: + - path + src: + type: object + properties: + schema: + type: string + pattern: ^[A-Za-z]+/[A-Za-z]+/v\d+(.0)?$ + name: + type: string + path: + type: string + additionalProperties: false + required: + - schema + - name + - path + additionalProperties: false + required: + - dest + - src + storagePolicy: + type: string + enum: + - encrypted + - cleartext + additionalProperties: false + required: + - schema + - name + data: + type: + - "null" + - string + - integer + - array + - object + additionalProperties: false + required: + - schema + - metadata + - data diff --git a/deckhand/engine/schemas/certificate_authority_key_schema.yaml b/deckhand/engine/schemas/certificate_authority_key_schema.yaml new file mode 100644 index 00000000..3eaaf537 --- /dev/null +++ b/deckhand/engine/schemas/certificate_authority_key_schema.yaml @@ -0,0 +1,65 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +schema: deckhand/DataSchema/v1 +metadata: + name: deckhand/CertificateAuthorityKey/v1 + schema: metadata/Control/v1 +data: + $schema: http://json-schema.org/schema# + properties: + metadata: + type: object + properties: + layeringDefinition: + type: object + properties: + layer: + type: string + abstract: + type: boolean + parentSelector: + type: object + actions: + type: array + items: + type: object + properties: + method: + enum: + - replace + - delete + - merge + path: + type: string + additionalProperties: false + required: + - method + - path + required: + - layer + storagePolicy: + type: string + enum: + - encrypted + - cleartext + required: + - layeringDefinition + - storagePolicy + data: + type: string +required: + - metadata + - data diff --git a/deckhand/engine/schemas/certificate_authority_schema.yaml b/deckhand/engine/schemas/certificate_authority_schema.yaml new file mode 100644 index 00000000..15e5b896 --- /dev/null +++ b/deckhand/engine/schemas/certificate_authority_schema.yaml @@ -0,0 +1,65 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +schema: deckhand/DataSchema/v1 +metadata: + name: deckhand/CertificateAuthority/v1 + schema: metadata/Control/v1 +data: + $schema: http://json-schema.org/schema# + properties: + metadata: + type: object + properties: + layeringDefinition: + type: object + properties: + layer: + type: string + abstract: + type: boolean + parentSelector: + type: object + actions: + type: array + items: + type: object + properties: + method: + enum: + - replace + - delete + - merge + path: + type: string + additionalProperties: false + required: + - method + - path + required: + - layer + storagePolicy: + type: string + enum: + - encrypted + - cleartext + required: + - layeringDefinition + - storagePolicy + data: + type: string +required: + - metadata + - data diff --git a/deckhand/engine/schemas/certificate_key_schema.yaml b/deckhand/engine/schemas/certificate_key_schema.yaml new file mode 100644 index 00000000..553fc110 --- /dev/null +++ b/deckhand/engine/schemas/certificate_key_schema.yaml @@ -0,0 +1,65 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +schema: deckhand/DataSchema/v1 +metadata: + name: deckhand/CertificateKey/v1 + schema: metadata/Control/v1 +data: + $schema: http://json-schema.org/schema# + properties: + metadata: + type: object + properties: + layeringDefinition: + type: object + properties: + layer: + type: string + abstract: + type: boolean + parentSelector: + type: object + actions: + type: array + items: + type: object + properties: + method: + enum: + - replace + - delete + - merge + path: + type: string + additionalProperties: false + required: + - method + - path + required: + - layer + storagePolicy: + type: string + enum: + - encrypted + - cleartext + required: + - layeringDefinition + - storagePolicy + data: + type: string +required: + - metadata + - data diff --git a/deckhand/engine/schemas/certificate_schema.yaml b/deckhand/engine/schemas/certificate_schema.yaml new file mode 100644 index 00000000..0c40e5bc --- /dev/null +++ b/deckhand/engine/schemas/certificate_schema.yaml @@ -0,0 +1,65 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +schema: deckhand/DataSchema/v1 +metadata: + name: deckhand/Certificate/v1 + schema: metadata/Control/v1 +data: + $schema: http://json-schema.org/schema# + properties: + metadata: + type: object + properties: + layeringDefinition: + type: object + properties: + layer: + type: string + abstract: + type: boolean + parentSelector: + type: object + actions: + type: array + items: + type: object + properties: + method: + enum: + - replace + - delete + - merge + path: + type: string + additionalProperties: false + required: + - method + - path + required: + - layer + storagePolicy: + type: string + enum: + - encrypted + - cleartext + required: + - layeringDefinition + - storagePolicy + data: + type: string +required: + - metadata + - data diff --git a/deckhand/engine/schemas/dataschema_schema.yaml b/deckhand/engine/schemas/dataschema_schema.yaml new file mode 100644 index 00000000..2865bbe5 --- /dev/null +++ b/deckhand/engine/schemas/dataschema_schema.yaml @@ -0,0 +1,33 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +schema: deckhand/DataSchema/v1 +metadata: + name: deckhand/DataSchema/v1 + schema: metadata/Control/v1 +data: + $schema: http://json-schema.org/schema# + type: object + properties: + data: + type: object + properties: + $schema: + type: string + additionalProperties: true + required: + - $schema + required: + - data diff --git a/deckhand/engine/schemas/layering_policy_schema.yaml b/deckhand/engine/schemas/layering_policy_schema.yaml new file mode 100644 index 00000000..5afba5d8 --- /dev/null +++ b/deckhand/engine/schemas/layering_policy_schema.yaml @@ -0,0 +1,35 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +schema: deckhand/DataSchema/v1 +metadata: + name: deckhand/LayeringPolicy/v1 + schema: metadata/Control/v1 +data: + $schema: http://json-schema.org/schema# + type: object + properties: + data: + type: object + properties: + layerOrder: + type: array + items: + type: string + additionalProperties: false + required: + - layerOrder + required: + - data diff --git a/deckhand/engine/schemas/passphrase_schema.yaml b/deckhand/engine/schemas/passphrase_schema.yaml new file mode 100644 index 00000000..949e3458 --- /dev/null +++ b/deckhand/engine/schemas/passphrase_schema.yaml @@ -0,0 +1,65 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +schema: deckhand/DataSchema/v1 +metadata: + name: deckhand/Passphrase/v1 + schema: metadata/Control/v1 +data: + $schema: http://json-schema.org/schema# + properties: + metadata: + type: object + properties: + layeringDefinition: + type: object + properties: + layer: + type: string + abstract: + type: boolean + parentSelector: + type: object + actions: + type: array + items: + type: object + properties: + method: + enum: + - replace + - delete + - merge + path: + type: string + additionalProperties: false + required: + - method + - path + required: + - layer + storagePolicy: + type: string + enum: + - encrypted + - cleartext + required: + - layeringDefinition + - storagePolicy + data: + type: string +required: + - metadata + - data diff --git a/deckhand/engine/schemas/private_key_schema.yaml b/deckhand/engine/schemas/private_key_schema.yaml new file mode 100644 index 00000000..6c6fd6d0 --- /dev/null +++ b/deckhand/engine/schemas/private_key_schema.yaml @@ -0,0 +1,65 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +schema: deckhand/DataSchema/v1 +metadata: + name: deckhand/PrivateKey/v1 + schema: metadata/Control/v1 +data: + $schema: http://json-schema.org/schema# + properties: + metadata: + type: object + properties: + layeringDefinition: + type: object + properties: + layer: + type: string + abstract: + type: boolean + parentSelector: + type: object + actions: + type: array + items: + type: object + properties: + method: + enum: + - replace + - delete + - merge + path: + type: string + additionalProperties: false + required: + - method + - path + required: + - layer + storagePolicy: + type: string + enum: + - encrypted + - cleartext + required: + - layeringDefinition + - storagePolicy + data: + type: string +required: + - metadata + - data diff --git a/deckhand/engine/schemas/public_key_schema.yaml b/deckhand/engine/schemas/public_key_schema.yaml new file mode 100644 index 00000000..de9887de --- /dev/null +++ b/deckhand/engine/schemas/public_key_schema.yaml @@ -0,0 +1,65 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +schema: deckhand/DataSchema/v1 +metadata: + name: deckhand/PublicKey/v1 + schema: metadata/Control/v1 +data: + $schema: http://json-schema.org/schema# + properties: + metadata: + type: object + properties: + layeringDefinition: + type: object + properties: + layer: + type: string + abstract: + type: boolean + parentSelector: + type: object + actions: + type: array + items: + type: object + properties: + method: + enum: + - replace + - delete + - merge + path: + type: string + additionalProperties: false + required: + - method + - path + required: + - layer + storagePolicy: + type: string + enum: + - encrypted + - cleartext + required: + - layeringDefinition + - storagePolicy + data: + type: string +required: + - metadata + - data diff --git a/deckhand/engine/schemas/validation_policy_schema.yaml b/deckhand/engine/schemas/validation_policy_schema.yaml new file mode 100644 index 00000000..ff4f090a --- /dev/null +++ b/deckhand/engine/schemas/validation_policy_schema.yaml @@ -0,0 +1,76 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +schema: deckhand/DataSchema/v1 +metadata: + name: deckhand/ValidationPolicy/v1 + schema: metadata/Control/v1 +data: + $schema: http://json-schema.org/schema# + type: object + properties: + metadata: + type: object + properties: + layeringDefinition: + type: object + properties: + layer: + type: string + abstract: + type: boolean + parentSelector: + type: object + actions: + type: array + items: + type: object + properties: + method: + enum: + - replace + - delete + - merge + path: + type: string + additionalProperties: false + required: + - method + - path + required: + - layer + required: + - layeringDefinition + data: + properties: + validations: + type: array + items: + type: object + properties: + name: + type: string + pattern: ^.*-(validation|verification)$ + expiresAfter: + type: string + additionalProperties: false + required: + - name + required: + - validations + additionalProperties: false + required: + - metadata + - data diff --git a/deckhand/errors.py b/deckhand/errors.py index e27c04bd..91a70a57 100644 --- a/deckhand/errors.py +++ b/deckhand/errors.py @@ -175,8 +175,8 @@ class InvalidDocumentFormat(DeckhandException): **Troubleshoot:** """ - msg_fmt = ("The provided document(s) failed schema validation. Details: " - "%(details)s") + msg_fmt = ("The provided document [%(document_schema)s] %(document_name)s " + "failed schema validation. Errors: %(errors)s") code = 400 @@ -206,7 +206,6 @@ class InvalidDocumentParent(DeckhandException): msg_fmt = ("The document parent [%(parent_schema)s] %(parent_name)s is " "invalid for document [%(document_schema)s] %(document_name)s. " "Reason: %(reason)s") - code = 400 class IndeterminateDocumentParent(DeckhandException): diff --git a/deckhand/factories.py b/deckhand/factories.py index 51606806..c967380e 100644 --- a/deckhand/factories.py +++ b/deckhand/factories.py @@ -27,14 +27,9 @@ DOCUMENT_TEST_SCHEMA = 'example/Kind/v1' @six.add_metaclass(abc.ABCMeta) class DeckhandFactory(object): - # TODO(fmontei): Allow this to be overriden in ``__init__``. + # TODO(fmontei): Allow this to be overridden in ``__init__``. API_VERSION = '1.0' - @abc.abstractmethod - def gen(self, *args): - """Generate an object for usage by the Deckhand `engine` module.""" - pass - @abc.abstractmethod def gen_test(self, *args, **kwargs): """Generate an object with randomized values for a test.""" @@ -51,7 +46,11 @@ class DataSchemaFactory(DeckhandFactory): "metadata": { "schema": "metadata/Control/v1", "name": "", - "labels": {} + "labels": {}, + "layeringDefinition": { + "abstract": True, + "layer": "site" + } }, "schema": "deckhand/DataSchema/v1" } @@ -73,9 +72,6 @@ class DataSchemaFactory(DeckhandFactory): ... """ - def gen(self): - raise NotImplementedError() - def gen_test(self, metadata_name, data, **metadata_labels): data_schema_template = copy.deepcopy(self.DATA_SCHEMA_TEMPLATE) @@ -90,18 +86,22 @@ class DataSchemaFactory(DeckhandFactory): class DocumentFactory(DeckhandFactory): """Class for auto-generating document templates for testing.""" - LAYERING_DEFINITION = { + LAYERING_POLICY_TEMPLATE = { "data": { "layerOrder": [] }, "metadata": { "name": "placeholder", - "schema": "metadata/Control/v%s" % DeckhandFactory.API_VERSION + "schema": "metadata/Control/v%s" % DeckhandFactory.API_VERSION, + "layeringDefinition": { + "abstract": False, + "layer": "" + } }, "schema": "deckhand/LayeringPolicy/v%s" % DeckhandFactory.API_VERSION } - LAYER_TEMPLATE = { + DOCUMENT_TEMPLATE = { "data": {}, "metadata": { "labels": {"": ""}, @@ -159,10 +159,12 @@ class DocumentFactory(DeckhandFactory): layer_order = ["global", "region", "site"] else: raise ValueError("'num_layers' must be a value between 1 - 3.") - self.layering_definition = copy.deepcopy(self.LAYERING_DEFINITION) - self.layering_definition['metadata']['name'] = test_utils.rand_name( + self.layering_policy = copy.deepcopy(self.LAYERING_POLICY_TEMPLATE) + self.layering_policy['metadata']['name'] = test_utils.rand_name( 'layering-policy') - self.layering_definition['data']['layerOrder'] = layer_order + self.layering_policy['data']['layerOrder'] = layer_order + self.layering_policy['metadata']['layeringDefinition'][ + 'layer'] = layer_order[0] if not isinstance(docs_per_layer, (list, tuple)): raise TypeError("'docs_per_layer' must be a list or tuple " @@ -179,9 +181,6 @@ class DocumentFactory(DeckhandFactory): self.num_layers = num_layers self.docs_per_layer = docs_per_layer - def gen(self): - raise NotImplementedError() - def gen_test(self, mapping, site_abstract=True, region_abstract=True, global_abstract=True, site_parent_selectors=None): """Generate the document template. @@ -228,12 +227,12 @@ class DocumentFactory(DeckhandFactory): :type site_parent_selectors: list :returns: Rendered template of the form specified above. """ - rendered_template = [self.layering_definition] + rendered_template = [self.layering_policy] layer_order = rendered_template[0]['data']['layerOrder'] for layer_idx in range(self.num_layers): for count in range(self.docs_per_layer[layer_idx]): - layer_template = copy.deepcopy(self.LAYER_TEMPLATE) + layer_template = copy.deepcopy(self.DOCUMENT_TEMPLATE) layer_name = layer_order[layer_idx] layer_template = copy.deepcopy(layer_template) @@ -332,7 +331,11 @@ class DocumentSecretFactory(DeckhandFactory): "metadata": { "schema": "metadata/Document/v1", "name": "", - "storagePolicy": "" + "layeringDefinition": { + "abstract": False, + "layer": "site" + }, + "storagePolicy": "", }, "schema": "deckhand/%s/v1" } @@ -358,9 +361,6 @@ class DocumentSecretFactory(DeckhandFactory): ... """ - def gen(self): - raise NotImplementedError() - def gen_test(self, schema, storage_policy, data=None, name=None): if data is None: data = test_utils.rand_password() diff --git a/deckhand/tests/unit/control/test_validations_controller.py b/deckhand/tests/unit/control/test_validations_controller.py index 355cdeb5..78ef7ac7 100644 --- a/deckhand/tests/unit/control/test_validations_controller.py +++ b/deckhand/tests/unit/control/test_validations_controller.py @@ -13,6 +13,7 @@ # limitations under the License. import copy +import os import yaml import mock @@ -20,7 +21,6 @@ from oslo_config import cfg from deckhand.control import buckets from deckhand.engine import document_validation -from deckhand.engine.schema.v1_0 import document_schema from deckhand import factories from deckhand.tests import test_utils from deckhand.tests.unit.control import base as test_base @@ -93,6 +93,11 @@ class TestValidationsControllerPostValidate(ValidationsControllerBaseTest): def setUp(self): super(TestValidationsControllerPostValidate, self).setUp() + dataschema_schema = os.path.join( + os.getcwd(), 'deckhand', 'engine', 'schemas', + 'dataschema_schema.yaml') + with open(dataschema_schema, 'r') as f: + self.dataschema_schema = yaml.safe_load(f.read()) self._monkey_patch_document_validation() def _monkey_patch_document_validation(self): @@ -556,25 +561,24 @@ class TestValidationsControllerPostValidate(ValidationsControllerBaseTest): 'properties': { 'a': { 'type': 'integer' # Test doc will fail b/c of wrong type. + }, + 'b': { + 'type': 'string' } }, - 'required': ['a'] + 'required': ['a', 'b'] } data_schema = data_schema_factory.gen_test( metadata_name, data=schema_to_use) - # Failure #1. - # Create the test document that fails the validation due to the - # schema defined by the `DataSchema` document. + # Failure #1: Provide wrong type for property "a". + # Failure #2: Don't include required property "b". doc_factory = factories.DocumentFactory(1, [1]) doc_to_test = doc_factory.gen_test( {'_GLOBAL_DATA_1_': {'data': {'a': 'fail'}}}, global_abstract=False)[-1] doc_to_test['schema'] = 'example/foo/v1' doc_to_test['metadata']['name'] = 'test_doc' - # Failure #2. - # Remove required metadata property, causing error to be generated. - del doc_to_test['metadata']['layeringDefinition'] revision_id = self._create_revision(payload=[doc_to_test, data_schema]) @@ -597,16 +601,19 @@ class TestValidationsControllerPostValidate(ValidationsControllerBaseTest): 'error_section': { 'data': {'a': 'fail'}, 'metadata': {'labels': {'global': 'global1'}, - 'name': 'test_doc', - 'schema': 'metadata/Document/v1.0'}, - 'schema': 'example/foo/v1' + 'layeringDefinition': {'abstract': False, + 'actions': [], + 'layer': 'global'}, + 'name': doc_to_test['metadata']['name'], + 'schema': doc_to_test['metadata']['schema']}, + 'schema': doc_to_test['schema'] }, 'name': 'test_doc', - 'path': '.metadata', + 'path': '.data', 'schema': 'example/foo/v1', - 'message': "'layeringDefinition' is a required property", - 'validation_schema': document_schema.schema, - 'schema_path': '.properties.metadata.required' + 'message': "'b' is a required property", + 'validation_schema': schema_to_use, + 'schema_path': '.required' }, { 'error_section': {'a': 'fail'}, 'name': 'test_doc', @@ -624,7 +631,8 @@ class TestValidationsControllerPostValidate(ValidationsControllerBaseTest): body = yaml.safe_load(resp.text) self.assertEqual('failure', body['status']) - self.assertEqual(expected_errors, body['errors']) + self.assertEqual(sorted(expected_errors, key=lambda x: x['path']), + sorted(body['errors'], key=lambda x: x['path'])) def test_validation_with_registered_data_schema_expect_mixed(self): rules = {'deckhand:create_cleartext_documents': '@', @@ -721,15 +729,12 @@ class TestValidationsControllerPostValidate(ValidationsControllerBaseTest): self.assertIn('errors', body) self.assertEqual(expected_errors, body['errors']) - def test_document_without_data_section_saves_but_fails_validation(self): - """Validate that a document without the data section is saved to the - database, but fails validation. This is a valid use case because a - document in a bucket can be created without a data section, which - depends on substitution from another document. + def test_document_without_data_section_ingested(self): + """Validate that a document without the data section is ingested + successfully. """ rules = {'deckhand:create_cleartext_documents': '@', - 'deckhand:list_validations': '@', - 'deckhand:show_validation': '@'} + 'deckhand:list_validations': '@'} self.policy.set_rules(rules) documents_factory = factories.DocumentFactory(1, [1]) @@ -751,41 +756,11 @@ class TestValidationsControllerPostValidate(ValidationsControllerBaseTest): body = yaml.safe_load(resp.text) expected_body = { 'count': 2, - 'results': [{'id': 0, 'status': 'failure'}, # Document. + 'results': [{'id': 0, 'status': 'success'}, # Document. {'id': 1, 'status': 'success'}] # DataSchema. } self.assertEqual(expected_body, body) - # Validate that the created document failed validation for the expected - # reason. - resp = self.app.simulate_get( - '/api/v1.0/revisions/%s/validations/%s/entries/0' % ( - revision_id, types.DECKHAND_SCHEMA_VALIDATION), - headers={'Content-Type': 'application/x-yaml'}) - self.assertEqual(200, resp.status_code) - body = yaml.safe_load(resp.text) - expected_errors = [{ - 'error_section': { - 'data': None, - 'metadata': {'labels': {'global': 'global1'}, - 'layeringDefinition': {'abstract': False, - 'actions': [], - 'layer': 'global'}, - 'name': document['metadata']['name'], - 'schema': 'metadata/Document/v1.0'}, - 'schema': document['schema'] - }, - 'name': document['metadata']['name'], - 'path': '.data', - 'schema': document['schema'], - 'message': ( - "None is not of type 'string', 'integer', 'array', 'object'"), - 'validation_schema': document_schema.schema, - 'schema_path': '.properties.data.type' - }] - self.assertIn('errors', body) - self.assertEqual(expected_errors, body['errors']) - def test_validation_only_new_data_schema_registered(self): """Validate whether newly created DataSchemas replace old DataSchemas when it comes to validation. @@ -859,7 +834,7 @@ class TestValidationsControllerPreValidate(ValidationsControllerBaseTest): Validations controller. """ - def test_pre_validate_flag_skips_over_dataschema_validations(self): + def test_pre_validate_flag_skips_registered_dataschema_validations(self): rules = {'deckhand:create_cleartext_documents': '@', 'deckhand:list_validations': '@'} self.policy.set_rules(rules) diff --git a/deckhand/tests/unit/engine/test_document_layering_negative.py b/deckhand/tests/unit/engine/test_document_layering_negative.py index 099bed2f..aaa17026 100644 --- a/deckhand/tests/unit/engine/test_document_layering_negative.py +++ b/deckhand/tests/unit/engine/test_document_layering_negative.py @@ -71,6 +71,7 @@ class TestDocumentLayeringNegative( def test_layering_with_empty_layer(self, mock_log): doc_factory = factories.DocumentFactory(1, [1]) documents = doc_factory.gen_test({}, global_abstract=False) + del documents[0]['metadata']['layeringDefinition'] # Only pass in the LayeringPolicy. self._test_layering([documents[0]], global_expected=None) @@ -243,3 +244,29 @@ class TestDocumentLayeringValidationNegative( self.assertRaises(errors.InvalidDocumentFormat, self._test_layering, [layering_policy, document], validate=True) + + def test_layering_invalid_document_format_generates_error_messages(self): + doc_factory = factories.DocumentFactory(1, [1]) + lp_template, document = doc_factory.gen_test({ + "_GLOBAL_SUBSTITUTIONS_1_": [{ + "dest": { + "path": ".c" + }, + "src": { + "schema": "deckhand/Certificate/v1", + "name": "global-cert", + "path": "." + } + + }], + }, global_abstract=False) + + layering_policy = copy.deepcopy(lp_template) + del layering_policy['data']['layerOrder'] + error_re = ("The provided document \[%s\] %s failed schema validation." + " Errors: 'layerOrder' is a required property" % ( + layering_policy['schema'], + layering_policy['metadata']['name'])) + self.assertRaisesRegexp( + errors.InvalidDocumentFormat, error_re, self._test_layering, + [layering_policy, document], validate=True) diff --git a/deckhand/tests/unit/engine/test_document_validation.py b/deckhand/tests/unit/engine/test_document_validation.py index 4a38d113..575410d8 100644 --- a/deckhand/tests/unit/engine/test_document_validation.py +++ b/deckhand/tests/unit/engine/test_document_validation.py @@ -15,9 +15,8 @@ import mock from deckhand.engine import document_validation -from deckhand.tests.unit.engine import base as engine_test_base - from deckhand import factories +from deckhand.tests.unit.engine import base as engine_test_base from deckhand import utils @@ -59,7 +58,7 @@ class TestDocumentValidation(engine_test_base.TestDocumentValidationBase): abstract_document = utils.jsonpath_replace( test_document, True, '.metadata.layeringDefinition.abstract') document_validation.DocumentValidation( - abstract_document).validate_all() + abstract_document, pre_validate=False).validate_all() self.assertTrue(mock_log.info.called) self.assertIn("Skipping schema validation for abstract document", mock_log.info.mock_calls[0][1][0]) @@ -81,8 +80,12 @@ class TestDocumentValidation(engine_test_base.TestDocumentValidationBase): 'scary-secret', utils.jsonpath_parse(test_document['data'], sub['dest']['path'])) + data_schema_factory = factories.DataSchemaFactory() + data_schema = data_schema_factory.gen_test(test_document['schema'], {}) + validations = document_validation.DocumentValidation( - test_document).validate_all() + test_document, existing_data_schemas=[data_schema], + pre_validate=False).validate_all() self.assertEqual(1, len(validations[0]['errors'])) self.assertIn('Sanitized to avoid exposing secret.', diff --git a/deckhand/tests/unit/engine/test_document_validation_negative.py b/deckhand/tests/unit/engine/test_document_validation_negative.py index d8fa2514..9c927dc3 100644 --- a/deckhand/tests/unit/engine/test_document_validation_negative.py +++ b/deckhand/tests/unit/engine/test_document_validation_negative.py @@ -16,7 +16,6 @@ import mock from deckhand.engine import document_validation from deckhand import errors -from deckhand import factories from deckhand.tests.unit.engine import base as test_base from deckhand import types @@ -24,20 +23,32 @@ from deckhand import types class TestDocumentValidationNegative(test_base.TestDocumentValidationBase): """Negative testing suite for document validation.""" - # The 'data' key is mandatory but not critical if excluded. - exception_map = { - 'metadata': errors.InvalidDocumentFormat, - 'metadata.schema': errors.InvalidDocumentFormat, - 'metadata.name': errors.InvalidDocumentFormat, - 'schema': errors.InvalidDocumentFormat, - } + BASIC_PROPERTIES = ( + 'metadata', + 'metadata.schema', + 'metadata.name', + 'metadata.layeringDefinition', + 'metadata.layeringDefinition.layer', + 'schema' + ) + + CRITICAL_PROPERTIES = ( + 'schema', + 'metadata', + 'metadata.schema', + 'metadata.name', + 'metadata.substitutions.0.dest', + 'metadata.substitutions.0.dest.path', + 'metadata.substitutions.0.src', + 'metadata.substitutions.0.src.schema', + 'metadata.substitutions.0.src.name', + 'metadata.substitutions.0.src.path' + ) def _do_validations(self, document_validator, expected, expected_err): validations = document_validator.validate_all() - self.assertEqual(2, len(validations)) - # The DataSchema document itself should've validated - # successfully. - self.assertEqual('success', validations[0]['status']) + + self.assertEqual(1, len(validations)) self.assertEqual('failure', validations[-1]['status']) self.assertEqual({'version': '1.0', 'name': 'deckhand'}, validations[-1]['validator']) @@ -61,62 +72,65 @@ class TestDocumentValidationNegative(test_base.TestDocumentValidationBase): missing_prop = property_to_remove.split('.')[-1] invalid_data = self._corrupt_data(document, property_to_remove) - exception_raised = self.exception_map.get(property_to_remove, None) + exception_raised = property_to_remove in self.CRITICAL_PROPERTIES expected_err_msg = "'%s' is a required property" % missing_prop - dataschema_factory = factories.DataSchemaFactory() - dataschema = dataschema_factory.gen_test( - invalid_data.get('schema', ''), {}) - payload = [dataschema, invalid_data] - - doc_validator = document_validation.DocumentValidation(payload) + payload = [invalid_data] + doc_validator = document_validation.DocumentValidation( + payload, pre_validate=False) if exception_raised: self.assertRaises( - exception_raised, doc_validator.validate_all) + errors.InvalidDocumentFormat, doc_validator.validate_all) else: self._do_validations(doc_validator, invalid_data, expected_err_msg) + def test_certificate_authority_key_missing_required_sections(self): + document = self._read_data('sample_certificate_authority_key') + properties_to_remove = tuple(self.BASIC_PROPERTIES) + ( + 'metadata.storagePolicy',) + self._test_missing_required_sections(document, properties_to_remove) + + def test_certificate_authority_missing_required_sections(self): + document = self._read_data('sample_certificate_authority') + properties_to_remove = tuple(self.BASIC_PROPERTIES) + ( + 'metadata.storagePolicy',) + self._test_missing_required_sections(document, properties_to_remove) + def test_certificate_key_missing_required_sections(self): document = self._read_data('sample_certificate_key') - properties_to_remove = tuple(self.exception_map.keys()) + ( + properties_to_remove = tuple(self.BASIC_PROPERTIES) + ( 'metadata.storagePolicy',) self._test_missing_required_sections(document, properties_to_remove) def test_certificate_missing_required_sections(self): document = self._read_data('sample_certificate') - properties_to_remove = tuple(self.exception_map.keys()) + ( + properties_to_remove = tuple(self.BASIC_PROPERTIES) + ( 'metadata.storagePolicy',) self._test_missing_required_sections(document, properties_to_remove) def test_data_schema_missing_required_sections(self): + properties_to_remove = ( + 'metadata', + 'metadata.schema', + 'metadata.name', + 'schema', + 'data.$schema' + ) document = self._read_data('sample_data_schema') - properties_to_remove = tuple(self.exception_map.keys()) + ( - 'data.$schema',) self._test_missing_required_sections(document, properties_to_remove) - def test_document_missing_required_sections(self): + def test_generic_document_missing_required_sections(self): document = self._read_data('sample_document') - properties_to_remove = tuple(self.exception_map.keys()) + ( - 'metadata.layeringDefinition', - 'metadata.layeringDefinition.layer', - 'metadata.layeringDefinition.actions.0.method', - 'metadata.layeringDefinition.actions.0.path', - 'metadata.substitutions.0.dest', - 'metadata.substitutions.0.dest.path', - 'metadata.substitutions.0.src', - 'metadata.substitutions.0.src.schema', - 'metadata.substitutions.0.src.name', - 'metadata.substitutions.0.src.path') + properties_to_remove = self.CRITICAL_PROPERTIES self._test_missing_required_sections(document, properties_to_remove) - def test_document_missing_multiple_required_sections(self): + def test_generic_document_missing_multiple_required_sections(self): """Validates that multiple errors are reported for a document with multiple validation errors. """ document = self._read_data('sample_document') properties_to_remove = ( - 'metadata.layeringDefinition.layer', 'metadata.layeringDefinition.actions.0.method', 'metadata.layeringDefinition.actions.0.path', 'metadata.substitutions.0.dest.path', @@ -128,55 +142,61 @@ class TestDocumentValidationNegative(test_base.TestDocumentValidationBase): document = self._corrupt_data(document, property_to_remove) doc_validator = document_validation.DocumentValidation(document) - validations = doc_validator.validate_all() + e = self.assertRaises(errors.InvalidDocumentFormat, + doc_validator.validate_all) - errors = validations[0]['errors'] - self.assertEqual(len(properties_to_remove), len(errors)) - - # Sort the errors to match the order in ``properties_to_remove``. - errors = sorted(errors, key=lambda x: (x['path'], x['message'])) - - # Validate that an error was generated for each missing property in - # ``properties_to_remove`` that was removed from ``document``. for idx, property_to_remove in enumerate(properties_to_remove): parts = property_to_remove.split('.') - parent_path = '.' + '.'.join(parts[:-1]) missing_property = parts[-1] expected_err = "'%s' is a required property" % missing_property - self.assertEqual(expected_err, errors[idx]['message']) - self.assertEqual(parent_path, errors[idx]['path']) + self.assertIn(expected_err, e.message) def test_document_invalid_layering_definition_action(self): document = self._read_data('sample_document') missing_data = self._corrupt_data( document, 'metadata.layeringDefinition.actions.0.method', 'invalid', op='replace') - expected_err = "'invalid' is not one of ['replace', 'delete', 'merge']" + expected_err = ( + r".+ 'invalid' is not one of \['replace', 'delete', 'merge'\]") - # Ensure that a dataschema document exists for the random document - # schema via mocking. - dataschema_factory = factories.DataSchemaFactory() - dataschema = dataschema_factory.gen_test(document['schema'], {}) - payload = [dataschema, missing_data] + payload = [missing_data] doc_validator = document_validation.DocumentValidation(payload) - self._do_validations(doc_validator, document, expected_err) + self.assertRaisesRegexp(errors.InvalidDocumentFormat, expected_err, + doc_validator.validate_all) def test_layering_policy_missing_required_sections(self): + properties_to_remove = ( + 'metadata', + 'metadata.schema', + 'metadata.name', + 'schema', + 'data.layerOrder' + ) document = self._read_data('sample_layering_policy') - properties_to_remove = tuple(self.exception_map.keys()) + ( - 'data.layerOrder',) self._test_missing_required_sections(document, properties_to_remove) def test_passphrase_missing_required_sections(self): document = self._read_data('sample_passphrase') - properties_to_remove = tuple(self.exception_map.keys()) + ( + properties_to_remove = tuple(self.BASIC_PROPERTIES) + ( + 'metadata.storagePolicy',) + self._test_missing_required_sections(document, properties_to_remove) + + def test_privatekey_missing_required_sections(self): + document = self._read_data('sample_private_key') + properties_to_remove = tuple(self.BASIC_PROPERTIES) + ( + 'metadata.storagePolicy',) + self._test_missing_required_sections(document, properties_to_remove) + + def test_publickey_missing_required_sections(self): + document = self._read_data('sample_public_key') + properties_to_remove = tuple(self.BASIC_PROPERTIES) + ( 'metadata.storagePolicy',) self._test_missing_required_sections(document, properties_to_remove) def test_validation_policy_missing_required_sections(self): document = self._read_data('sample_validation_policy') - properties_to_remove = tuple(self.exception_map.keys()) + ( + properties_to_remove = tuple(self.BASIC_PROPERTIES) + ( 'data.validations', 'data.validations.0.name') self._test_missing_required_sections(document, properties_to_remove) @@ -206,25 +226,12 @@ class TestDocumentValidationNegative(test_base.TestDocumentValidationBase): def test_invalid_validation_schema_raises_runtime_error(self): document = self._read_data('sample_passphrase') - fake_schema = mock.MagicMock(schema='fake') - fake_schema_map = {'v1': {'deckhand/Passphrase': fake_schema}} # Validate that broken built-in base schema raises RuntimeError. - with mock.patch.object(document_validation, 'base_schema', - new_callable=mock.PropertyMock( - return_value=fake_schema)): - doc_validator = document_validation.DocumentValidation(document) - with self.assertRaisesRegexp(RuntimeError, 'Unknown error'): - doc_validator.validate_all() - - # Validate that broken built-in schema for ``SchemaValidator`` raises - # RuntimeError. - with mock.patch.object(document_validation.SchemaValidator, - '_schema_map', new_callable=mock.PropertyMock( - return_value=fake_schema_map)): - doc_validator = document_validation.DocumentValidation(document) - with self.assertRaisesRegexp(RuntimeError, 'Unknown error'): - doc_validator.validate_all() + doc_validator = document_validation.DocumentValidation(document) + doc_validator._validators[0].base_schema = 'fake' + with self.assertRaisesRegexp(RuntimeError, 'Unknown error'): + doc_validator.validate_all() # Validate that broken data schema for ``DataSchemaValidator`` raises # RuntimeError. @@ -233,6 +240,6 @@ class TestDocumentValidationNegative(test_base.TestDocumentValidationBase): data_schema['metadata']['name'] = document['schema'] data_schema['data'] = 'fake' doc_validator = document_validation.DocumentValidation( - [document, data_schema]) + [document, data_schema], pre_validate=False) with self.assertRaisesRegexp(RuntimeError, 'Unknown error'): doc_validator.validate_all() diff --git a/deckhand/tests/unit/resources/sample_certificate.yaml b/deckhand/tests/unit/resources/sample_certificate.yaml index fdd18b11..7efb9212 100644 --- a/deckhand/tests/unit/resources/sample_certificate.yaml +++ b/deckhand/tests/unit/resources/sample_certificate.yaml @@ -4,6 +4,9 @@ metadata: schema: metadata/Document/v1.0 name: application-api storagePolicy: cleartext + layeringDefinition: + abstract: False + layer: site data: |- -----BEGIN CERTIFICATE----- MIIDYDCCAkigAwIBAgIUKG41PW4VtiphzASAMY4/3hL8OtAwDQYJKoZIhvcNAQEL diff --git a/deckhand/tests/unit/resources/sample_certificate_authority.yaml b/deckhand/tests/unit/resources/sample_certificate_authority.yaml new file mode 100644 index 00000000..324d8525 --- /dev/null +++ b/deckhand/tests/unit/resources/sample_certificate_authority.yaml @@ -0,0 +1,16 @@ +--- +schema: deckhand/CertificateAuthority/v1.0 +metadata: + schema: metadata/Document/v1.0 + name: application-api + storagePolicy: cleartext + layeringDefinition: + abstract: False + layer: site +data: |- + -----BEGIN CERTIFICATE----- + MIIDYDCCAkigAwIBAgIUKG41PW4VtiphzASAMY4/3hL8OtAwDQYJKoZIhvcNAQEL + ...snip... + P3WT9CfFARnsw2nKjnglQcwKkKLYip0WY2wh3FE7nrQZP6xKNaSRlh6p2pCGwwwH + HkvVwA== + -----END CERTIFICATE----- diff --git a/deckhand/tests/unit/resources/sample_certificate_authority_key.yaml b/deckhand/tests/unit/resources/sample_certificate_authority_key.yaml new file mode 100644 index 00000000..2b4402b8 --- /dev/null +++ b/deckhand/tests/unit/resources/sample_certificate_authority_key.yaml @@ -0,0 +1,16 @@ +--- +schema: deckhand/CertificateAuthorityKey/v1.0 +metadata: + schema: metadata/Document/v1.0 + name: application-api + storagePolicy: cleartext + layeringDefinition: + abstract: False + layer: site +data: |- + -----BEGIN CERTIFICATE----- + MIIDYDCCAkigAwIBAgIUKG41PW4VtiphzASAMY4/3hL8OtAwDQYJKoZIhvcNAQEL + ...snip... + P3WT9CfFARnsw2nKjnglQcwKkKLYip0WY2wh3FE7nrQZP6xKNaSRlh6p2pCGwwwH + HkvVwA== + -----END CERTIFICATE----- diff --git a/deckhand/tests/unit/resources/sample_certificate_key.yaml b/deckhand/tests/unit/resources/sample_certificate_key.yaml index 39ffc9c1..ec4514b1 100644 --- a/deckhand/tests/unit/resources/sample_certificate_key.yaml +++ b/deckhand/tests/unit/resources/sample_certificate_key.yaml @@ -4,6 +4,9 @@ metadata: schema: metadata/Document/v1.0 name: application-api storagePolicy: encrypted + layeringDefinition: + abstract: False + layer: site data: |- -----BEGIN RSA PRIVATE KEY----- MIIEpQIBAAKCAQEAx+m1+ao7uTVEs+I/Sie9YsXL0B9mOXFlzEdHX8P8x4nx78/T diff --git a/deckhand/tests/unit/resources/sample_data_schema.yaml b/deckhand/tests/unit/resources/sample_data_schema.yaml index d07804a5..a202e9b1 100644 --- a/deckhand/tests/unit/resources/sample_data_schema.yaml +++ b/deckhand/tests/unit/resources/sample_data_schema.yaml @@ -5,5 +5,8 @@ metadata: name: promenade/Node/v1.0 # Specifies the documents to be used for validation. labels: application: promenade + layeringDefinition: + abstract: False + layer: site data: # Valid JSON Schema is expected here. $schema: http://blah diff --git a/deckhand/tests/unit/resources/sample_layering_policy.yaml b/deckhand/tests/unit/resources/sample_layering_policy.yaml index 5a1391bc..e27f7369 100644 --- a/deckhand/tests/unit/resources/sample_layering_policy.yaml +++ b/deckhand/tests/unit/resources/sample_layering_policy.yaml @@ -4,6 +4,9 @@ schema: deckhand/LayeringPolicy/v1.0 metadata: schema: metadata/Control/v1.0 name: a-unique-config-name-12345 + layeringDefinition: + abstract: False + layer: site data: layerOrder: - global diff --git a/deckhand/tests/unit/resources/sample_passphrase.yaml b/deckhand/tests/unit/resources/sample_passphrase.yaml index 21caeb6b..c7c8442b 100644 --- a/deckhand/tests/unit/resources/sample_passphrase.yaml +++ b/deckhand/tests/unit/resources/sample_passphrase.yaml @@ -6,4 +6,5 @@ metadata: storagePolicy: encrypted layeringDefinition: abstract: False + layer: site data: some-password diff --git a/deckhand/tests/unit/resources/sample_private_key.yaml b/deckhand/tests/unit/resources/sample_private_key.yaml new file mode 100644 index 00000000..41095783 --- /dev/null +++ b/deckhand/tests/unit/resources/sample_private_key.yaml @@ -0,0 +1,10 @@ +--- +schema: deckhand/PrivateKey/v1.0 +metadata: + schema: metadata/Document/v1.0 + name: application-private-password + storagePolicy: encrypted + layeringDefinition: + abstract: False + layer: site +data: some-private-key diff --git a/deckhand/tests/unit/resources/sample_public_key.yaml b/deckhand/tests/unit/resources/sample_public_key.yaml new file mode 100644 index 00000000..6fc93b71 --- /dev/null +++ b/deckhand/tests/unit/resources/sample_public_key.yaml @@ -0,0 +1,10 @@ +--- +schema: deckhand/PublicKey/v1.0 +metadata: + schema: metadata/Document/v1.0 + name: application-public-password + storagePolicy: encrypted + layeringDefinition: + abstract: False + layer: site +data: some-public-key diff --git a/deckhand/tests/unit/resources/sample_validation_policy.yaml b/deckhand/tests/unit/resources/sample_validation_policy.yaml index 4c8353d8..9fb97341 100644 --- a/deckhand/tests/unit/resources/sample_validation_policy.yaml +++ b/deckhand/tests/unit/resources/sample_validation_policy.yaml @@ -4,6 +4,9 @@ schema: deckhand/ValidationPolicy/v1.0 metadata: schema: metadata/Control/v1.0 name: later-validation + layeringDefinition: + abstract: False + layer: site data: validations: - name: deckhand-schema-validation diff --git a/doc/source/document_types.rst b/doc/source/document_types.rst index 0bc36428..64f22478 100644 --- a/doc/source/document_types.rst +++ b/doc/source/document_types.rst @@ -33,9 +33,9 @@ DataSchema that Deckhand can use for validation. No ``DataSchema`` documents with names beginning with ``deckhand/`` or ``metadata/`` are allowed. The ``metadata.name`` field of each ``DataSchema`` document specifies the top level ``schema`` that it -is used to validate. +is used to validate against. -The contents of its ``data`` key are expected to be the json schema definition +The contents of its ``data`` key are expected to be the JSON schema definition for the target document type from the target's top level ``data`` key down. .. TODO: give valid, tiny schema as example diff --git a/doc/source/validation.rst b/doc/source/validation.rst index 72639781..a3ab56b9 100644 --- a/doc/source/validation.rst +++ b/doc/source/validation.rst @@ -106,6 +106,25 @@ a specific action. and succeed, then services can check whether the documents in a bucket are stable, in accordance with the ``ValidationPolicy``. +Validation Stages +----------------- + +Deckhand performs pre- and post-rendering validation on documents. For +pre-rendering validation 3 types of behavior are possible: + +#. Successfully validated docuemnts are stored in Deckhand's database. +#. Failure to validate a document's basic properties will result in a 400 + Bad Request error getting raised. +#. Failure to validate a document's schema-specific properties will result + in a validation error created in the database, which can be later + returned via the Validations API. + +For post-rendering validation, 2 types of behavior are possible: + +#. Successfully valdiated post-rendered documents are returned to the user. +#. Failure to validate post-rendered documents results in a 500 Internal Server + Error getting raised. + Validation Module ----------------- @@ -120,35 +139,162 @@ Validation Schemas Below are the schemas Deckhand uses to validate documents. -.. automodule:: deckhand.engine.schema.base_schema - :members: schema +Base Schema +----------- -.. automodule:: deckhand.engine.schema.v1_0.certificate_authority_key_schema - :members: schema +* Base schema. -.. automodule:: deckhand.engine.schema.v1_0.certificate_authority_schema - :members: schema + Base JSON schema against which all Deckhand documents are validated. -.. automodule:: deckhand.engine.schema.v1_0.certificate_key_schema - :members: schema + .. literalinclude:: ../../deckhand/engine/schemas/base_schema.yaml + :language: yaml + :lines: 15- + :caption: Base schema that applies to all documents. -.. automodule:: deckhand.engine.schema.v1_0.certificate_schema - :members: schema + This schema is used to sanity-check all documents that are passed to + Deckhand. Failure to pass this schema results in a critical error. -.. automodule:: deckhand.engine.schema.v1_0.data_schema_schema - :members: schema +DataSchema Schemas +------------------ -.. automodule:: deckhand.engine.schema.v1_0.layering_policy_schema - :members: schema +All schemas below are ``DataSchema`` documents. They define additional +properties not included in the base schema or override default properties in +the base schema. -.. automodule:: deckhand.engine.schema.v1_0.passphrase_schema - :members: schema +These schemas are only enforced after validation for the base schema has +passed. Failure to pass these schemas will result in an error entry being +created for the validation with name ``deckhand-schema-validation`` +corresponding to the created revision. -.. automodule:: deckhand.engine.schema.v1_0.private_key_schema - :members: schema +* ``CertificateAuthorityKey`` schema. -.. automodule:: deckhand.engine.schema.v1_0.public_key_schema - :members: schema + JSON schema against which all documents with + ``deckhand/CertificateAuthorityKey/v1`` schema are validated. -.. automodule:: deckhand.engine.schema.v1_0.validation_policy_schema - :members: schema + .. literalinclude:: + ../../deckhand/engine/schemas/certificate_authority_key_schema.yaml + :language: yaml + :lines: 15- + :caption: Schema for ``CertificateAuthorityKey`` documents. + + This schema is used to sanity-check all CertificateAuthorityKey documents + that are passed to Deckhand. + +* ``CertificateAuthority`` schema. + + JSON schema against which all documents with + ``deckhand/CertificateAuthority/v1`` schema are validated. + + .. literalinclude:: + ../../deckhand/engine/schemas/certificate_authority_schema.yaml + :language: yaml + :lines: 15- + :caption: Schema for ``CertificateAuthority`` documents. + + This schema is used to sanity-check all ``CertificateAuthority`` documents + that are passed to Deckhand. + +* ``CertificateKey`` schema. + + JSON schema against which all documents with ``deckhand/CertificateKey/v1`` + schema are validated. + + .. literalinclude:: ../../deckhand/engine/schemas/certificate_key_schema.yaml + :language: yaml + :lines: 15- + :caption: Schema for ``CertificateKey`` documents. + + This schema is used to sanity-check all ``CertificateKey`` documents that are + passed to Deckhand. + +* ``Certificate`` schema. + + JSON schema against which all documents with ``deckhand/Certificate/v1`` + schema are validated. + + .. literalinclude:: ../../deckhand/engine/schemas/certificate_schema.yaml + :language: yaml + :lines: 15- + :caption: Schema for ``Certificate`` documents. + + This schema is used to sanity-check all ``Certificate`` documents that are + passed to Deckhand. + +* ``DataSchema`` schema. + + JSON schema against which all documents with ``deckhand/DataSchema/v1`` + schema are validated. + + .. literalinclude:: ../../deckhand/engine/schemas/dataschema_schema.yaml + :language: yaml + :lines: 15- + :caption: Schema for ``DataSchema`` documents. + + This schema is used to sanity-check all ``DataSchema`` documents that are + passed to Deckhand. + +* ``LayeringPolicy`` schema. + + JSON schema against which all documents with ``deckhand/LayeringPolicy/v1`` + schema are validated. + + .. literalinclude:: ../../deckhand/engine/schemas/layering_policy_schema.yaml + :language: yaml + :lines: 15- + :caption: Schema for ``LayeringPolicy`` documents. + + This schema is used to sanity-check all ``LayeringPolicy`` documents that are + passed to Deckhand. + +* ``PrivateKey`` schema. + + JSON schema against which all documents with ``deckhand/PrivateKey/v1`` + schema are validated. + + .. literalinclude:: ../../deckhand/engine/schemas/passphrase_schema.yaml + :language: yaml + :lines: 15- + :caption: Schema for ``PrivateKey`` documents. + + This schema is used to sanity-check all ``PrivateKey`` documents that are + passed to Deckhand. + +* ``PublicKey`` schema. + + JSON schema against which all documents with ``deckhand/PublicKey/v1`` + schema are validated. + + .. literalinclude:: ../../deckhand/engine/schemas/public_key_schema.yaml + :language: yaml + :lines: 15- + :caption: Schema for ``PublicKey`` documents. + + This schema is used to sanity-check all ``PublicKey`` documents that are + passed to Deckhand. + +* ``Passphrase`` schema. + + JSON schema against which all documents with ``deckhand/Passphrase/v1`` + schema are validated. + + .. literalinclude:: ../../deckhand/engine/schemas/private_key_schema.yaml + :language: yaml + :lines: 15- + :caption: Schema for ``Passphrase`` documents. + + This schema is used to sanity-check all ``Passphrase`` documents that are + passed to Deckhand. + +* ``ValidationPolicy`` schema. + + JSON schema against which all documents with ``deckhand/ValidationPolicy/v1`` + schema are validated. + + .. literalinclude:: + ../../deckhand/engine/schemas/validation_policy_schema.yaml + :language: yaml + :lines: 15- + :caption: Schema for ``ValidationPolicy`` documents. + + This schema is used to sanity-check all ``ValidationPolicy`` documents that + are passed to Deckhand.