diff --git a/deckhand/common/document.py b/deckhand/common/document.py index 54009272..eb3d5546 100644 --- a/deckhand/common/document.py +++ b/deckhand/common/document.py @@ -71,7 +71,7 @@ class DocumentDict(dict): @property def is_control(self): - return self.metadata.get('schema', '').startswith('deckhand/Control') + return self.metadata.get('schema', '').startswith('metadata/Control') @property def schema(self): diff --git a/deckhand/control/revision_documents.py b/deckhand/control/revision_documents.py index 9dc10678..a7293b1f 100644 --- a/deckhand/control/revision_documents.py +++ b/deckhand/control/revision_documents.py @@ -164,8 +164,8 @@ class RenderedDocumentsResource(api_base.BaseResource): rendered_documents = rendered_documents[:limit] resp.status = falcon.HTTP_200 - resp.body = self.view_builder.list(rendered_documents) self._post_validate(rendered_documents) + resp.body = self.view_builder.list(rendered_documents) def _retrieve_documents_for_rendering(self, revision_id, **filters): """Retrieve all necessary documents needed for rendering. If a layering diff --git a/deckhand/db/sqlalchemy/models.py b/deckhand/db/sqlalchemy/models.py index 676c5ade..e1d6a6e4 100644 --- a/deckhand/db/sqlalchemy/models.py +++ b/deckhand/db/sqlalchemy/models.py @@ -38,6 +38,8 @@ LOG = logging.getLogger(__name__) # relative to that base. BASE = None +# TODO(felipemonteiro): Make most (all?) of these tables immutable. + class DeckhandBase(models.ModelBase, models.TimestampMixin): """Base class for Deckhand Models.""" diff --git a/deckhand/engine/document_validation.py b/deckhand/engine/document_validation.py index 68f8a978..63cfbe6f 100644 --- a/deckhand/engine/document_validation.py +++ b/deckhand/engine/document_validation.py @@ -88,14 +88,6 @@ class BaseValidator(object): global _DEFAULT_SCHEMAS self._schema_map = _DEFAULT_SCHEMAS - @abc.abstractmethod - def matches(self, document): - """Whether this Validator should be used to validate ``document``. - - :param dict document: Document to validate. - :returns: True if Validator applies to ``document``, else False. - """ - @abc.abstractmethod def validate(self, document): """Validate whether ``document`` passes schema validation.""" @@ -117,9 +109,32 @@ class GenericValidator(BaseValidator): super(GenericValidator, self).__init__() self.base_schema = self._schema_map['v1']['deckhand/Base'] - def matches(self, document): - # Applies to all schemas, so unconditionally returns True. - return True + def validate_metadata(self, metadata): + """Validate ``metadata`` against the given schema. + + The ``metadata`` section of a Deckhand document describes a schema + defining just the ``metadata`` section. Use that declaration to + choose a schema for validating ``metadata``. + + :param dict metadata: Document metadata section to validate + :returns: list of validation errors or empty list for success + """ + errors = list() + + schema_name, schema_ver = _get_schema_parts(metadata) + schema = self._schema_map.get(schema_ver, {}).get(schema_name, {}) + + if not schema: + return ['Invalid metadata schema %s version %s specified.' + % (schema_name, schema_ver)] + + LOG.debug("Validating document metadata with schema %s/%s.", + schema_name, schema_ver) + jsonschema.Draft4Validator.check_schema(schema) + schema_validator = jsonschema.Draft4Validator(schema) + errors.extend([e.message + for e in schema_validator.iter_errors(metadata)]) + return errors def validate(self, document, **kwargs): """Validate ``document``against basic schema validation. @@ -144,6 +159,10 @@ class GenericValidator(BaseValidator): schema_validator = jsonschema.Draft4Validator(self.base_schema) error_messages = [ e.message for e in schema_validator.iter_errors(document)] + + if not error_messages: + error_messages.extend( + self.validate_metadata(document.metadata)) except Exception as e: raise RuntimeError( 'Unknown error occurred while attempting to use Deckhand ' @@ -201,14 +220,6 @@ class DataSchemaValidator(GenericValidator): self._external_data_schemas = [d.data for d in data_schemas] self._schema_map = self._build_schema_map(data_schemas) - def matches(self, document): - if document.is_abstract: - LOG.info('Skipping schema validation for abstract document [%s]: ' - '%s.', document.schema, document.name) - return False - schema_prefix, schema_version = _get_schema_parts(document) - return schema_prefix in self._schema_map.get(schema_version, {}) - def _generate_validation_error_output(self, schema, document, error, root_path): """Returns a formatted output with necessary details for debugging why @@ -308,6 +319,18 @@ class DataSchemaValidator(GenericValidator): :rtype: Generator[Tuple[str, str]] """ + super(DataSchemaValidator, self).validate(document) + + # if this is a pre_validate, the only validation needed is structural + # for non-control documents + if not document.is_control and pre_validate: + return + + if document.is_abstract: + LOG.info('Skipping schema validation for abstract document [%s, ' + '%s] %s.', *document.meta) + return + schemas_to_use = self._get_schemas(document) if not schemas_to_use: LOG.debug('Document schema %s not recognized by %s. No further ' @@ -315,28 +338,12 @@ class DataSchemaValidator(GenericValidator): self.__class__.__name__) for schema in schemas_to_use: - is_builtin_schema = schema not in self._external_data_schemas - # NOTE(fmontei): The purpose of this `continue` is to not - # PRE-validate documents against externally registered - # `DataSchema` documents, in order to avoid raising spurious - # errors. These spurious errors arise from `DataSchema` documents - # really only applying post-rendering, when documents have all - # the substitutions they need to pass externally registered - # `DataSchema` validations. - if not is_builtin_schema and pre_validate: - continue - - if is_builtin_schema: - root_path = '.' - to_validate = document - else: - root_path = '.data' - to_validate = document.get('data', {}) + root_path = '.data' try: jsonschema.Draft4Validator.check_schema(schema) schema_validator = jsonschema.Draft4Validator(schema) - errors = schema_validator.iter_errors(to_validate) + errors = schema_validator.iter_errors(document.get('data', {})) except Exception as e: LOG.exception(six.text_type(e)) raise RuntimeError( @@ -417,10 +424,7 @@ class DocumentValidation(object): self._documents.append(document) - # NOTE(fmontei): The order of the validators is important. The - # ``GenericValidator`` must come first. self._validators = [ - GenericValidator(), DataSchemaValidator(self._external_data_schemas) ] @@ -476,11 +480,10 @@ class DocumentValidation(object): LOG.info(message) for validator in self._validators: - if validator.matches(document): - error_outputs = validator.validate( - document, pre_validate=self._pre_validate) - if error_outputs: - result['errors'].extend(error_outputs) + error_outputs = validator.validate( + document, pre_validate=self._pre_validate) + if error_outputs: + result['errors'].extend(error_outputs) if result['errors']: result.setdefault('status', 'failure') diff --git a/deckhand/engine/schemas/base_schema.yaml b/deckhand/engine/schemas/base_schema.yaml index 49b3a401..d6cfe71b 100644 --- a/deckhand/engine/schemas/base_schema.yaml +++ b/deckhand/engine/schemas/base_schema.yaml @@ -19,131 +19,36 @@ metadata: schema: metadata/Control/v1 data: $schema: http://json-schema.org/schema# - - definitions: - parent_selector_requires_actions: - dependencies: - # Requires that if parentSelector is provided, then actions is - # required and must contain at least 1 item. - parentSelector: - required: - - actions - actions_requires_parent_selector: - dependencies: - # Requires that if actions are provided, then so too must - # parentSelector. - actions: - required: - - parentSelector - substitution_dest: - type: object - properties: - path: - type: string - pattern: - type: string - additionalProperties: false - required: - - path - properties: schema: type: string pattern: ^[A-Za-z]+/[A-Za-z]+/v\d+$ metadata: + # True validation of the metadata section will be done using + # the schema specfied in the metadata section type: object properties: + name: + type: string schema: anyOf: - type: string pattern: ^metadata/Document/v\d+$ - type: string pattern: ^metadata/Control/v\d+$ - name: - type: string - labels: - type: object - replacement: - type: boolean - layeringDefinition: - type: object - properties: - layer: - type: string - abstract: - type: boolean - parentSelector: - type: object - minProperties: 1 - actions: - type: array - minItems: 1 - items: - type: object - properties: - method: - enum: - - replace - - delete - - merge - path: - type: string - additionalProperties: false - required: - - method - - path - additionalProperties: false - allOf: - - $ref: "#/definitions/parent_selector_requires_actions" - - $ref: "#/definitions/actions_requires_parent_selector" - substitutions: - type: array - items: - type: object - properties: - dest: - anyOf: - - $ref: "#/definitions/substitution_dest" - - type: array - minItems: 1 - items: - $ref: "#/definitions/substitution_dest" - src: - type: object - properties: - schema: - type: string - pattern: ^[A-Za-z]+/[A-Za-z]+/v\d+$ - name: - type: string - path: - type: string - additionalProperties: false - required: - - schema - - name - - path - additionalProperties: false - required: - - dest - - src - storagePolicy: - type: string - enum: - - encrypted - - cleartext - additionalProperties: false + additionalProperties: true required: - - schema - - name + - 'name' + - 'schema' + # This schema should allow anything in the data section data: type: - - "null" - - string - - integer - - array - - object - + - 'null' + - 'string' + - 'object' + - 'array' + - 'number' + - 'boolean' additionalProperties: false required: - schema diff --git a/deckhand/engine/schemas/certificate_authority_key_schema.yaml b/deckhand/engine/schemas/certificate_authority_key_schema.yaml index 3eaaf537..3f4812e6 100644 --- a/deckhand/engine/schemas/certificate_authority_key_schema.yaml +++ b/deckhand/engine/schemas/certificate_authority_key_schema.yaml @@ -19,47 +19,4 @@ metadata: schema: metadata/Control/v1 data: $schema: http://json-schema.org/schema# - properties: - metadata: - type: object - properties: - layeringDefinition: - type: object - properties: - layer: - type: string - abstract: - type: boolean - parentSelector: - type: object - actions: - type: array - items: - type: object - properties: - method: - enum: - - replace - - delete - - merge - path: - type: string - additionalProperties: false - required: - - method - - path - required: - - layer - storagePolicy: - type: string - enum: - - encrypted - - cleartext - required: - - layeringDefinition - - storagePolicy - data: - type: string -required: - - metadata - - data + type: string diff --git a/deckhand/engine/schemas/certificate_authority_schema.yaml b/deckhand/engine/schemas/certificate_authority_schema.yaml index 15e5b896..828ce17a 100644 --- a/deckhand/engine/schemas/certificate_authority_schema.yaml +++ b/deckhand/engine/schemas/certificate_authority_schema.yaml @@ -19,47 +19,4 @@ metadata: schema: metadata/Control/v1 data: $schema: http://json-schema.org/schema# - properties: - metadata: - type: object - properties: - layeringDefinition: - type: object - properties: - layer: - type: string - abstract: - type: boolean - parentSelector: - type: object - actions: - type: array - items: - type: object - properties: - method: - enum: - - replace - - delete - - merge - path: - type: string - additionalProperties: false - required: - - method - - path - required: - - layer - storagePolicy: - type: string - enum: - - encrypted - - cleartext - required: - - layeringDefinition - - storagePolicy - data: - type: string -required: - - metadata - - data + type: string diff --git a/deckhand/engine/schemas/certificate_key_schema.yaml b/deckhand/engine/schemas/certificate_key_schema.yaml index 553fc110..a2d08f5b 100644 --- a/deckhand/engine/schemas/certificate_key_schema.yaml +++ b/deckhand/engine/schemas/certificate_key_schema.yaml @@ -19,47 +19,4 @@ metadata: schema: metadata/Control/v1 data: $schema: http://json-schema.org/schema# - properties: - metadata: - type: object - properties: - layeringDefinition: - type: object - properties: - layer: - type: string - abstract: - type: boolean - parentSelector: - type: object - actions: - type: array - items: - type: object - properties: - method: - enum: - - replace - - delete - - merge - path: - type: string - additionalProperties: false - required: - - method - - path - required: - - layer - storagePolicy: - type: string - enum: - - encrypted - - cleartext - required: - - layeringDefinition - - storagePolicy - data: - type: string -required: - - metadata - - data + type: string diff --git a/deckhand/engine/schemas/certificate_schema.yaml b/deckhand/engine/schemas/certificate_schema.yaml index 0c40e5bc..d89897a4 100644 --- a/deckhand/engine/schemas/certificate_schema.yaml +++ b/deckhand/engine/schemas/certificate_schema.yaml @@ -19,47 +19,4 @@ metadata: schema: metadata/Control/v1 data: $schema: http://json-schema.org/schema# - properties: - metadata: - type: object - properties: - layeringDefinition: - type: object - properties: - layer: - type: string - abstract: - type: boolean - parentSelector: - type: object - actions: - type: array - items: - type: object - properties: - method: - enum: - - replace - - delete - - merge - path: - type: string - additionalProperties: false - required: - - method - - path - required: - - layer - storagePolicy: - type: string - enum: - - encrypted - - cleartext - required: - - layeringDefinition - - storagePolicy - data: - type: string -required: - - metadata - - data + type: string diff --git a/deckhand/engine/schemas/dataschema_schema.yaml b/deckhand/engine/schemas/dataschema_schema.yaml deleted file mode 100644 index 2865bbe5..00000000 --- a/deckhand/engine/schemas/dataschema_schema.yaml +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2017 AT&T Intellectual Property. All other rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -schema: deckhand/DataSchema/v1 -metadata: - name: deckhand/DataSchema/v1 - schema: metadata/Control/v1 -data: - $schema: http://json-schema.org/schema# - type: object - properties: - data: - type: object - properties: - $schema: - type: string - additionalProperties: true - required: - - $schema - required: - - data diff --git a/deckhand/engine/schemas/layering_policy_schema.yaml b/deckhand/engine/schemas/layering_policy_schema.yaml index 5afba5d8..60314c59 100644 --- a/deckhand/engine/schemas/layering_policy_schema.yaml +++ b/deckhand/engine/schemas/layering_policy_schema.yaml @@ -21,15 +21,10 @@ data: $schema: http://json-schema.org/schema# type: object properties: - data: - type: object - properties: - layerOrder: - type: array - items: - type: string - additionalProperties: false - required: - - layerOrder + layerOrder: + type: array + items: + type: string + additionalProperties: false required: - - data + - layerOrder diff --git a/deckhand/engine/schemas/metadata_control.yaml b/deckhand/engine/schemas/metadata_control.yaml new file mode 100644 index 00000000..8a7adb1b --- /dev/null +++ b/deckhand/engine/schemas/metadata_control.yaml @@ -0,0 +1,28 @@ +--- +schema: deckhand/DataSchema/v1 +metadata: + name: metadata/Control/v1 + schema: metadata/Control/v1 +data: + $schema: http://json-schema.org/schema# + type: object + properties: + schema: + anyOf: + - type: string + pattern: ^metadata/Document/v\d+$ + - type: string + pattern: ^metadata/Control/v\d+$ + name: + type: string + labels: + type: object + additionalProperties: + type: string + additionalProperties: true + required: + - schema + - name + # NOTE(felipemonteiro): layeringDefinition is not needed for any control + # documents as neither LayeringPolicy, ValidationPolicy or DataSchema + # documents are ever layered together. diff --git a/deckhand/engine/schemas/metadata_document.yaml b/deckhand/engine/schemas/metadata_document.yaml new file mode 100644 index 00000000..57379b12 --- /dev/null +++ b/deckhand/engine/schemas/metadata_document.yaml @@ -0,0 +1,121 @@ +--- +schema: deckhand/DataSchema/v1 +metadata: + name: metadata/Document/v1 + schema: metadata/Control/v1 +data: + $schema: http://json-schema.org/schema# + definitions: + parent_selector_requires_actions: + dependencies: + # Requires that if parentSelector is provided, then actions is + # required and must contain at least 1 item. + parentSelector: + required: + - actions + actions_requires_parent_selector: + dependencies: + # Requires that if actions are provided, then so too must + # parentSelector. + actions: + required: + - parentSelector + substitution_dest: + type: object + properties: + path: + type: string + pattern: + type: string + additionalProperties: false + required: + - path + type: object + properties: + schema: + anyOf: + - type: string + pattern: ^metadata/Document/v\d+$ + - type: string + pattern: ^metadata/Control/v\d+$ + name: + type: string + labels: + type: object + replacement: + type: boolean + layeringDefinition: + type: object + properties: + layer: + type: string + abstract: + type: boolean + parentSelector: + type: object + minProperties: 1 + actions: + type: array + minItems: 1 + items: + type: object + properties: + method: + enum: + - replace + - delete + - merge + path: + type: string + additionalProperties: false + required: + - method + - path + additionalProperties: false + required: + - 'layer' + allOf: + - $ref: "#/definitions/parent_selector_requires_actions" + - $ref: "#/definitions/actions_requires_parent_selector" + substitutions: + type: array + items: + type: object + properties: + dest: + anyOf: + - $ref: "#/definitions/substitution_dest" + - type: array + minItems: 1 + items: + $ref: "#/definitions/substitution_dest" + src: + type: object + properties: + schema: + type: string + pattern: ^[A-Za-z]+/[A-Za-z]+/v\d+$ + name: + type: string + path: + type: string + additionalProperties: false + required: + - schema + - name + - path + additionalProperties: false + required: + - dest + - src + storagePolicy: + type: string + enum: + - encrypted + - cleartext + additionalProperties: false + required: + - schema + - name + - storagePolicy + - layeringDefinition diff --git a/deckhand/engine/schemas/passphrase_schema.yaml b/deckhand/engine/schemas/passphrase_schema.yaml index 949e3458..f1f12265 100644 --- a/deckhand/engine/schemas/passphrase_schema.yaml +++ b/deckhand/engine/schemas/passphrase_schema.yaml @@ -19,47 +19,4 @@ metadata: schema: metadata/Control/v1 data: $schema: http://json-schema.org/schema# - properties: - metadata: - type: object - properties: - layeringDefinition: - type: object - properties: - layer: - type: string - abstract: - type: boolean - parentSelector: - type: object - actions: - type: array - items: - type: object - properties: - method: - enum: - - replace - - delete - - merge - path: - type: string - additionalProperties: false - required: - - method - - path - required: - - layer - storagePolicy: - type: string - enum: - - encrypted - - cleartext - required: - - layeringDefinition - - storagePolicy - data: - type: string -required: - - metadata - - data + type: string diff --git a/deckhand/engine/schemas/private_key_schema.yaml b/deckhand/engine/schemas/private_key_schema.yaml index 6c6fd6d0..68582251 100644 --- a/deckhand/engine/schemas/private_key_schema.yaml +++ b/deckhand/engine/schemas/private_key_schema.yaml @@ -19,47 +19,4 @@ metadata: schema: metadata/Control/v1 data: $schema: http://json-schema.org/schema# - properties: - metadata: - type: object - properties: - layeringDefinition: - type: object - properties: - layer: - type: string - abstract: - type: boolean - parentSelector: - type: object - actions: - type: array - items: - type: object - properties: - method: - enum: - - replace - - delete - - merge - path: - type: string - additionalProperties: false - required: - - method - - path - required: - - layer - storagePolicy: - type: string - enum: - - encrypted - - cleartext - required: - - layeringDefinition - - storagePolicy - data: - type: string -required: - - metadata - - data + type: string diff --git a/deckhand/engine/schemas/public_key_schema.yaml b/deckhand/engine/schemas/public_key_schema.yaml index de9887de..7696cfa2 100644 --- a/deckhand/engine/schemas/public_key_schema.yaml +++ b/deckhand/engine/schemas/public_key_schema.yaml @@ -19,47 +19,4 @@ metadata: schema: metadata/Control/v1 data: $schema: http://json-schema.org/schema# - properties: - metadata: - type: object - properties: - layeringDefinition: - type: object - properties: - layer: - type: string - abstract: - type: boolean - parentSelector: - type: object - actions: - type: array - items: - type: object - properties: - method: - enum: - - replace - - delete - - merge - path: - type: string - additionalProperties: false - required: - - method - - path - required: - - layer - storagePolicy: - type: string - enum: - - encrypted - - cleartext - required: - - layeringDefinition - - storagePolicy - data: - type: string -required: - - metadata - - data + type: string diff --git a/deckhand/engine/schemas/validation_policy_schema.yaml b/deckhand/engine/schemas/validation_policy_schema.yaml index ff4f090a..94a2a657 100644 --- a/deckhand/engine/schemas/validation_policy_schema.yaml +++ b/deckhand/engine/schemas/validation_policy_schema.yaml @@ -21,56 +21,19 @@ data: $schema: http://json-schema.org/schema# type: object properties: - metadata: - type: object - properties: - layeringDefinition: - type: object - properties: - layer: - type: string - abstract: - type: boolean - parentSelector: - type: object - actions: - type: array - items: - type: object - properties: - method: - enum: - - replace - - delete - - merge - path: - type: string - additionalProperties: false - required: - - method - - path - required: - - layer - required: - - layeringDefinition - data: - properties: - validations: - type: array - items: - type: object - properties: - name: - type: string - pattern: ^.*-(validation|verification)$ - expiresAfter: - type: string - additionalProperties: false - required: - - name - required: - - validations - additionalProperties: false + validations: + type: array + items: + type: object + properties: + name: + type: string + pattern: ^.*-(validation|verification)$ + expiresAfter: + type: string + additionalProperties: false + required: + - name required: - - metadata - - data + - validations + additionalProperties: false diff --git a/deckhand/factories.py b/deckhand/factories.py index ef063ecb..ca0d592f 100644 --- a/deckhand/factories.py +++ b/deckhand/factories.py @@ -103,6 +103,7 @@ class DocumentFactory(DeckhandFactory): "data": {}, "metadata": { "labels": {"": ""}, + "storagePolicy": "cleartext", "layeringDefinition": { "abstract": False, "layer": "layer" diff --git a/deckhand/tests/functional/gabbits/layering/layering-with-replacement-single-bucket.yaml b/deckhand/tests/functional/gabbits/layering/layering-with-replacement-single-bucket.yaml index 516f536a..7b2f4167 100644 --- a/deckhand/tests/functional/gabbits/layering/layering-with-replacement-single-bucket.yaml +++ b/deckhand/tests/functional/gabbits/layering/layering-with-replacement-single-bucket.yaml @@ -43,6 +43,7 @@ tests: metadata: schema: metadata/Document/v1 name: a + storagePolicy: cleartext labels: selector: foo layeringDefinition: @@ -56,6 +57,7 @@ tests: metadata: schema: metadata/Document/v1 name: a + storagePolicy: cleartext labels: selector: baz replacement: true @@ -75,6 +77,7 @@ tests: metadata: schema: metadata/Document/v1 name: c + storagePolicy: cleartext layeringDefinition: abstract: False layer: global @@ -140,6 +143,7 @@ tests: metadata: schema: metadata/Document/v1 name: a + storagePolicy: cleartext labels: selector: foo layeringDefinition: @@ -153,6 +157,7 @@ tests: metadata: schema: metadata/Document/v1 name: a + storagePolicy: cleartext labels: selector: baz replacement: true @@ -172,6 +177,7 @@ tests: metadata: schema: metadata/Document/v1 name: b + storagePolicy: cleartext labels: selector: qux layeringDefinition: @@ -190,6 +196,7 @@ tests: metadata: schema: metadata/Document/v1 name: c + storagePolicy: cleartext layeringDefinition: abstract: False layer: global diff --git a/deckhand/tests/functional/gabbits/replacement/multi-layer-replacement.yaml b/deckhand/tests/functional/gabbits/replacement/multi-layer-replacement.yaml index 8cd970aa..eb8991e4 100644 --- a/deckhand/tests/functional/gabbits/replacement/multi-layer-replacement.yaml +++ b/deckhand/tests/functional/gabbits/replacement/multi-layer-replacement.yaml @@ -46,6 +46,7 @@ tests: metadata: schema: metadata/Document/v1 name: nova-global + storagePolicy: cleartext labels: name: nova-global component: nova @@ -62,6 +63,7 @@ tests: metadata: schema: metadata/Document/v1 name: nova + storagePolicy: cleartext labels: name: nova-5ec component: nova @@ -80,6 +82,7 @@ tests: schema: metadata/Document/v1 replacement: true name: nova + storagePolicy: cleartext layeringDefinition: abstract: false layer: site diff --git a/deckhand/tests/functional/gabbits/resources/chained-substitution.yaml b/deckhand/tests/functional/gabbits/resources/chained-substitution.yaml index 6e159e1b..c9a2ec55 100644 --- a/deckhand/tests/functional/gabbits/resources/chained-substitution.yaml +++ b/deckhand/tests/functional/gabbits/resources/chained-substitution.yaml @@ -11,6 +11,7 @@ schema: example/Source/v1 metadata: schema: metadata/Document/v1 name: source + storagePolicy: cleartext layeringDefinition: abstract: false layer: one @@ -20,6 +21,7 @@ schema: example/Middle/v1 metadata: schema: metadata/Document/v1 name: middle + storagePolicy: cleartext layeringDefinition: abstract: false layer: one @@ -36,6 +38,7 @@ schema: example/Dest/v1 metadata: schema: metadata/Document/v1 name: dest + storagePolicy: cleartext layeringDefinition: abstract: false layer: one diff --git a/deckhand/tests/functional/gabbits/resources/design-doc-layering-sample-2-layers.yaml b/deckhand/tests/functional/gabbits/resources/design-doc-layering-sample-2-layers.yaml index fa1c306e..d09e62c3 100644 --- a/deckhand/tests/functional/gabbits/resources/design-doc-layering-sample-2-layers.yaml +++ b/deckhand/tests/functional/gabbits/resources/design-doc-layering-sample-2-layers.yaml @@ -12,6 +12,7 @@ schema: example/Kind/v1 metadata: schema: metadata/Document/v1 name: global-1234 + storagePolicy: cleartext labels: key1: value1 key2: value2 @@ -27,6 +28,7 @@ schema: example/Kind/v1 metadata: schema: metadata/Document/v1 name: site-1234 + storagePolicy: cleartext layeringDefinition: layer: site parentSelector: diff --git a/deckhand/tests/functional/gabbits/resources/design-doc-layering-sample-3-layers.yaml b/deckhand/tests/functional/gabbits/resources/design-doc-layering-sample-3-layers.yaml index da321250..146af548 100644 --- a/deckhand/tests/functional/gabbits/resources/design-doc-layering-sample-3-layers.yaml +++ b/deckhand/tests/functional/gabbits/resources/design-doc-layering-sample-3-layers.yaml @@ -13,6 +13,7 @@ schema: example/Kind/v1 metadata: schema: metadata/Document/v1 name: global-1234 + storagePolicy: cleartext labels: key1: value1 layeringDefinition: @@ -27,6 +28,7 @@ schema: example/Kind/v1 metadata: schema: metadata/Document/v1 name: region-1234 + storagePolicy: cleartext labels: key1: value1 layeringDefinition: @@ -45,6 +47,7 @@ schema: example/Kind/v1 metadata: schema: metadata/Document/v1 name: site-1234 + storagePolicy: cleartext labels: foo: bar baz: qux diff --git a/deckhand/tests/functional/gabbits/resources/design-doc-layering-sample-split-bucket-a.yaml b/deckhand/tests/functional/gabbits/resources/design-doc-layering-sample-split-bucket-a.yaml index 39b673cb..6cae707c 100644 --- a/deckhand/tests/functional/gabbits/resources/design-doc-layering-sample-split-bucket-a.yaml +++ b/deckhand/tests/functional/gabbits/resources/design-doc-layering-sample-split-bucket-a.yaml @@ -3,6 +3,7 @@ schema: example/Kind/v1 metadata: schema: metadata/Document/v1 name: global-1234 + storagePolicy: cleartext labels: key1: value1 layeringDefinition: diff --git a/deckhand/tests/functional/gabbits/resources/design-doc-layering-sample-split-bucket-b.yaml b/deckhand/tests/functional/gabbits/resources/design-doc-layering-sample-split-bucket-b.yaml index 7b598a47..c433f6c9 100644 --- a/deckhand/tests/functional/gabbits/resources/design-doc-layering-sample-split-bucket-b.yaml +++ b/deckhand/tests/functional/gabbits/resources/design-doc-layering-sample-split-bucket-b.yaml @@ -3,6 +3,7 @@ schema: example/Kind/v1 metadata: schema: metadata/Document/v1 name: region-1234 + storagePolicy: cleartext labels: key1: value1 layeringDefinition: @@ -21,6 +22,7 @@ schema: example/Kind/v1 metadata: schema: metadata/Document/v1 name: site-with-merge-action + storagePolicy: cleartext labels: foo: bar baz: qux @@ -38,6 +40,7 @@ schema: example/Kind/v1 metadata: schema: metadata/Document/v1 name: site-with-delete-action + storagePolicy: cleartext layeringDefinition: layer: site parentSelector: diff --git a/deckhand/tests/functional/gabbits/resources/design-doc-layering-sample-with-delete.yaml b/deckhand/tests/functional/gabbits/resources/design-doc-layering-sample-with-delete.yaml index 064b678f..68064e56 100644 --- a/deckhand/tests/functional/gabbits/resources/design-doc-layering-sample-with-delete.yaml +++ b/deckhand/tests/functional/gabbits/resources/design-doc-layering-sample-with-delete.yaml @@ -13,6 +13,7 @@ schema: example/Kind/v1 metadata: schema: metadata/Document/v1 name: global-1234 + storagePolicy: cleartext labels: key1: value1 layeringDefinition: @@ -27,6 +28,7 @@ schema: example/Kind/v1 metadata: schema: metadata/Document/v1 name: site-1234 + storagePolicy: cleartext layeringDefinition: layer: site parentSelector: diff --git a/deckhand/tests/functional/gabbits/resources/design-doc-layering-sample-with-update.yaml b/deckhand/tests/functional/gabbits/resources/design-doc-layering-sample-with-update.yaml index e667fd0b..3d87e9a1 100644 --- a/deckhand/tests/functional/gabbits/resources/design-doc-layering-sample-with-update.yaml +++ b/deckhand/tests/functional/gabbits/resources/design-doc-layering-sample-with-update.yaml @@ -13,6 +13,7 @@ schema: example/Kind/v1 metadata: schema: metadata/Document/v1 name: global-1234 + storagePolicy: cleartext labels: key1: value1 layeringDefinition: @@ -27,6 +28,7 @@ schema: example/Kind/v1 metadata: schema: metadata/Document/v1 name: region-1234 + storagePolicy: cleartext labels: key1: value1 layeringDefinition: @@ -45,6 +47,7 @@ schema: example/Kind/v1 metadata: schema: metadata/Document/v1 name: site-1234 + storagePolicy: cleartext layeringDefinition: layer: site parentSelector: diff --git a/deckhand/tests/functional/gabbits/resources/design-doc-substitution-generic-sample.yaml b/deckhand/tests/functional/gabbits/resources/design-doc-substitution-generic-sample.yaml index 6ea6b25f..7e9404ec 100644 --- a/deckhand/tests/functional/gabbits/resources/design-doc-substitution-generic-sample.yaml +++ b/deckhand/tests/functional/gabbits/resources/design-doc-substitution-generic-sample.yaml @@ -41,6 +41,7 @@ schema: armada/Chart/v1 metadata: name: example-chart-01 schema: metadata/Document/v1 + storagePolicy: cleartext layeringDefinition: layer: region substitutions: diff --git a/deckhand/tests/functional/gabbits/resources/design-doc-substitution-sample-split-bucket-b.yaml b/deckhand/tests/functional/gabbits/resources/design-doc-substitution-sample-split-bucket-b.yaml index 51ef71c0..5959ba32 100644 --- a/deckhand/tests/functional/gabbits/resources/design-doc-substitution-sample-split-bucket-b.yaml +++ b/deckhand/tests/functional/gabbits/resources/design-doc-substitution-sample-split-bucket-b.yaml @@ -31,6 +31,7 @@ schema: armada/Chart/v1 metadata: name: example-chart-01 schema: metadata/Document/v1 + storagePolicy: cleartext layeringDefinition: layer: region substitutions: diff --git a/deckhand/tests/functional/gabbits/resources/design-doc-substitution-sample.yaml b/deckhand/tests/functional/gabbits/resources/design-doc-substitution-sample.yaml index 66086a5b..1bbca885 100644 --- a/deckhand/tests/functional/gabbits/resources/design-doc-substitution-sample.yaml +++ b/deckhand/tests/functional/gabbits/resources/design-doc-substitution-sample.yaml @@ -41,6 +41,7 @@ schema: armada/Chart/v1 metadata: name: example-chart-01 schema: metadata/Document/v1 + storagePolicy: cleartext layeringDefinition: layer: region substitutions: diff --git a/deckhand/tests/functional/gabbits/resources/layering-and-substitution-dag-sample.yaml b/deckhand/tests/functional/gabbits/resources/layering-and-substitution-dag-sample.yaml index d3ce220b..85884070 100644 --- a/deckhand/tests/functional/gabbits/resources/layering-and-substitution-dag-sample.yaml +++ b/deckhand/tests/functional/gabbits/resources/layering-and-substitution-dag-sample.yaml @@ -35,6 +35,7 @@ schema: armada/Chart/v1 metadata: schema: metadata/Document/v1 name: armada-chart-03 + storagePolicy: cleartext layeringDefinition: abstract: false layer: site @@ -72,6 +73,7 @@ schema: armada/Chart/v1 metadata: schema: metadata/Document/v1 name: armada-chart-02 + storagePolicy: cleartext layeringDefinition: abstract: false layer: site @@ -109,6 +111,7 @@ schema: armada/Chart/v1 metadata: schema: metadata/Document/v1 name: armada-chart-01 + storagePolicy: cleartext labels: key1: value1 layeringDefinition: diff --git a/deckhand/tests/functional/gabbits/resources/layering-and-substitution-sample.yaml b/deckhand/tests/functional/gabbits/resources/layering-and-substitution-sample.yaml index 7bd6ad00..f2d7d344 100644 --- a/deckhand/tests/functional/gabbits/resources/layering-and-substitution-sample.yaml +++ b/deckhand/tests/functional/gabbits/resources/layering-and-substitution-sample.yaml @@ -3,6 +3,7 @@ schema: deckhand/Certificate/v1 metadata: name: example-cert schema: metadata/Document/v1 + storagePolicy: cleartext layeringDefinition: layer: site storagePolicy: cleartext @@ -13,6 +14,7 @@ schema: deckhand/CertificateKey/v1 metadata: name: example-key schema: metadata/Document/v1 + storagePolicy: cleartext layeringDefinition: layer: site storagePolicy: cleartext @@ -23,6 +25,7 @@ schema: deckhand/Passphrase/v1 metadata: name: example-password schema: metadata/Document/v1 + storagePolicy: cleartext layeringDefinition: abstract: false layer: site @@ -33,6 +36,7 @@ schema: armada/Chart/v1 metadata: schema: metadata/Document/v1 name: armada-chartgroup-01 + storagePolicy: cleartext labels: key1: value1 layeringDefinition: @@ -45,6 +49,7 @@ schema: armada/Chart/v1 metadata: schema: metadata/Document/v1 name: armada-chart-01 + storagePolicy: cleartext layeringDefinition: layer: site parentSelector: diff --git a/deckhand/tests/functional/gabbits/resources/layering-needs-substitution-source.yaml b/deckhand/tests/functional/gabbits/resources/layering-needs-substitution-source.yaml index 795c298f..4985ca3e 100644 --- a/deckhand/tests/functional/gabbits/resources/layering-needs-substitution-source.yaml +++ b/deckhand/tests/functional/gabbits/resources/layering-needs-substitution-source.yaml @@ -13,6 +13,7 @@ schema: example/Kind/v1 metadata: schema: metadata/Document/v1 name: abstract-1234 + storagePolicy: cleartext labels: key1: value1 layeringDefinition: @@ -27,6 +28,7 @@ schema: example/Kind/v1 metadata: schema: metadata/Document/v1 name: concrete-1234 + storagePolicy: cleartext layeringDefinition: layer: site parentSelector: diff --git a/deckhand/tests/functional/gabbits/resources/passphrase.yaml b/deckhand/tests/functional/gabbits/resources/passphrase.yaml index 40c164f4..b853c286 100644 --- a/deckhand/tests/functional/gabbits/resources/passphrase.yaml +++ b/deckhand/tests/functional/gabbits/resources/passphrase.yaml @@ -4,5 +4,8 @@ metadata: schema: metadata/Document/v1 name: my-passphrase storagePolicy: cleartext + layeringDefinition: + abstract: false + layer: site data: not-a-real-password ... diff --git a/deckhand/tests/functional/gabbits/resources/replacement.yaml b/deckhand/tests/functional/gabbits/resources/replacement.yaml index 3caac479..770a4019 100644 --- a/deckhand/tests/functional/gabbits/resources/replacement.yaml +++ b/deckhand/tests/functional/gabbits/resources/replacement.yaml @@ -32,6 +32,7 @@ schema: armada/Chart/v1 metadata: name: example-chart-01 schema: metadata/Document/v1 + storagePolicy: cleartext labels: name: parent-chart layeringDefinition: @@ -53,6 +54,7 @@ schema: armada/Chart/v1 metadata: name: example-chart-01 schema: metadata/Document/v1 + storagePolicy: cleartext replacement: true layeringDefinition: layer: site diff --git a/deckhand/tests/functional/gabbits/resources/sample-doc.yaml b/deckhand/tests/functional/gabbits/resources/sample-doc.yaml index ba7be94c..37d8617d 100644 --- a/deckhand/tests/functional/gabbits/resources/sample-doc.yaml +++ b/deckhand/tests/functional/gabbits/resources/sample-doc.yaml @@ -3,6 +3,7 @@ schema: promenade/ResourceType/v1 metadata: schema: metadata/Document/v1 name: a-unique-config-name-12345 + storagePolicy: cleartext labels: component: apiserver hostname: server0 diff --git a/deckhand/tests/functional/gabbits/resources/substitution-results-in-none-bug.yaml b/deckhand/tests/functional/gabbits/resources/substitution-results-in-none-bug.yaml index 4f4ba985..d3c6a4b5 100644 --- a/deckhand/tests/functional/gabbits/resources/substitution-results-in-none-bug.yaml +++ b/deckhand/tests/functional/gabbits/resources/substitution-results-in-none-bug.yaml @@ -22,6 +22,7 @@ schema: deckhand/Dest/v1 metadata: name: dest schema: metadata/Document/v1 + storagePolicy: cleartext layeringDefinition: layer: one substitutions: diff --git a/deckhand/tests/functional/gabbits/revision-diff/revision-diff-success.yaml b/deckhand/tests/functional/gabbits/revision-diff/revision-diff-success.yaml index d81f4df0..5393251a 100644 --- a/deckhand/tests/functional/gabbits/revision-diff/revision-diff-success.yaml +++ b/deckhand/tests/functional/gabbits/revision-diff/revision-diff-success.yaml @@ -39,6 +39,7 @@ tests: metadata: schema: metadata/Document/v1 name: doc-a + storagePolicy: cleartext layeringDefinition: abstract: false layer: site @@ -56,6 +57,7 @@ tests: metadata: schema: metadata/Document/v1 name: doc-b + storagePolicy: cleartext layeringDefinition: abstract: false layer: site @@ -73,6 +75,7 @@ tests: metadata: schema: metadata/Document/v1 name: doc-c + storagePolicy: cleartext layeringDefinition: abstract: false layer: site @@ -90,6 +93,7 @@ tests: metadata: schema: metadata/Document/v1 name: doc-d + storagePolicy: cleartext layeringDefinition: abstract: false layer: site @@ -157,6 +161,7 @@ tests: metadata: schema: metadata/Document/v1 name: doc-c + storagePolicy: cleartext layeringDefinition: abstract: false layer: site @@ -192,6 +197,7 @@ tests: metadata: schema: metadata/Document/v1 name: doc-m + storagePolicy: cleartext layeringDefinition: abstract: false layer: site @@ -231,6 +237,7 @@ tests: metadata: schema: metadata/Document/v1 name: doc-e + storagePolicy: cleartext layeringDefinition: abstract: false layer: site diff --git a/deckhand/tests/functional/gabbits/schema-validation/schema-validation-success.yaml b/deckhand/tests/functional/gabbits/schema-validation/schema-validation-success.yaml index dfbfa056..d984a1cd 100644 --- a/deckhand/tests/functional/gabbits/schema-validation/schema-validation-success.yaml +++ b/deckhand/tests/functional/gabbits/schema-validation/schema-validation-success.yaml @@ -73,6 +73,7 @@ tests: metadata: schema: metadata/Document/v1 name: good + storagePolicy: cleartext layeringDefinition: abstract: false layer: site @@ -114,6 +115,7 @@ tests: metadata: schema: metadata/Document/v1 name: bad + storagePolicy: cleartext layeringDefinition: abstract: false layer: site @@ -173,6 +175,7 @@ tests: metadata: schema: metadata/Document/v1 name: bad + storagePolicy: cleartext layeringDefinition: abstract: false layer: site @@ -191,6 +194,7 @@ tests: metadata: name: test-certificate schema: metadata/Document/v1 + storagePolicy: cleartext layeringDefinition: layer: site storagePolicy: cleartext diff --git a/deckhand/tests/functional/gabbits/substitution/substitution-source-feeds-multi-dest.yaml b/deckhand/tests/functional/gabbits/substitution/substitution-source-feeds-multi-dest.yaml index 2eddf647..613ef89d 100644 --- a/deckhand/tests/functional/gabbits/substitution/substitution-source-feeds-multi-dest.yaml +++ b/deckhand/tests/functional/gabbits/substitution/substitution-source-feeds-multi-dest.yaml @@ -50,6 +50,7 @@ tests: metadata: name: example-chart-01 schema: metadata/Document/v1 + storagePolicy: cleartext layeringDefinition: layer: site substitutions: diff --git a/deckhand/tests/unit/control/test_revision_documents_controller.py b/deckhand/tests/unit/control/test_revision_documents_controller.py index e525d2d4..85911efc 100644 --- a/deckhand/tests/unit/control/test_revision_documents_controller.py +++ b/deckhand/tests/unit/control/test_revision_documents_controller.py @@ -38,6 +38,7 @@ schema: aic/Versions/v1 metadata: name: with-anchor schema: metadata/Document/v1 + storagePolicy: cleartext labels: selector: foo1 layeringDefinition: diff --git a/deckhand/tests/unit/control/test_validations_controller.py b/deckhand/tests/unit/control/test_validations_controller.py index 2832b3df..5b1cebdf 100644 --- a/deckhand/tests/unit/control/test_validations_controller.py +++ b/deckhand/tests/unit/control/test_validations_controller.py @@ -678,6 +678,7 @@ class TestValidationsControllerPostValidate(BaseValidationsControllerTest): 'error_section': { 'data': {'a': 'fail'}, 'metadata': {'labels': {'global': 'global1'}, + 'storagePolicy': 'cleartext', 'layeringDefinition': {'abstract': False, 'layer': 'global'}, 'name': doc_to_test['metadata']['name'], diff --git a/deckhand/tests/unit/engine/test_document_layering_and_replacement.py b/deckhand/tests/unit/engine/test_document_layering_and_replacement.py index a500078b..057bd58d 100644 --- a/deckhand/tests/unit/engine/test_document_layering_and_replacement.py +++ b/deckhand/tests/unit/engine/test_document_layering_and_replacement.py @@ -30,6 +30,7 @@ schema: deckhand/LayeringPolicy/v1 metadata: schema: metadata/Control/v1 name: layering-policy + storagePolicy: cleartext data: layerOrder: - global @@ -39,6 +40,7 @@ schema: aic/Versions/v1 metadata: schema: metadata/Document/v1 name: a + storagePolicy: cleartext labels: selector: foo layeringDefinition: @@ -52,6 +54,7 @@ schema: aic/Versions/v1 metadata: schema: metadata/Document/v1 name: a + storagePolicy: cleartext labels: selector: baz replacement: true @@ -71,6 +74,7 @@ schema: armada/Chart/v1 metadata: schema: metadata/Document/v1 name: c + storagePolicy: cleartext layeringDefinition: abstract: False layer: global @@ -124,6 +128,7 @@ schema: deckhand/LayeringPolicy/v1 metadata: schema: metadata/Control/v1 name: layering-policy + storagePolicy: cleartext data: layerOrder: - global @@ -133,6 +138,7 @@ schema: aic/Versions/v1 metadata: schema: metadata/Document/v1 name: a + storagePolicy: cleartext labels: selector: foo layeringDefinition: @@ -149,6 +155,7 @@ metadata: labels: selector: baz replacement: true + storagePolicy: cleartext layeringDefinition: abstract: False layer: site @@ -165,6 +172,7 @@ schema: aic/Versions/v1 metadata: schema: metadata/Document/v1 name: b + storagePolicy: cleartext labels: selector: qux layeringDefinition: @@ -183,6 +191,7 @@ schema: armada/Chart/v1 metadata: schema: metadata/Document/v1 name: c + storagePolicy: cleartext layeringDefinition: abstract: False layer: global @@ -265,6 +274,7 @@ schema: deckhand/LayeringPolicy/v1 metadata: schema: metadata/Control/v1 name: layering-policy + storagePolicy: cleartext data: layerOrder: - global @@ -275,6 +285,7 @@ schema: armada/Chart/v1 metadata: schema: metadata/Document/v1 name: nova-global + storagePolicy: cleartext labels: name: nova-global component: nova @@ -291,6 +302,7 @@ schema: armada/Chart/v1 metadata: schema: metadata/Document/v1 name: nova + storagePolicy: cleartext labels: name: nova-5ec component: nova @@ -308,6 +320,7 @@ schema: armada/Chart/v1 metadata: schema: metadata/Document/v1 replacement: true + storagePolicy: cleartext name: nova layeringDefinition: abstract: false diff --git a/deckhand/tests/unit/engine/test_document_validation.py b/deckhand/tests/unit/engine/test_document_validation.py index 720335cc..db04c54b 100644 --- a/deckhand/tests/unit/engine/test_document_validation.py +++ b/deckhand/tests/unit/engine/test_document_validation.py @@ -66,8 +66,10 @@ class TestDocumentValidation(engine_test_base.TestDocumentValidationBase): self, mock_jsonschema): m_args = mock.Mock() mock_jsonschema.Draft4Validator(m_args).iter_errors.side_effect = [ - # Return empty list of errors for base schema validator and pretend - # that 1 error is returned for next validator. + # Return empty list of errors for base schema and metadata + # validator and pretend that 1 error is returned for next + # validator. + [], [], [mock.Mock(path=[], schema_path=[], message='scary-secret-here')] ] diff --git a/deckhand/tests/unit/engine/test_document_validation_negative.py b/deckhand/tests/unit/engine/test_document_validation_negative.py index 251f74b9..82c430ef 100644 --- a/deckhand/tests/unit/engine/test_document_validation_negative.py +++ b/deckhand/tests/unit/engine/test_document_validation_negative.py @@ -23,26 +23,39 @@ from deckhand import types class TestDocumentValidationNegative(test_base.TestDocumentValidationBase): """Negative testing suite for document validation.""" - BASIC_PROPERTIES = ( + # Control documents don't require layeringDefinition as none of them + # are rendered -- they are static documents. It is also not meaningful + # to encrypt control documents. + BASIC_CONTROL_PROPERTIES = ( 'metadata', 'metadata.schema', 'metadata.name', - 'metadata.layeringDefinition', - 'metadata.layeringDefinition.layer', - 'schema' + 'schema', ) - CRITICAL_PROPERTIES = ( + BASIC_DOCUMENT_PROPERTIES = BASIC_CONTROL_PROPERTIES + ( + 'metadata.layeringDefinition', + 'metadata.layeringDefinition.layer', + 'metadata.storagePolicy', + ) + + CRITICAL_CONTROL_PROPERTIES = ( 'schema', 'metadata', 'metadata.schema', 'metadata.name', + ) + + CRITICAL_DOCUMENT_PROPERTIES = CRITICAL_CONTROL_PROPERTIES + ( + 'metadata.layeringDefinition', + 'metadata.layeringDefinition.layer', 'metadata.substitutions.0.dest', 'metadata.substitutions.0.dest.path', 'metadata.substitutions.0.src', 'metadata.substitutions.0.src.schema', 'metadata.substitutions.0.src.name', - 'metadata.substitutions.0.src.path' + 'metadata.substitutions.0.src.path', + 'metadata.storagePolicy', ) def _do_validations(self, document_validator, expected, expected_err): @@ -68,11 +81,19 @@ class TestDocumentValidationNegative(test_base.TestDocumentValidationBase): validations[-1]['errors'][-1]['message']) def _test_missing_required_sections(self, document, properties_to_remove): + if document['metadata']['schema'].startswith(types.CONTROL): + critial_properties = self.CRITICAL_CONTROL_PROPERTIES + elif document['metadata']['schema'].startswith(types.DOCUMENT): + critial_properties = self.CRITICAL_DOCUMENT_PROPERTIES + else: + self.fail('Document `metadata.schema` must start with ' + '"metadata/Document" or "metadata/Control".') + for idx, property_to_remove in enumerate(properties_to_remove): missing_prop = property_to_remove.split('.')[-1] invalid_data = self._corrupt_data(document, property_to_remove) - exception_raised = property_to_remove in self.CRITICAL_PROPERTIES + exception_raised = property_to_remove in critial_properties expected_err_msg = "'%s' is a required property" % missing_prop payload = [invalid_data] @@ -87,42 +108,36 @@ class TestDocumentValidationNegative(test_base.TestDocumentValidationBase): def test_certificate_authority_key_missing_required_sections(self): document = self._read_data('sample_certificate_authority_key') - properties_to_remove = tuple(self.BASIC_PROPERTIES) + ( + properties_to_remove = tuple(self.BASIC_DOCUMENT_PROPERTIES) + ( 'metadata.storagePolicy',) self._test_missing_required_sections(document, properties_to_remove) def test_certificate_authority_missing_required_sections(self): document = self._read_data('sample_certificate_authority') - properties_to_remove = tuple(self.BASIC_PROPERTIES) + ( + properties_to_remove = tuple(self.BASIC_DOCUMENT_PROPERTIES) + ( 'metadata.storagePolicy',) self._test_missing_required_sections(document, properties_to_remove) def test_certificate_key_missing_required_sections(self): document = self._read_data('sample_certificate_key') - properties_to_remove = tuple(self.BASIC_PROPERTIES) + ( + properties_to_remove = tuple(self.BASIC_DOCUMENT_PROPERTIES) + ( 'metadata.storagePolicy',) self._test_missing_required_sections(document, properties_to_remove) def test_certificate_missing_required_sections(self): document = self._read_data('sample_certificate') - properties_to_remove = tuple(self.BASIC_PROPERTIES) + ( + properties_to_remove = tuple(self.BASIC_DOCUMENT_PROPERTIES) + ( 'metadata.storagePolicy',) self._test_missing_required_sections(document, properties_to_remove) def test_data_schema_missing_required_sections(self): - properties_to_remove = ( - 'metadata', - 'metadata.schema', - 'metadata.name', - 'schema', - 'data.$schema' - ) document = self._read_data('sample_data_schema') + properties_to_remove = tuple(self.BASIC_CONTROL_PROPERTIES) self._test_missing_required_sections(document, properties_to_remove) def test_generic_document_missing_required_sections(self): document = self._read_data('sample_document') - properties_to_remove = self.CRITICAL_PROPERTIES + properties_to_remove = self.CRITICAL_DOCUMENT_PROPERTIES self._test_missing_required_sections(document, properties_to_remove) def test_generic_document_missing_multiple_required_sections(self): @@ -152,6 +167,12 @@ class TestDocumentValidationNegative(test_base.TestDocumentValidationBase): error_re = r"%s is a required property" % missing_property self.assertRegex(str(e.error_list).replace("\'", ""), error_re) + def test_layering_policy_missing_required_sections(self): + properties_to_remove = tuple(self.BASIC_CONTROL_PROPERTIES) + ( + 'data.layerOrder',) + document = self._read_data('sample_layering_policy') + self._test_missing_required_sections(document, properties_to_remove) + def test_document_invalid_layering_definition_action(self): document = self._read_data('sample_document') missing_data = self._corrupt_data( @@ -166,38 +187,27 @@ class TestDocumentValidationNegative(test_base.TestDocumentValidationBase): doc_validator.validate_all) self.assertRegex(str(e.error_list[0]).replace("\'", ""), error_re) - def test_layering_policy_missing_required_sections(self): - properties_to_remove = ( - 'metadata', - 'metadata.schema', - 'metadata.name', - 'schema', - 'data.layerOrder' - ) - document = self._read_data('sample_layering_policy') - self._test_missing_required_sections(document, properties_to_remove) - def test_passphrase_missing_required_sections(self): document = self._read_data('sample_passphrase') - properties_to_remove = tuple(self.BASIC_PROPERTIES) + ( + properties_to_remove = tuple(self.BASIC_DOCUMENT_PROPERTIES) + ( 'metadata.storagePolicy',) self._test_missing_required_sections(document, properties_to_remove) def test_privatekey_missing_required_sections(self): document = self._read_data('sample_private_key') - properties_to_remove = tuple(self.BASIC_PROPERTIES) + ( + properties_to_remove = tuple(self.BASIC_DOCUMENT_PROPERTIES) + ( 'metadata.storagePolicy',) self._test_missing_required_sections(document, properties_to_remove) def test_publickey_missing_required_sections(self): document = self._read_data('sample_public_key') - properties_to_remove = tuple(self.BASIC_PROPERTIES) + ( + properties_to_remove = tuple(self.BASIC_DOCUMENT_PROPERTIES) + ( 'metadata.storagePolicy',) self._test_missing_required_sections(document, properties_to_remove) def test_validation_policy_missing_required_sections(self): document = self._read_data('sample_validation_policy') - properties_to_remove = tuple(self.BASIC_PROPERTIES) + ( + properties_to_remove = tuple(self.BASIC_CONTROL_PROPERTIES) + ( 'data.validations', 'data.validations.0.name') self._test_missing_required_sections(document, properties_to_remove) diff --git a/deckhand/tests/unit/resources/sample_document.yaml b/deckhand/tests/unit/resources/sample_document.yaml index 42ac33b4..073c5d62 100644 --- a/deckhand/tests/unit/resources/sample_document.yaml +++ b/deckhand/tests/unit/resources/sample_document.yaml @@ -4,6 +4,7 @@ schema: promenade/ResourceType/v1 metadata: schema: metadata/Document/v1 name: a-unique-config-name-12345 + storagePolicy: cleartext labels: component: apiserver hostname: server0 diff --git a/deckhand/tests/unit/resources/sample_document_simple.yaml b/deckhand/tests/unit/resources/sample_document_simple.yaml index d688d9c4..49419153 100644 --- a/deckhand/tests/unit/resources/sample_document_simple.yaml +++ b/deckhand/tests/unit/resources/sample_document_simple.yaml @@ -3,6 +3,7 @@ schema: armada/Manifest/v1 metadata: schema: metadata/Document/v1 name: cluster-bootstrap + storagePolicy: cleartext layeringDefinition: abstract: false layer: site @@ -14,4 +15,4 @@ data: - dns - kubernetes - kubernetes-rbac -... \ No newline at end of file +... diff --git a/deckhand/types.py b/deckhand/types.py index c7c8a055..d2d8fe18 100644 --- a/deckhand/types.py +++ b/deckhand/types.py @@ -59,3 +59,12 @@ ENCRYPTION_TYPES = ( 'cleartext', 'encrypted', ) + + +METADATA_SCHEMA_TYPES = ( + CONTROL, + DOCUMENT +) = ( + 'metadata/Control', + 'metadata/Document' +) diff --git a/doc/source/images/architecture-pegleg.png b/doc/source/images/architecture-pegleg.png index ca821a8f..ace94678 100644 Binary files a/doc/source/images/architecture-pegleg.png and b/doc/source/images/architecture-pegleg.png differ diff --git a/doc/source/images/architecture.png b/doc/source/images/architecture.png index d82baa0b..a3457bb0 100644 Binary files a/doc/source/images/architecture.png and b/doc/source/images/architecture.png differ diff --git a/doc/source/validation.rst b/doc/source/validation.rst index 165b95e1..569a059d 100644 --- a/doc/source/validation.rst +++ b/doc/source/validation.rst @@ -218,9 +218,41 @@ Base Schema This schema is used to sanity-check all documents that are passed to Deckhand. Failure to pass this schema results in a critical error. +Metadata Schemas +---------------- + +Metadata schemas validate the ``metadata`` section of every document +ingested by Deckhand. + +* ``Metadata Control`` schema. + + JSON schema against which the metadata section of each ``metadata/Control`` + document type is validated. Applies to all static documents meant to + configure Deckhand behavior, like LayeringPolicy, ValidationPolicy, + and DataSchema documents. + + .. literalinclude:: ../../deckhand/engine/schemas/metadata_control.yaml + :language: yaml + :lines: 15- + :caption: Schema for ``metadata/Control`` metadata document sections. + +* ``Metadata Document`` schema. + + JSON schema against which the metadata section of each ``metadata/Document`` + document type is validated. Applies to all site definition documents or + "regular" documents that require rendering. + + .. literalinclude:: ../../deckhand/engine/schemas/metadata_document.yaml + :language: yaml + :lines: 15- + :caption: Schema for ``metadata/Document`` metadata document sections. + DataSchema Schemas ------------------ +DataSchema schemas validate the ``data`` section of every document ingested +by Deckhand. + All schemas below are ``DataSchema`` documents. They define additional properties not included in the base schema or override default properties in the base schema. @@ -284,19 +316,6 @@ corresponding to the created revision. This schema is used to sanity-check all ``Certificate`` documents that are passed to Deckhand. -* ``DataSchema`` schema. - - JSON schema against which all documents with ``deckhand/DataSchema/v1`` - schema are validated. - - .. literalinclude:: ../../deckhand/engine/schemas/dataschema_schema.yaml - :language: yaml - :lines: 15- - :caption: Schema for ``DataSchema`` documents. - - This schema is used to sanity-check all ``DataSchema`` documents that are - passed to Deckhand. - * ``LayeringPolicy`` schema. JSON schema against which all documents with ``deckhand/LayeringPolicy/v1``