diff --git a/deckhand/db/sqlalchemy/api.py b/deckhand/db/sqlalchemy/api.py index f1dc69b9..da440773 100644 --- a/deckhand/db/sqlalchemy/api.py +++ b/deckhand/db/sqlalchemy/api.py @@ -117,8 +117,9 @@ def documents_create(documents, session=None): def document_create(values, session=None): """Create a document.""" values = values.copy() - values['doc_metadata'] = values.pop('metadata') - values['schema_version'] = values.pop('schemaVersion') + values['_metadata'] = values.pop('metadata') + print(values) + values['name'] = values['_metadata']['name'] session = session or get_session() filters = models.Document.UNIQUE_CONSTRAINTS diff --git a/deckhand/db/sqlalchemy/models.py b/deckhand/db/sqlalchemy/models.py index 8ccffe06..88f9ab2b 100644 --- a/deckhand/db/sqlalchemy/models.py +++ b/deckhand/db/sqlalchemy/models.py @@ -133,18 +133,18 @@ class Revision(BASE, DeckhandBase): class Document(BASE, DeckhandBase): - UNIQUE_CONSTRAINTS = ('schema_version', 'kind', 'revision_id') + UNIQUE_CONSTRAINTS = ('schema', 'name', 'revision_id') __tablename__ = 'documents' __table_args__ = (DeckhandBase.gen_unqiue_contraint(*UNIQUE_CONSTRAINTS),) id = Column(String(36), primary_key=True, default=lambda: str(uuid.uuid4())) - schema_version = Column(String(64), nullable=False) - kind = Column(String(64), nullable=False) + schema = Column(String(64), nullable=False) + name = Column(String(64), nullable=False) # NOTE: Do not define a maximum length for these JSON data below. However, # this approach is not compatible with all database types. # "metadata" is reserved, so use "doc_metadata" instead. - doc_metadata = Column(JSONEncodedDict(), nullable=False) + _metadata = Column(JSONEncodedDict(), nullable=False) data = Column(JSONEncodedDict(), nullable=False) revision_id = Column(Integer, ForeignKey('revisions.id'), nullable=False) diff --git a/deckhand/engine/document_validation.py b/deckhand/engine/document_validation.py index ea7947c1..8c6e1abb 100644 --- a/deckhand/engine/document_validation.py +++ b/deckhand/engine/document_validation.py @@ -44,9 +44,8 @@ class DocumentValidation(object): schema_versions_info = [{'version': 'v1', 'kind': 'default', 'schema': default_schema}] - def __init__(self, schema_version, kind): + def __init__(self, schema_version): self.schema_version = schema_version - self.kind = kind @property def schema(self): @@ -60,21 +59,15 @@ class DocumentValidation(object): # TODO(fm577c): Query Deckhand API to validate "src" values. - @property - def doc_name(self): - return (self.data['schemaVersion'] + self.data['kind'] + - self.data['metadata']['name']) - def _validate_with_schema(self): # Validate the document using the schema defined by the document's # `schemaVersion` and `kind`. try: - schema_version = self.data['schemaVersion'].split('/')[-1] - doc_kind = self.data['kind'] - doc_schema_version = self.SchemaVersion(schema_version, doc_kind) + schema_version = self.data['schema'].split('/')[-1] + doc_schema_version = self.SchemaVersion(schema_version) except (AttributeError, IndexError, KeyError) as e: raise errors.InvalidFormat( - 'The provided schemaVersion is invalid or missing. Exception: ' + 'The provided schema is invalid or missing. Exception: ' '%s.' % e) try: jsonschema.validate(self.data, doc_schema_version.schema) diff --git a/deckhand/engine/schema/v1_0/default_schema.py b/deckhand/engine/schema/v1_0/default_schema.py index 7d888223..7c2e13fe 100644 --- a/deckhand/engine/schema/v1_0/default_schema.py +++ b/deckhand/engine/schema/v1_0/default_schema.py @@ -18,8 +18,7 @@ substitution_schema = { 'dest': { 'type': 'object', 'properties': { - 'path': {'type': 'string'}, - 'replacePattern': {'type': 'string'} + 'path': {'type': 'string'} }, 'additionalProperties': False, # 'replacePattern' is not required. @@ -28,12 +27,12 @@ substitution_schema = { 'src': { 'type': 'object', 'properties': { - 'kind': {'type': 'string'}, + 'schema': {'type': 'string'}, 'name': {'type': 'string'}, 'path': {'type': 'string'} }, 'additionalProperties': False, - 'required': ['kind', 'name', 'path'] + 'required': ['schema', 'name', 'path'] } }, 'additionalProperties': False, @@ -43,45 +42,46 @@ substitution_schema = { schema = { 'type': 'object', 'properties': { - 'schemaVersion': { + 'schema': { 'type': 'string', - 'pattern': '^([A-Za-z]+\/v[0-9]{1})$' + 'pattern': '^(.*\/v[0-9]{1})$' }, - # TODO: The kind should be an enum. - 'kind': {'type': 'string'}, 'metadata': { 'type': 'object', 'properties': { - 'metadataVersion': { + 'schema': { 'type': 'string', - 'pattern': '^([A-Za-z]+\/v[0-9]{1})$' + 'pattern': '^(.*/v[0-9]{1})$' }, 'name': {'type': 'string'}, + 'storagePolicy': {'type': 'string'}, 'labels': { - 'type': 'object', - 'properties': { - 'component': {'type': 'string'}, - 'hostname': {'type': 'string'} - }, - 'additionalProperties': False, - 'required': ['component', 'hostname'] + 'type': 'object' }, - 'layerDefinition': { + 'layeringDefinition': { 'type': 'object', 'properties': { - 'layer': {'enum': ['global', 'region', 'site']}, + 'layer': {'type': 'string'}, 'abstract': {'type': 'boolean'}, - 'childSelector': { - 'type': 'object', - 'properties': { - 'label': {'type': 'string'} - }, - 'additionalProperties': False, - 'required': ['label'] + 'parentSelector': { + 'type': 'object' + }, + 'actions': { + 'type': 'array', + 'items': { + 'type': 'object', + 'properties': { + 'method': {'enum': ['merge', 'delete', + 'replace']}, + 'path': {'type': 'string'} + }, + 'additionalProperties': False, + 'required': ['method', 'path'] + } } }, 'additionalProperties': False, - 'required': ['layer', 'abstract', 'childSelector'] + 'required': ['layer', 'abstract', 'parentSelector'] }, 'substitutions': { 'type': 'array', @@ -89,13 +89,13 @@ schema = { } }, 'additionalProperties': False, - 'required': ['metadataVersion', 'name', 'labels', - 'layerDefinition', 'substitutions'] + 'required': ['schema', 'name', 'storagePolicy', 'labels', + 'layeringDefinition', 'substitutions'] }, 'data': { 'type': 'object' } }, 'additionalProperties': False, - 'required': ['schemaVersion', 'kind', 'metadata', 'data'] + 'required': ['schema', 'metadata', 'data'] } diff --git a/deckhand/engine/secret_substitution.py b/deckhand/engine/secret_substitution.py deleted file mode 100644 index 38229547..00000000 --- a/deckhand/engine/secret_substitution.py +++ /dev/null @@ -1,138 +0,0 @@ -# Copyright 2017 AT&T Intellectual Property. All other rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import yaml - -import jsonschema - -from deckhand.engine.schema.v1_0 import default_schema -from deckhand import errors - - -class SecretSubstitution(object): - """Class for secret substitution logic for YAML files. - - This class is responsible for parsing, validating and retrieving secret - values for values stored in the YAML file. Afterward, secret values will be - substituted or "forward-repalced" into the YAML file. The end result is a - YAML file containing all necessary secrets to be handed off to other - services. - - :param data: YAML data that requires secrets to be validated, merged and - consolidated. - """ - - def __init__(self, data): - try: - self.data = yaml.safe_load(data) - except yaml.YAMLError: - raise errors.InvalidFormat( - 'The provided YAML file cannot be parsed.') - - self.pre_validate_data() - - class SchemaVersion(object): - """Class for retrieving correct schema for pre-validation on YAML. - - Retrieves the schema that corresponds to "apiVersion" in the YAML - data. This schema is responsible for performing pre-validation on - YAML data. - """ - - # TODO: Update kind according to requirements. - schema_versions_info = [{'version': 'v1', 'kind': 'default', - 'schema': default_schema}] - - def __init__(self, schema_version, kind): - self.schema_version = schema_version - self.kind = kind - - @property - def schema(self): - # TODO: return schema based on version and kind. - return [v['schema'] for v in self.schema_versions_info - if v['version'] == self.schema_version][0].schema - - def pre_validate_data(self): - """Pre-validate that the YAML file is correctly formatted.""" - self._validate_with_schema() - - # Validate that each "dest" field exists in the YAML data. - # FIXME(fm577c): Dest fields will be injected if not present - the - # validation below needs to be updated or removed. - substitutions = self.data['metadata']['substitutions'] - destinations = [s['dest'] for s in substitutions] - sub_data = self.data['data'] - - for dest in destinations: - result, missing_attr = self._multi_getattr(dest['path'], sub_data) - if not result: - raise errors.InvalidFormat( - 'The attribute "%s" included in the "dest" field "%s" is ' - 'missing from the YAML data: "%s".' % ( - missing_attr, dest, sub_data)) - - # TODO(fm577c): Query Deckhand API to validate "src" values. - - def _validate_with_schema(self): - # Validate the document using the schema defined by the document's - # `schemaVersion` and `kind`. - try: - schema_version = self.data['schemaVersion'].split('/')[-1] - doc_kind = self.data['kind'] - doc_schema_version = self.SchemaVersion(schema_version, doc_kind) - except (AttributeError, IndexError, KeyError) as e: - raise errors.InvalidFormat( - 'The provided schemaVersion is invalid or missing. Exception: ' - '%s.' % e) - try: - jsonschema.validate(self.data, doc_schema_version.schema) - except jsonschema.exceptions.ValidationError as e: - raise errors.InvalidFormat('The provided YAML file is invalid. ' - 'Exception: %s.' % e.message) - - def _multi_getattr(self, multi_key, substitutable_data): - """Iteratively check for nested attributes in the YAML data. - - Check for nested attributes included in "dest" attributes in the data - section of the YAML file. For example, a "dest" attribute of - ".foo.bar.baz" should mean that the YAML data adheres to: - - .. code-block:: yaml - - --- - foo: - bar: - baz: - - :param multi_key: A multi-part key that references nested data in the - substitutable part of the YAML data, e.g. ".foo.bar.baz". - :param substitutable_data: The section of data in the YAML data that - is intended to be substituted with secrets. - :returns: Tuple where first value is a boolean indicating that the - nested attribute was found and the second value is the attribute - that was not found, if applicable. - """ - attrs = multi_key.split('.') - # Ignore the first attribute if it is "." as that is a self-reference. - if attrs[0] == '': - attrs = attrs[1:] - - data = substitutable_data - for attr in attrs: - if attr not in data: - return False, attr - data = data.get(attr) - - return True, None diff --git a/deckhand/tests/unit/db/test_documents.py b/deckhand/tests/unit/db/test_documents.py index 424b25dc..97a6777a 100644 --- a/deckhand/tests/unit/db/test_documents.py +++ b/deckhand/tests/unit/db/test_documents.py @@ -26,9 +26,8 @@ class DocumentFixture(object): def get_minimal_fixture(self, **kwargs): fixture = {'data': 'fake document data', - 'metadata': 'fake metadata', - 'kind': 'FakeConfigType', - 'schemaVersion': 'deckhand/v1'} + 'metadata': {'name': 'fake metadata'}, + 'schema': 'deckhand/v1'} fixture.update(kwargs) return fixture @@ -36,8 +35,7 @@ class DocumentFixture(object): class TestDocumentsApi(base.DeckhandWithDBTestCase): def _validate_document(self, expected, actual): - expected['doc_metadata'] = expected.pop('metadata') - expected['schema_version'] = expected.pop('schemaVersion') + expected['_metadata'] = expected.pop('metadata') # TODO: Validate "status" fields, like created_at. self.assertIsInstance(actual, dict) @@ -74,13 +72,13 @@ class TestDocumentsApi(base.DeckhandWithDBTestCase): fixture = DocumentFixture().get_minimal_fixture() child_document = db_api.document_create(fixture) - fixture['metadata'] = 'modified fake metadata' + fixture['metadata'] = {'name': 'modified fake metadata'} parent_document = db_api.document_create(fixture) self._validate_document(fixture, parent_document) # Validate that the new document was created. - self.assertEqual('modified fake metadata', - parent_document['doc_metadata']) + self.assertEqual({'name': 'modified fake metadata'}, + parent_document['_metadata']) self.assertNotEqual(child_document['id'], parent_document['id']) # Validate that the parent document has a different revision and diff --git a/deckhand/tests/unit/engine/test_document_validation.py b/deckhand/tests/unit/engine/test_document_validation.py index 0f0e5fb1..835b531e 100644 --- a/deckhand/tests/unit/engine/test_document_validation.py +++ b/deckhand/tests/unit/engine/test_document_validation.py @@ -81,8 +81,7 @@ class TestDocumentValidation(testtools.TestCase): invalid_data = [ (self._corrupt_data('data'), 'data'), (self._corrupt_data('metadata'), 'metadata'), - (self._corrupt_data('metadata.metadataVersion'), - 'metadataVersion'), + (self._corrupt_data('metadata.schema'), 'schema'), (self._corrupt_data('metadata.name'), 'name'), (self._corrupt_data('metadata.substitutions'), 'substitutions'), (self._corrupt_data('metadata.substitutions.0.dest'), 'dest'), diff --git a/deckhand/tests/unit/engine/test_secret_substitution.py b/deckhand/tests/unit/engine/test_secret_substitution.py deleted file mode 100644 index df017948..00000000 --- a/deckhand/tests/unit/engine/test_secret_substitution.py +++ /dev/null @@ -1,123 +0,0 @@ -# Copyright 2017 AT&T Intellectual Property. All other rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import copy -import os -import testtools -import yaml - -import six - -from deckhand.engine import secret_substitution -from deckhand import errors - - -class TestSecretSubtitution(testtools.TestCase): - - def setUp(self): - super(TestSecretSubtitution, self).setUp() - dir_path = os.path.dirname(os.path.realpath(__file__)) - test_yaml_path = os.path.abspath(os.path.join( - dir_path, os.pardir, 'resources', 'sample.yaml')) - - with open(test_yaml_path, 'r') as yaml_file: - yaml_data = yaml_file.read() - self.data = yaml.safe_load(yaml_data) - - def _corrupt_data(self, key, data=None): - """Corrupt test data to check that pre-validation works. - - Corrupt data by removing a key from the document. Each key must - correspond to a value that is a dictionary. - - :param key: The document key to be removed. The key can have the - following formats: - * 'data' => document.pop('data') - * 'metadata.name' => document['metadata'].pop('name') - * 'metadata.substitutions.0.dest' => - document['metadata']['substitutions'][0].pop('dest') - :returns: Corrupted YAML data. - """ - if data is None: - data = self.data - corrupted_data = copy.deepcopy(data) - - if '.' in key: - _corrupted_data = corrupted_data - nested_keys = key.split('.') - for nested_key in nested_keys: - if nested_key == nested_keys[-1]: - break - if nested_key.isdigit(): - _corrupted_data = _corrupted_data[int(nested_key)] - else: - _corrupted_data = _corrupted_data[nested_key] - _corrupted_data.pop(nested_keys[-1]) - else: - corrupted_data.pop(key) - - return self._format_data(corrupted_data) - - def _format_data(self, data=None): - """Re-formats dict data as YAML to pass to ``SecretSubstitution``.""" - if data is None: - data = self.data - return yaml.safe_dump(data) - - def test_initialization(self): - sub = secret_substitution.SecretSubstitution(self._format_data()) - self.assertIsInstance(sub, secret_substitution.SecretSubstitution) - - def test_initialization_missing_sections(self): - expected_err = ("The provided YAML file is invalid. Exception: '%s' " - "is a required property.") - invalid_data = [ - (self._corrupt_data('data'), 'data'), - (self._corrupt_data('metadata'), 'metadata'), - (self._corrupt_data('metadata.metadataVersion'), 'metadataVersion'), - (self._corrupt_data('metadata.name'), 'name'), - (self._corrupt_data('metadata.substitutions'), 'substitutions'), - (self._corrupt_data('metadata.substitutions.0.dest'), 'dest'), - (self._corrupt_data('metadata.substitutions.0.src'), 'src') - ] - - for invalid_entry, missing_key in invalid_data: - with six.assertRaisesRegex(self, errors.InvalidFormat, - expected_err % missing_key): - secret_substitution.SecretSubstitution(invalid_entry) - - def test_initialization_bad_substitutions(self): - expected_err = ('The attribute "%s" included in the "dest" field "%s" ' - 'is missing from the YAML data') - invalid_data = [] - - data = copy.deepcopy(self.data) - data['metadata']['substitutions'][0]['dest'] = {'path': 'foo'} - invalid_data.append(self._format_data(data)) - - data = copy.deepcopy(self.data) - data['metadata']['substitutions'][0]['dest'] = { - 'path': 'tls_endpoint.bar'} - invalid_data.append(self._format_data(data)) - - def _test(invalid_entry, field, dest): - _expected_err = expected_err % (field, dest) - with six.assertRaisesRegex(self, errors.InvalidFormat, - _expected_err): - secret_substitution.SecretSubstitution(invalid_entry) - - # Verify that invalid body dest reference is invalid. - _test(invalid_data[0], "foo", {'path': 'foo'}) - # Verify that nested invalid body dest reference is invalid. - _test(invalid_data[1], "bar", {'path': 'tls_endpoint.bar'}) diff --git a/deckhand/tests/unit/resources/sample.yaml b/deckhand/tests/unit/resources/sample.yaml index ec083907..d1e83e0d 100644 --- a/deckhand/tests/unit/resources/sample.yaml +++ b/deckhand/tests/unit/resources/sample.yaml @@ -1,33 +1,38 @@ -# Sample YAML file for testing forward replacement. --- -schemaVersion: promenade/v1 -kind: SomeConfigType +schema: some-service/ResourceType/v1 metadata: - metadataVersion: deckhand/v1 - name: a-unique-config-name-12345 + schema: metadata/Document/v1 + name: unique-name-given-schema + storagePolicy: cleartext labels: - component: apiserver - hostname: server0 - layerDefinition: - layer: global - abstract: True - childSelector: - label: value + genesis: enabled + master: enabled + layeringDefinition: + abstract: true + layer: region + parentSelector: + required_key_a: required_label_a + required_key_b: required_label_b + actions: + - method: merge + path: .path.to.merge.into.parent + - method: delete + path: .path.to.delete substitutions: - dest: - path: .tls_endpoint.certificate - replacePattern: 'test.pattern' + path: .substitution.target src: - kind: Certificate - name: some-certificate-asdf-1234 - path: .cert - - dest: - path: .tls_endpoint.key - src: - kind: CertificateKey - name: some-certificate-asdf-1234 - path: .key + schema: another-service/SourceType/v1 + name: name-of-source-document + path: .source.path data: - tls_endpoint: - certificate: '.cert' - key: deckhand/v1:some-certificate-asdf-1234 + path: + to: + merge: + into: + parent: + foo: bar + ignored: # Will not be part of the resultant document after layering. + data: here + substitution: + target: null # Paths do not need to exist to be specified as substitution destinations.