Update schema/db model/db api to align with design document.
This commit is contained in:
parent
dbc80fbfae
commit
7f6788db89
|
@ -117,8 +117,9 @@ def documents_create(documents, session=None):
|
|||
def document_create(values, session=None):
|
||||
"""Create a document."""
|
||||
values = values.copy()
|
||||
values['doc_metadata'] = values.pop('metadata')
|
||||
values['schema_version'] = values.pop('schemaVersion')
|
||||
values['_metadata'] = values.pop('metadata')
|
||||
print(values)
|
||||
values['name'] = values['_metadata']['name']
|
||||
session = session or get_session()
|
||||
|
||||
filters = models.Document.UNIQUE_CONSTRAINTS
|
||||
|
|
|
@ -133,18 +133,18 @@ class Revision(BASE, DeckhandBase):
|
|||
|
||||
|
||||
class Document(BASE, DeckhandBase):
|
||||
UNIQUE_CONSTRAINTS = ('schema_version', 'kind', 'revision_id')
|
||||
UNIQUE_CONSTRAINTS = ('schema', 'name', 'revision_id')
|
||||
__tablename__ = 'documents'
|
||||
__table_args__ = (DeckhandBase.gen_unqiue_contraint(*UNIQUE_CONSTRAINTS),)
|
||||
|
||||
id = Column(String(36), primary_key=True,
|
||||
default=lambda: str(uuid.uuid4()))
|
||||
schema_version = Column(String(64), nullable=False)
|
||||
kind = Column(String(64), nullable=False)
|
||||
schema = Column(String(64), nullable=False)
|
||||
name = Column(String(64), nullable=False)
|
||||
# NOTE: Do not define a maximum length for these JSON data below. However,
|
||||
# this approach is not compatible with all database types.
|
||||
# "metadata" is reserved, so use "doc_metadata" instead.
|
||||
doc_metadata = Column(JSONEncodedDict(), nullable=False)
|
||||
_metadata = Column(JSONEncodedDict(), nullable=False)
|
||||
data = Column(JSONEncodedDict(), nullable=False)
|
||||
revision_id = Column(Integer, ForeignKey('revisions.id'), nullable=False)
|
||||
|
||||
|
|
|
@ -44,9 +44,8 @@ class DocumentValidation(object):
|
|||
schema_versions_info = [{'version': 'v1', 'kind': 'default',
|
||||
'schema': default_schema}]
|
||||
|
||||
def __init__(self, schema_version, kind):
|
||||
def __init__(self, schema_version):
|
||||
self.schema_version = schema_version
|
||||
self.kind = kind
|
||||
|
||||
@property
|
||||
def schema(self):
|
||||
|
@ -60,21 +59,15 @@ class DocumentValidation(object):
|
|||
|
||||
# TODO(fm577c): Query Deckhand API to validate "src" values.
|
||||
|
||||
@property
|
||||
def doc_name(self):
|
||||
return (self.data['schemaVersion'] + self.data['kind'] +
|
||||
self.data['metadata']['name'])
|
||||
|
||||
def _validate_with_schema(self):
|
||||
# Validate the document using the schema defined by the document's
|
||||
# `schemaVersion` and `kind`.
|
||||
try:
|
||||
schema_version = self.data['schemaVersion'].split('/')[-1]
|
||||
doc_kind = self.data['kind']
|
||||
doc_schema_version = self.SchemaVersion(schema_version, doc_kind)
|
||||
schema_version = self.data['schema'].split('/')[-1]
|
||||
doc_schema_version = self.SchemaVersion(schema_version)
|
||||
except (AttributeError, IndexError, KeyError) as e:
|
||||
raise errors.InvalidFormat(
|
||||
'The provided schemaVersion is invalid or missing. Exception: '
|
||||
'The provided schema is invalid or missing. Exception: '
|
||||
'%s.' % e)
|
||||
try:
|
||||
jsonschema.validate(self.data, doc_schema_version.schema)
|
||||
|
|
|
@ -18,8 +18,7 @@ substitution_schema = {
|
|||
'dest': {
|
||||
'type': 'object',
|
||||
'properties': {
|
||||
'path': {'type': 'string'},
|
||||
'replacePattern': {'type': 'string'}
|
||||
'path': {'type': 'string'}
|
||||
},
|
||||
'additionalProperties': False,
|
||||
# 'replacePattern' is not required.
|
||||
|
@ -28,12 +27,12 @@ substitution_schema = {
|
|||
'src': {
|
||||
'type': 'object',
|
||||
'properties': {
|
||||
'kind': {'type': 'string'},
|
||||
'schema': {'type': 'string'},
|
||||
'name': {'type': 'string'},
|
||||
'path': {'type': 'string'}
|
||||
},
|
||||
'additionalProperties': False,
|
||||
'required': ['kind', 'name', 'path']
|
||||
'required': ['schema', 'name', 'path']
|
||||
}
|
||||
},
|
||||
'additionalProperties': False,
|
||||
|
@ -43,45 +42,46 @@ substitution_schema = {
|
|||
schema = {
|
||||
'type': 'object',
|
||||
'properties': {
|
||||
'schemaVersion': {
|
||||
'schema': {
|
||||
'type': 'string',
|
||||
'pattern': '^([A-Za-z]+\/v[0-9]{1})$'
|
||||
'pattern': '^(.*\/v[0-9]{1})$'
|
||||
},
|
||||
# TODO: The kind should be an enum.
|
||||
'kind': {'type': 'string'},
|
||||
'metadata': {
|
||||
'type': 'object',
|
||||
'properties': {
|
||||
'metadataVersion': {
|
||||
'schema': {
|
||||
'type': 'string',
|
||||
'pattern': '^([A-Za-z]+\/v[0-9]{1})$'
|
||||
'pattern': '^(.*/v[0-9]{1})$'
|
||||
},
|
||||
'name': {'type': 'string'},
|
||||
'storagePolicy': {'type': 'string'},
|
||||
'labels': {
|
||||
'type': 'object',
|
||||
'properties': {
|
||||
'component': {'type': 'string'},
|
||||
'hostname': {'type': 'string'}
|
||||
},
|
||||
'additionalProperties': False,
|
||||
'required': ['component', 'hostname']
|
||||
'type': 'object'
|
||||
},
|
||||
'layerDefinition': {
|
||||
'layeringDefinition': {
|
||||
'type': 'object',
|
||||
'properties': {
|
||||
'layer': {'enum': ['global', 'region', 'site']},
|
||||
'layer': {'type': 'string'},
|
||||
'abstract': {'type': 'boolean'},
|
||||
'childSelector': {
|
||||
'type': 'object',
|
||||
'properties': {
|
||||
'label': {'type': 'string'}
|
||||
},
|
||||
'additionalProperties': False,
|
||||
'required': ['label']
|
||||
'parentSelector': {
|
||||
'type': 'object'
|
||||
},
|
||||
'actions': {
|
||||
'type': 'array',
|
||||
'items': {
|
||||
'type': 'object',
|
||||
'properties': {
|
||||
'method': {'enum': ['merge', 'delete',
|
||||
'replace']},
|
||||
'path': {'type': 'string'}
|
||||
},
|
||||
'additionalProperties': False,
|
||||
'required': ['method', 'path']
|
||||
}
|
||||
}
|
||||
},
|
||||
'additionalProperties': False,
|
||||
'required': ['layer', 'abstract', 'childSelector']
|
||||
'required': ['layer', 'abstract', 'parentSelector']
|
||||
},
|
||||
'substitutions': {
|
||||
'type': 'array',
|
||||
|
@ -89,13 +89,13 @@ schema = {
|
|||
}
|
||||
},
|
||||
'additionalProperties': False,
|
||||
'required': ['metadataVersion', 'name', 'labels',
|
||||
'layerDefinition', 'substitutions']
|
||||
'required': ['schema', 'name', 'storagePolicy', 'labels',
|
||||
'layeringDefinition', 'substitutions']
|
||||
},
|
||||
'data': {
|
||||
'type': 'object'
|
||||
}
|
||||
},
|
||||
'additionalProperties': False,
|
||||
'required': ['schemaVersion', 'kind', 'metadata', 'data']
|
||||
'required': ['schema', 'metadata', 'data']
|
||||
}
|
||||
|
|
|
@ -1,138 +0,0 @@
|
|||
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import yaml
|
||||
|
||||
import jsonschema
|
||||
|
||||
from deckhand.engine.schema.v1_0 import default_schema
|
||||
from deckhand import errors
|
||||
|
||||
|
||||
class SecretSubstitution(object):
|
||||
"""Class for secret substitution logic for YAML files.
|
||||
|
||||
This class is responsible for parsing, validating and retrieving secret
|
||||
values for values stored in the YAML file. Afterward, secret values will be
|
||||
substituted or "forward-repalced" into the YAML file. The end result is a
|
||||
YAML file containing all necessary secrets to be handed off to other
|
||||
services.
|
||||
|
||||
:param data: YAML data that requires secrets to be validated, merged and
|
||||
consolidated.
|
||||
"""
|
||||
|
||||
def __init__(self, data):
|
||||
try:
|
||||
self.data = yaml.safe_load(data)
|
||||
except yaml.YAMLError:
|
||||
raise errors.InvalidFormat(
|
||||
'The provided YAML file cannot be parsed.')
|
||||
|
||||
self.pre_validate_data()
|
||||
|
||||
class SchemaVersion(object):
|
||||
"""Class for retrieving correct schema for pre-validation on YAML.
|
||||
|
||||
Retrieves the schema that corresponds to "apiVersion" in the YAML
|
||||
data. This schema is responsible for performing pre-validation on
|
||||
YAML data.
|
||||
"""
|
||||
|
||||
# TODO: Update kind according to requirements.
|
||||
schema_versions_info = [{'version': 'v1', 'kind': 'default',
|
||||
'schema': default_schema}]
|
||||
|
||||
def __init__(self, schema_version, kind):
|
||||
self.schema_version = schema_version
|
||||
self.kind = kind
|
||||
|
||||
@property
|
||||
def schema(self):
|
||||
# TODO: return schema based on version and kind.
|
||||
return [v['schema'] for v in self.schema_versions_info
|
||||
if v['version'] == self.schema_version][0].schema
|
||||
|
||||
def pre_validate_data(self):
|
||||
"""Pre-validate that the YAML file is correctly formatted."""
|
||||
self._validate_with_schema()
|
||||
|
||||
# Validate that each "dest" field exists in the YAML data.
|
||||
# FIXME(fm577c): Dest fields will be injected if not present - the
|
||||
# validation below needs to be updated or removed.
|
||||
substitutions = self.data['metadata']['substitutions']
|
||||
destinations = [s['dest'] for s in substitutions]
|
||||
sub_data = self.data['data']
|
||||
|
||||
for dest in destinations:
|
||||
result, missing_attr = self._multi_getattr(dest['path'], sub_data)
|
||||
if not result:
|
||||
raise errors.InvalidFormat(
|
||||
'The attribute "%s" included in the "dest" field "%s" is '
|
||||
'missing from the YAML data: "%s".' % (
|
||||
missing_attr, dest, sub_data))
|
||||
|
||||
# TODO(fm577c): Query Deckhand API to validate "src" values.
|
||||
|
||||
def _validate_with_schema(self):
|
||||
# Validate the document using the schema defined by the document's
|
||||
# `schemaVersion` and `kind`.
|
||||
try:
|
||||
schema_version = self.data['schemaVersion'].split('/')[-1]
|
||||
doc_kind = self.data['kind']
|
||||
doc_schema_version = self.SchemaVersion(schema_version, doc_kind)
|
||||
except (AttributeError, IndexError, KeyError) as e:
|
||||
raise errors.InvalidFormat(
|
||||
'The provided schemaVersion is invalid or missing. Exception: '
|
||||
'%s.' % e)
|
||||
try:
|
||||
jsonschema.validate(self.data, doc_schema_version.schema)
|
||||
except jsonschema.exceptions.ValidationError as e:
|
||||
raise errors.InvalidFormat('The provided YAML file is invalid. '
|
||||
'Exception: %s.' % e.message)
|
||||
|
||||
def _multi_getattr(self, multi_key, substitutable_data):
|
||||
"""Iteratively check for nested attributes in the YAML data.
|
||||
|
||||
Check for nested attributes included in "dest" attributes in the data
|
||||
section of the YAML file. For example, a "dest" attribute of
|
||||
".foo.bar.baz" should mean that the YAML data adheres to:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
---
|
||||
foo:
|
||||
bar:
|
||||
baz: <data_to_be_substituted_here>
|
||||
|
||||
:param multi_key: A multi-part key that references nested data in the
|
||||
substitutable part of the YAML data, e.g. ".foo.bar.baz".
|
||||
:param substitutable_data: The section of data in the YAML data that
|
||||
is intended to be substituted with secrets.
|
||||
:returns: Tuple where first value is a boolean indicating that the
|
||||
nested attribute was found and the second value is the attribute
|
||||
that was not found, if applicable.
|
||||
"""
|
||||
attrs = multi_key.split('.')
|
||||
# Ignore the first attribute if it is "." as that is a self-reference.
|
||||
if attrs[0] == '':
|
||||
attrs = attrs[1:]
|
||||
|
||||
data = substitutable_data
|
||||
for attr in attrs:
|
||||
if attr not in data:
|
||||
return False, attr
|
||||
data = data.get(attr)
|
||||
|
||||
return True, None
|
|
@ -26,9 +26,8 @@ class DocumentFixture(object):
|
|||
|
||||
def get_minimal_fixture(self, **kwargs):
|
||||
fixture = {'data': 'fake document data',
|
||||
'metadata': 'fake metadata',
|
||||
'kind': 'FakeConfigType',
|
||||
'schemaVersion': 'deckhand/v1'}
|
||||
'metadata': {'name': 'fake metadata'},
|
||||
'schema': 'deckhand/v1'}
|
||||
fixture.update(kwargs)
|
||||
return fixture
|
||||
|
||||
|
@ -36,8 +35,7 @@ class DocumentFixture(object):
|
|||
class TestDocumentsApi(base.DeckhandWithDBTestCase):
|
||||
|
||||
def _validate_document(self, expected, actual):
|
||||
expected['doc_metadata'] = expected.pop('metadata')
|
||||
expected['schema_version'] = expected.pop('schemaVersion')
|
||||
expected['_metadata'] = expected.pop('metadata')
|
||||
|
||||
# TODO: Validate "status" fields, like created_at.
|
||||
self.assertIsInstance(actual, dict)
|
||||
|
@ -74,13 +72,13 @@ class TestDocumentsApi(base.DeckhandWithDBTestCase):
|
|||
fixture = DocumentFixture().get_minimal_fixture()
|
||||
child_document = db_api.document_create(fixture)
|
||||
|
||||
fixture['metadata'] = 'modified fake metadata'
|
||||
fixture['metadata'] = {'name': 'modified fake metadata'}
|
||||
parent_document = db_api.document_create(fixture)
|
||||
self._validate_document(fixture, parent_document)
|
||||
|
||||
# Validate that the new document was created.
|
||||
self.assertEqual('modified fake metadata',
|
||||
parent_document['doc_metadata'])
|
||||
self.assertEqual({'name': 'modified fake metadata'},
|
||||
parent_document['_metadata'])
|
||||
self.assertNotEqual(child_document['id'], parent_document['id'])
|
||||
|
||||
# Validate that the parent document has a different revision and
|
||||
|
|
|
@ -81,8 +81,7 @@ class TestDocumentValidation(testtools.TestCase):
|
|||
invalid_data = [
|
||||
(self._corrupt_data('data'), 'data'),
|
||||
(self._corrupt_data('metadata'), 'metadata'),
|
||||
(self._corrupt_data('metadata.metadataVersion'),
|
||||
'metadataVersion'),
|
||||
(self._corrupt_data('metadata.schema'), 'schema'),
|
||||
(self._corrupt_data('metadata.name'), 'name'),
|
||||
(self._corrupt_data('metadata.substitutions'), 'substitutions'),
|
||||
(self._corrupt_data('metadata.substitutions.0.dest'), 'dest'),
|
||||
|
|
|
@ -1,123 +0,0 @@
|
|||
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import copy
|
||||
import os
|
||||
import testtools
|
||||
import yaml
|
||||
|
||||
import six
|
||||
|
||||
from deckhand.engine import secret_substitution
|
||||
from deckhand import errors
|
||||
|
||||
|
||||
class TestSecretSubtitution(testtools.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
super(TestSecretSubtitution, self).setUp()
|
||||
dir_path = os.path.dirname(os.path.realpath(__file__))
|
||||
test_yaml_path = os.path.abspath(os.path.join(
|
||||
dir_path, os.pardir, 'resources', 'sample.yaml'))
|
||||
|
||||
with open(test_yaml_path, 'r') as yaml_file:
|
||||
yaml_data = yaml_file.read()
|
||||
self.data = yaml.safe_load(yaml_data)
|
||||
|
||||
def _corrupt_data(self, key, data=None):
|
||||
"""Corrupt test data to check that pre-validation works.
|
||||
|
||||
Corrupt data by removing a key from the document. Each key must
|
||||
correspond to a value that is a dictionary.
|
||||
|
||||
:param key: The document key to be removed. The key can have the
|
||||
following formats:
|
||||
* 'data' => document.pop('data')
|
||||
* 'metadata.name' => document['metadata'].pop('name')
|
||||
* 'metadata.substitutions.0.dest' =>
|
||||
document['metadata']['substitutions'][0].pop('dest')
|
||||
:returns: Corrupted YAML data.
|
||||
"""
|
||||
if data is None:
|
||||
data = self.data
|
||||
corrupted_data = copy.deepcopy(data)
|
||||
|
||||
if '.' in key:
|
||||
_corrupted_data = corrupted_data
|
||||
nested_keys = key.split('.')
|
||||
for nested_key in nested_keys:
|
||||
if nested_key == nested_keys[-1]:
|
||||
break
|
||||
if nested_key.isdigit():
|
||||
_corrupted_data = _corrupted_data[int(nested_key)]
|
||||
else:
|
||||
_corrupted_data = _corrupted_data[nested_key]
|
||||
_corrupted_data.pop(nested_keys[-1])
|
||||
else:
|
||||
corrupted_data.pop(key)
|
||||
|
||||
return self._format_data(corrupted_data)
|
||||
|
||||
def _format_data(self, data=None):
|
||||
"""Re-formats dict data as YAML to pass to ``SecretSubstitution``."""
|
||||
if data is None:
|
||||
data = self.data
|
||||
return yaml.safe_dump(data)
|
||||
|
||||
def test_initialization(self):
|
||||
sub = secret_substitution.SecretSubstitution(self._format_data())
|
||||
self.assertIsInstance(sub, secret_substitution.SecretSubstitution)
|
||||
|
||||
def test_initialization_missing_sections(self):
|
||||
expected_err = ("The provided YAML file is invalid. Exception: '%s' "
|
||||
"is a required property.")
|
||||
invalid_data = [
|
||||
(self._corrupt_data('data'), 'data'),
|
||||
(self._corrupt_data('metadata'), 'metadata'),
|
||||
(self._corrupt_data('metadata.metadataVersion'), 'metadataVersion'),
|
||||
(self._corrupt_data('metadata.name'), 'name'),
|
||||
(self._corrupt_data('metadata.substitutions'), 'substitutions'),
|
||||
(self._corrupt_data('metadata.substitutions.0.dest'), 'dest'),
|
||||
(self._corrupt_data('metadata.substitutions.0.src'), 'src')
|
||||
]
|
||||
|
||||
for invalid_entry, missing_key in invalid_data:
|
||||
with six.assertRaisesRegex(self, errors.InvalidFormat,
|
||||
expected_err % missing_key):
|
||||
secret_substitution.SecretSubstitution(invalid_entry)
|
||||
|
||||
def test_initialization_bad_substitutions(self):
|
||||
expected_err = ('The attribute "%s" included in the "dest" field "%s" '
|
||||
'is missing from the YAML data')
|
||||
invalid_data = []
|
||||
|
||||
data = copy.deepcopy(self.data)
|
||||
data['metadata']['substitutions'][0]['dest'] = {'path': 'foo'}
|
||||
invalid_data.append(self._format_data(data))
|
||||
|
||||
data = copy.deepcopy(self.data)
|
||||
data['metadata']['substitutions'][0]['dest'] = {
|
||||
'path': 'tls_endpoint.bar'}
|
||||
invalid_data.append(self._format_data(data))
|
||||
|
||||
def _test(invalid_entry, field, dest):
|
||||
_expected_err = expected_err % (field, dest)
|
||||
with six.assertRaisesRegex(self, errors.InvalidFormat,
|
||||
_expected_err):
|
||||
secret_substitution.SecretSubstitution(invalid_entry)
|
||||
|
||||
# Verify that invalid body dest reference is invalid.
|
||||
_test(invalid_data[0], "foo", {'path': 'foo'})
|
||||
# Verify that nested invalid body dest reference is invalid.
|
||||
_test(invalid_data[1], "bar", {'path': 'tls_endpoint.bar'})
|
|
@ -1,33 +1,38 @@
|
|||
# Sample YAML file for testing forward replacement.
|
||||
---
|
||||
schemaVersion: promenade/v1
|
||||
kind: SomeConfigType
|
||||
schema: some-service/ResourceType/v1
|
||||
metadata:
|
||||
metadataVersion: deckhand/v1
|
||||
name: a-unique-config-name-12345
|
||||
schema: metadata/Document/v1
|
||||
name: unique-name-given-schema
|
||||
storagePolicy: cleartext
|
||||
labels:
|
||||
component: apiserver
|
||||
hostname: server0
|
||||
layerDefinition:
|
||||
layer: global
|
||||
abstract: True
|
||||
childSelector:
|
||||
label: value
|
||||
genesis: enabled
|
||||
master: enabled
|
||||
layeringDefinition:
|
||||
abstract: true
|
||||
layer: region
|
||||
parentSelector:
|
||||
required_key_a: required_label_a
|
||||
required_key_b: required_label_b
|
||||
actions:
|
||||
- method: merge
|
||||
path: .path.to.merge.into.parent
|
||||
- method: delete
|
||||
path: .path.to.delete
|
||||
substitutions:
|
||||
- dest:
|
||||
path: .tls_endpoint.certificate
|
||||
replacePattern: 'test.pattern'
|
||||
path: .substitution.target
|
||||
src:
|
||||
kind: Certificate
|
||||
name: some-certificate-asdf-1234
|
||||
path: .cert
|
||||
- dest:
|
||||
path: .tls_endpoint.key
|
||||
src:
|
||||
kind: CertificateKey
|
||||
name: some-certificate-asdf-1234
|
||||
path: .key
|
||||
schema: another-service/SourceType/v1
|
||||
name: name-of-source-document
|
||||
path: .source.path
|
||||
data:
|
||||
tls_endpoint:
|
||||
certificate: '.cert'
|
||||
key: deckhand/v1:some-certificate-asdf-1234
|
||||
path:
|
||||
to:
|
||||
merge:
|
||||
into:
|
||||
parent:
|
||||
foo: bar
|
||||
ignored: # Will not be part of the resultant document after layering.
|
||||
data: here
|
||||
substitution:
|
||||
target: null # Paths do not need to exist to be specified as substitution destinations.
|
||||
|
|
Loading…
Reference in New Issue