Merge branch 'master' into documents-api
This commit is contained in:
Felipe Monteiro
GitHub
commit
55e1360cbf
|
|
@ -0,0 +1,138 @@
|
|||
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import yaml
|
||||
|
||||
import jsonschema
|
||||
|
||||
from deckhand.engine.schema.v1_0 import default_schema
|
||||
from deckhand import errors
|
||||
|
||||
|
||||
class SecretSubstitution(object):
|
||||
"""Class for secret substitution logic for YAML files.
|
||||
|
||||
This class is responsible for parsing, validating and retrieving secret
|
||||
values for values stored in the YAML file. Afterward, secret values will be
|
||||
substituted or "forward-repalced" into the YAML file. The end result is a
|
||||
YAML file containing all necessary secrets to be handed off to other
|
||||
services.
|
||||
|
||||
:param data: YAML data that requires secrets to be validated, merged and
|
||||
consolidated.
|
||||
"""
|
||||
|
||||
def __init__(self, data):
|
||||
try:
|
||||
self.data = yaml.safe_load(data)
|
||||
except yaml.YAMLError:
|
||||
raise errors.InvalidFormat(
|
||||
'The provided YAML file cannot be parsed.')
|
||||
|
||||
self.pre_validate_data()
|
||||
|
||||
class SchemaVersion(object):
|
||||
"""Class for retrieving correct schema for pre-validation on YAML.
|
||||
|
||||
Retrieves the schema that corresponds to "apiVersion" in the YAML
|
||||
data. This schema is responsible for performing pre-validation on
|
||||
YAML data.
|
||||
"""
|
||||
|
||||
# TODO: Update kind according to requirements.
|
||||
schema_versions_info = [{'version': 'v1', 'kind': 'default',
|
||||
'schema': default_schema}]
|
||||
|
||||
def __init__(self, schema_version, kind):
|
||||
self.schema_version = schema_version
|
||||
self.kind = kind
|
||||
|
||||
@property
|
||||
def schema(self):
|
||||
# TODO: return schema based on version and kind.
|
||||
return [v['schema'] for v in self.schema_versions_info
|
||||
if v['version'] == self.schema_version][0].schema
|
||||
|
||||
def pre_validate_data(self):
|
||||
"""Pre-validate that the YAML file is correctly formatted."""
|
||||
self._validate_with_schema()
|
||||
|
||||
# Validate that each "dest" field exists in the YAML data.
|
||||
# FIXME(fm577c): Dest fields will be injected if not present - the
|
||||
# validation below needs to be updated or removed.
|
||||
substitutions = self.data['metadata']['substitutions']
|
||||
destinations = [s['dest'] for s in substitutions]
|
||||
sub_data = self.data['data']
|
||||
|
||||
for dest in destinations:
|
||||
result, missing_attr = self._multi_getattr(dest['path'], sub_data)
|
||||
if not result:
|
||||
raise errors.InvalidFormat(
|
||||
'The attribute "%s" included in the "dest" field "%s" is '
|
||||
'missing from the YAML data: "%s".' % (
|
||||
missing_attr, dest, sub_data))
|
||||
|
||||
# TODO(fm577c): Query Deckhand API to validate "src" values.
|
||||
|
||||
def _validate_with_schema(self):
|
||||
# Validate the document using the schema defined by the document's
|
||||
# `schemaVersion` and `kind`.
|
||||
try:
|
||||
schema_version = self.data['schemaVersion'].split('/')[-1]
|
||||
doc_kind = self.data['kind']
|
||||
doc_schema_version = self.SchemaVersion(schema_version, doc_kind)
|
||||
except (AttributeError, IndexError, KeyError) as e:
|
||||
raise errors.InvalidFormat(
|
||||
'The provided schemaVersion is invalid or missing. Exception: '
|
||||
'%s.' % e)
|
||||
try:
|
||||
jsonschema.validate(self.data, doc_schema_version.schema)
|
||||
except jsonschema.exceptions.ValidationError as e:
|
||||
raise errors.InvalidFormat('The provided YAML file is invalid. '
|
||||
'Exception: %s.' % e.message)
|
||||
|
||||
def _multi_getattr(self, multi_key, substitutable_data):
|
||||
"""Iteratively check for nested attributes in the YAML data.
|
||||
|
||||
Check for nested attributes included in "dest" attributes in the data
|
||||
section of the YAML file. For example, a "dest" attribute of
|
||||
".foo.bar.baz" should mean that the YAML data adheres to:
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
---
|
||||
foo:
|
||||
bar:
|
||||
baz: <data_to_be_substituted_here>
|
||||
|
||||
:param multi_key: A multi-part key that references nested data in the
|
||||
substitutable part of the YAML data, e.g. ".foo.bar.baz".
|
||||
:param substitutable_data: The section of data in the YAML data that
|
||||
is intended to be substituted with secrets.
|
||||
:returns: Tuple where first value is a boolean indicating that the
|
||||
nested attribute was found and the second value is the attribute
|
||||
that was not found, if applicable.
|
||||
"""
|
||||
attrs = multi_key.split('.')
|
||||
# Ignore the first attribute if it is "." as that is a self-reference.
|
||||
if attrs[0] == '':
|
||||
attrs = attrs[1:]
|
||||
|
||||
data = substitutable_data
|
||||
for attr in attrs:
|
||||
if attr not in data:
|
||||
return False, attr
|
||||
data = data.get(attr)
|
||||
|
||||
return True, None
|
||||
|
|
@ -56,3 +56,4 @@ class InvalidFormat(ApiError):
|
|||
class DocumentExists(DeckhandException):
|
||||
msg_fmt = ("Document with kind %(kind)s and schemaVersion "
|
||||
"%(schema_version)s already exists.")
|
||||
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ from deckhand.control import base as api_base
|
|||
|
||||
class TestApi(testtools.TestCase):
|
||||
|
||||
|
||||
@mock.patch.object(api, 'db_api', autospec=True)
|
||||
@mock.patch.object(api, 'config', autospec=True)
|
||||
@mock.patch.object(api, 'secrets', autospec=True)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,123 @@
|
|||
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import copy
|
||||
import os
|
||||
import testtools
|
||||
import yaml
|
||||
|
||||
import six
|
||||
|
||||
from deckhand.engine import secret_substitution
|
||||
from deckhand import errors
|
||||
|
||||
|
||||
class TestSecretSubtitution(testtools.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
super(TestSecretSubtitution, self).setUp()
|
||||
dir_path = os.path.dirname(os.path.realpath(__file__))
|
||||
test_yaml_path = os.path.abspath(os.path.join(
|
||||
dir_path, os.pardir, 'resources', 'sample.yaml'))
|
||||
|
||||
with open(test_yaml_path, 'r') as yaml_file:
|
||||
yaml_data = yaml_file.read()
|
||||
self.data = yaml.safe_load(yaml_data)
|
||||
|
||||
def _corrupt_data(self, key, data=None):
|
||||
"""Corrupt test data to check that pre-validation works.
|
||||
|
||||
Corrupt data by removing a key from the document. Each key must
|
||||
correspond to a value that is a dictionary.
|
||||
|
||||
:param key: The document key to be removed. The key can have the
|
||||
following formats:
|
||||
* 'data' => document.pop('data')
|
||||
* 'metadata.name' => document['metadata'].pop('name')
|
||||
* 'metadata.substitutions.0.dest' =>
|
||||
document['metadata']['substitutions'][0].pop('dest')
|
||||
:returns: Corrupted YAML data.
|
||||
"""
|
||||
if data is None:
|
||||
data = self.data
|
||||
corrupted_data = copy.deepcopy(data)
|
||||
|
||||
if '.' in key:
|
||||
_corrupted_data = corrupted_data
|
||||
nested_keys = key.split('.')
|
||||
for nested_key in nested_keys:
|
||||
if nested_key == nested_keys[-1]:
|
||||
break
|
||||
if nested_key.isdigit():
|
||||
_corrupted_data = _corrupted_data[int(nested_key)]
|
||||
else:
|
||||
_corrupted_data = _corrupted_data[nested_key]
|
||||
_corrupted_data.pop(nested_keys[-1])
|
||||
else:
|
||||
corrupted_data.pop(key)
|
||||
|
||||
return self._format_data(corrupted_data)
|
||||
|
||||
def _format_data(self, data=None):
|
||||
"""Re-formats dict data as YAML to pass to ``SecretSubstitution``."""
|
||||
if data is None:
|
||||
data = self.data
|
||||
return yaml.safe_dump(data)
|
||||
|
||||
def test_initialization(self):
|
||||
sub = secret_substitution.SecretSubstitution(self._format_data())
|
||||
self.assertIsInstance(sub, secret_substitution.SecretSubstitution)
|
||||
|
||||
def test_initialization_missing_sections(self):
|
||||
expected_err = ("The provided YAML file is invalid. Exception: '%s' "
|
||||
"is a required property.")
|
||||
invalid_data = [
|
||||
(self._corrupt_data('data'), 'data'),
|
||||
(self._corrupt_data('metadata'), 'metadata'),
|
||||
(self._corrupt_data('metadata.metadataVersion'), 'metadataVersion'),
|
||||
(self._corrupt_data('metadata.name'), 'name'),
|
||||
(self._corrupt_data('metadata.substitutions'), 'substitutions'),
|
||||
(self._corrupt_data('metadata.substitutions.0.dest'), 'dest'),
|
||||
(self._corrupt_data('metadata.substitutions.0.src'), 'src')
|
||||
]
|
||||
|
||||
for invalid_entry, missing_key in invalid_data:
|
||||
with six.assertRaisesRegex(self, errors.InvalidFormat,
|
||||
expected_err % missing_key):
|
||||
secret_substitution.SecretSubstitution(invalid_entry)
|
||||
|
||||
def test_initialization_bad_substitutions(self):
|
||||
expected_err = ('The attribute "%s" included in the "dest" field "%s" '
|
||||
'is missing from the YAML data')
|
||||
invalid_data = []
|
||||
|
||||
data = copy.deepcopy(self.data)
|
||||
data['metadata']['substitutions'][0]['dest'] = {'path': 'foo'}
|
||||
invalid_data.append(self._format_data(data))
|
||||
|
||||
data = copy.deepcopy(self.data)
|
||||
data['metadata']['substitutions'][0]['dest'] = {
|
||||
'path': 'tls_endpoint.bar'}
|
||||
invalid_data.append(self._format_data(data))
|
||||
|
||||
def _test(invalid_entry, field, dest):
|
||||
_expected_err = expected_err % (field, dest)
|
||||
with six.assertRaisesRegex(self, errors.InvalidFormat,
|
||||
_expected_err):
|
||||
secret_substitution.SecretSubstitution(invalid_entry)
|
||||
|
||||
# Verify that invalid body dest reference is invalid.
|
||||
_test(invalid_data[0], "foo", {'path': 'foo'})
|
||||
# Verify that nested invalid body dest reference is invalid.
|
||||
_test(invalid_data[1], "bar", {'path': 'tls_endpoint.bar'})
|
||||
Loading…
Reference in New Issue