Merge pull request #6 from att-comdev/init-yaml-framework

Initial engine framework
This commit is contained in:
Mark Burnett 2017-07-13 12:57:46 -05:00 committed by GitHub
commit 26a0dc8d3b
23 changed files with 541 additions and 20 deletions

1
.gitignore vendored
View File

@ -46,6 +46,7 @@ nosetests.xml
coverage.xml
*.cover
.hypothesis/
.testrepository/*
# Translations
*.mo

7
.testr.conf Normal file
View File

@ -0,0 +1,7 @@
[DEFAULT]
test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \
OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \
OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-60} \
${PYTHON:-python} -m subunit.run discover -t ./ . $LISTOPT $IDOPTION
test_id_option=--load-list $IDFILE
test_list_option=--list

View File

@ -20,7 +20,6 @@ class BarbicanDriver(object):
def __init__(self):
self.barbicanclient = client_wrapper.BarbicanClientWrapper()
def ca_list(self, **kwargs):
# FIXME(felipemonteiro): Testing cas.list endpoint.
ca_list = self.barbicanclient.call("cas.list", **kwargs)
return ca_list
def create_secret(self, **kwargs):
"""Create a secret."""
return self.barbicanclient.call("secrets.create", **kwargs)

View File

@ -58,7 +58,7 @@ def start_api(state_manager=None):
control_api = falcon.API(request_type=api_base.DeckhandRequest)
v1_0_routes = [
('/secrets', secrets.SecretsResource())
('secrets', secrets.SecretsResource())
]
for path, res in v1_0_routes:

View File

@ -63,8 +63,7 @@ class BaseResource(object):
return None
try:
json_body = json.loads(raw_body.decode('utf-8'))
return json_body
return json.loads(raw_body.decode('utf-8'))
except json.JSONDecodeError as jex:
raise errors.InvalidFormat("%s: Invalid JSON in body: %s" % (
req.path, jex))

View File

@ -23,8 +23,7 @@ from deckhand.control import base as api_base
class SecretsResource(api_base.BaseResource):
"""API resource for interacting with Barbican.
TODO(felipemonteiro): Once Barbican integration is fully implemented,
implement API endpoints below.
NOTE: Currently only supports Barbican.
"""
def __init__(self, **kwargs):
@ -32,8 +31,25 @@ class SecretsResource(api_base.BaseResource):
self.authorized_roles = ['user']
self.barbican_driver = driver.BarbicanDriver()
def on_get(self, req, resp):
# TODO(felipemonteiro): Implement this API endpoint.
ca_list = self.barbican_driver.ca_list() # Random endpoint to test.
resp.body = json.dumps({'secrets': [c.to_dict() for c in ca_list]})
def on_post(self, req, resp):
"""Create a secret.
:param name: The name of the secret. Required.
:param type: The type of the secret. Optional.
For a list of types, please refer to the following API documentation:
https://docs.openstack.org/barbican/latest/api/reference/secret_types.html
"""
secret_name = req.params.get('name', None)
secret_type = req.params.get('type', None)
if not secret_name:
resp.status = falcon.HTTP_400
# Do not allow users to call Barbican with all permitted kwargs.
# Selectively include only what we allow.
kwargs = {'name': secret_name, 'secret_type': secret_type}
secret = self.barbican_driver.create_secret(**kwargs)
resp.body = json.dumps(secret)
resp.status = falcon.HTTP_200

View File

View File

View File

View File

@ -0,0 +1,101 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
substitution_schema = {
'type': 'object',
'properties': {
'dest': {
'type': 'object',
'properties': {
'path': {'type': 'string'},
'replacePattern': {'type': 'string'}
},
'additionalProperties': False,
# 'replacePattern' is not required.
'required': ['path']
},
'src': {
'type': 'object',
'properties': {
'kind': {'type': 'string'},
'name': {'type': 'string'},
'path': {'type': 'string'}
},
'additionalProperties': False,
'required': ['kind', 'name', 'path']
}
},
'additionalProperties': False,
'required': ['dest', 'src']
}
schema = {
'type': 'object',
'properties': {
'schemaVersion': {
'type': 'string',
'pattern': '^([A-Za-z]+\/v[0-9]{1})$'
},
# TODO: The kind should be an enum.
'kind': {'type': 'string'},
'metadata': {
'type': 'object',
'properties': {
'metadataVersion': {
'type': 'string',
'pattern': '^([A-Za-z]+\/v[0-9]{1})$'
},
'name': {'type': 'string'},
'labels': {
'type': 'object',
'properties': {
'component': {'type': 'string'},
'hostname': {'type': 'string'}
},
'additionalProperties': False,
'required': ['component', 'hostname']
},
'layerDefinition': {
'type': 'object',
'properties': {
'layer': {'enum': ['global', 'region', 'site']},
'abstract': {'type': 'boolean'},
'childSelector': {
'type': 'object',
'properties': {
'label': {'type': 'string'}
},
'additionalProperties': False,
'required': ['label']
}
},
'additionalProperties': False,
'required': ['layer', 'abstract', 'childSelector']
},
'substitutions': {
'type': 'array',
'items': substitution_schema
}
},
'additionalProperties': False,
'required': ['metadataVersion', 'name', 'labels',
'layerDefinition', 'substitutions']
},
'data': {
'type': 'object'
}
},
'additionalProperties': False,
'required': ['schemaVersion', 'kind', 'metadata', 'data']
}

View File

@ -0,0 +1,138 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import yaml
import jsonschema
from deckhand.engine.schema.v1_0 import default_schema
from deckhand import errors
class SecretSubstitution(object):
"""Class for secret substitution logic for YAML files.
This class is responsible for parsing, validating and retrieving secret
values for values stored in the YAML file. Afterward, secret values will be
substituted or "forward-repalced" into the YAML file. The end result is a
YAML file containing all necessary secrets to be handed off to other
services.
:param data: YAML data that requires secrets to be validated, merged and
consolidated.
"""
def __init__(self, data):
try:
self.data = yaml.safe_load(data)
except yaml.YAMLError:
raise errors.InvalidFormat(
'The provided YAML file cannot be parsed.')
self.pre_validate_data()
class SchemaVersion(object):
"""Class for retrieving correct schema for pre-validation on YAML.
Retrieves the schema that corresponds to "apiVersion" in the YAML
data. This schema is responsible for performing pre-validation on
YAML data.
"""
# TODO: Update kind according to requirements.
schema_versions_info = [{'version': 'v1', 'kind': 'default',
'schema': default_schema}]
def __init__(self, schema_version, kind):
self.schema_version = schema_version
self.kind = kind
@property
def schema(self):
# TODO: return schema based on version and kind.
return [v['schema'] for v in self.schema_versions_info
if v['version'] == self.schema_version][0].schema
def pre_validate_data(self):
"""Pre-validate that the YAML file is correctly formatted."""
self._validate_with_schema()
# Validate that each "dest" field exists in the YAML data.
# FIXME(fm577c): Dest fields will be injected if not present - the
# validation below needs to be updated or removed.
substitutions = self.data['metadata']['substitutions']
destinations = [s['dest'] for s in substitutions]
sub_data = self.data['data']
for dest in destinations:
result, missing_attr = self._multi_getattr(dest['path'], sub_data)
if not result:
raise errors.InvalidFormat(
'The attribute "%s" included in the "dest" field "%s" is '
'missing from the YAML data: "%s".' % (
missing_attr, dest, sub_data))
# TODO(fm577c): Query Deckhand API to validate "src" values.
def _validate_with_schema(self):
# Validate the document using the schema defined by the document's
# `schemaVersion` and `kind`.
try:
schema_version = self.data['schemaVersion'].split('/')[-1]
doc_kind = self.data['kind']
doc_schema_version = self.SchemaVersion(schema_version, doc_kind)
except (AttributeError, IndexError, KeyError) as e:
raise errors.InvalidFormat(
'The provided schemaVersion is invalid or missing. Exception: '
'%s.' % e)
try:
jsonschema.validate(self.data, doc_schema_version.schema)
except jsonschema.exceptions.ValidationError as e:
raise errors.InvalidFormat('The provided YAML file is invalid. '
'Exception: %s.' % e.message)
def _multi_getattr(self, multi_key, substitutable_data):
"""Iteratively check for nested attributes in the YAML data.
Check for nested attributes included in "dest" attributes in the data
section of the YAML file. For example, a "dest" attribute of
".foo.bar.baz" should mean that the YAML data adheres to:
.. code-block:: yaml
---
foo:
bar:
baz: <data_to_be_substituted_here>
:param multi_key: A multi-part key that references nested data in the
substitutable part of the YAML data, e.g. ".foo.bar.baz".
:param substitutable_data: The section of data in the YAML data that
is intended to be substituted with secrets.
:returns: Tuple where first value is a boolean indicating that the
nested attribute was found and the second value is the attribute
that was not found, if applicable.
"""
attrs = multi_key.split('.')
# Ignore the first attribute if it is "." as that is a self-reference.
if attrs[0] == '':
attrs = attrs[1:]
data = substitutable_data
for attr in attrs:
if attr not in data:
return False, attr
data = data.get(attr)
return True, None

View File

@ -18,4 +18,4 @@ class ApiError(Exception):
class InvalidFormat(ApiError):
pass
"""The YAML file is incorrectly formatted and cannot be read."""

View File

View File

View File

View File

@ -0,0 +1,36 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import mock
import testtools
from deckhand.control import api
from deckhand.control import base as api_base
class TestApi(testtools.TestCase):
@mock.patch.object(api, 'secrets', autospec=True)
@mock.patch.object(api, 'falcon', autospec=True)
def test_start_api(self, mock_falcon, mock_secrets):
mock_falcon_api = mock_falcon.API.return_value
result = api.start_api()
self.assertEqual(mock_falcon_api, result)
mock_falcon.API.assert_called_once_with(
request_type=api_base.DeckhandRequest)
mock_falcon_api.add_route.assert_called_once_with(
'/api/v1.0/secrets', mock_secrets.SecretsResource())

View File

@ -0,0 +1,40 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import mock
import testtools
from deckhand.control import base as api_base
class TestBaseResource(testtools.TestCase):
def setUp(self):
super(TestBaseResource, self).setUp()
self.base_resource = api_base.BaseResource()
def test_on_options(self):
# Override `dir` so that ``dir(self)`` returns `methods`.
expected_methods = ['on_get', 'on_heat', 'on_post', 'on_put',
'on_delete', 'on_patch']
api_base.BaseResource.__dir__ = lambda x: expected_methods
mock_resp = mock.Mock(headers={})
self.base_resource.on_options(None, mock_resp)
self.assertIn('Allow', mock_resp.headers)
self.assertEqual('GET,POST,PUT,DELETE,PATCH',
mock_resp.headers['Allow'])
self.assertEqual('200 OK', mock_resp.status)

View File

View File

@ -0,0 +1,123 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import copy
import os
import testtools
import yaml
import six
from deckhand.engine import secret_substitution
from deckhand import errors
class TestSecretSubtitution(testtools.TestCase):
def setUp(self):
super(TestSecretSubtitution, self).setUp()
dir_path = os.path.dirname(os.path.realpath(__file__))
test_yaml_path = os.path.abspath(os.path.join(
dir_path, os.pardir, 'resources', 'sample.yaml'))
with open(test_yaml_path, 'r') as yaml_file:
yaml_data = yaml_file.read()
self.data = yaml.safe_load(yaml_data)
def _corrupt_data(self, key, data=None):
"""Corrupt test data to check that pre-validation works.
Corrupt data by removing a key from the document. Each key must
correspond to a value that is a dictionary.
:param key: The document key to be removed. The key can have the
following formats:
* 'data' => document.pop('data')
* 'metadata.name' => document['metadata'].pop('name')
* 'metadata.substitutions.0.dest' =>
document['metadata']['substitutions'][0].pop('dest')
:returns: Corrupted YAML data.
"""
if data is None:
data = self.data
corrupted_data = copy.deepcopy(data)
if '.' in key:
_corrupted_data = corrupted_data
nested_keys = key.split('.')
for nested_key in nested_keys:
if nested_key == nested_keys[-1]:
break
if nested_key.isdigit():
_corrupted_data = _corrupted_data[int(nested_key)]
else:
_corrupted_data = _corrupted_data[nested_key]
_corrupted_data.pop(nested_keys[-1])
else:
corrupted_data.pop(key)
return self._format_data(corrupted_data)
def _format_data(self, data=None):
"""Re-formats dict data as YAML to pass to ``SecretSubstitution``."""
if data is None:
data = self.data
return yaml.safe_dump(data)
def test_initialization(self):
sub = secret_substitution.SecretSubstitution(self._format_data())
self.assertIsInstance(sub, secret_substitution.SecretSubstitution)
def test_initialization_missing_sections(self):
expected_err = ("The provided YAML file is invalid. Exception: '%s' "
"is a required property.")
invalid_data = [
(self._corrupt_data('data'), 'data'),
(self._corrupt_data('metadata'), 'metadata'),
(self._corrupt_data('metadata.metadataVersion'), 'metadataVersion'),
(self._corrupt_data('metadata.name'), 'name'),
(self._corrupt_data('metadata.substitutions'), 'substitutions'),
(self._corrupt_data('metadata.substitutions.0.dest'), 'dest'),
(self._corrupt_data('metadata.substitutions.0.src'), 'src')
]
for invalid_entry, missing_key in invalid_data:
with six.assertRaisesRegex(self, errors.InvalidFormat,
expected_err % missing_key):
secret_substitution.SecretSubstitution(invalid_entry)
def test_initialization_bad_substitutions(self):
expected_err = ('The attribute "%s" included in the "dest" field "%s" '
'is missing from the YAML data')
invalid_data = []
data = copy.deepcopy(self.data)
data['metadata']['substitutions'][0]['dest'] = {'path': 'foo'}
invalid_data.append(self._format_data(data))
data = copy.deepcopy(self.data)
data['metadata']['substitutions'][0]['dest'] = {
'path': 'tls_endpoint.bar'}
invalid_data.append(self._format_data(data))
def _test(invalid_entry, field, dest):
_expected_err = expected_err % (field, dest)
with six.assertRaisesRegex(self, errors.InvalidFormat,
_expected_err):
secret_substitution.SecretSubstitution(invalid_entry)
# Verify that invalid body dest reference is invalid.
_test(invalid_data[0], "foo", {'path': 'foo'})
# Verify that nested invalid body dest reference is invalid.
_test(invalid_data[1], "bar", {'path': 'tls_endpoint.bar'})

View File

@ -0,0 +1,33 @@
# Sample YAML file for testing forward replacement.
---
schemaVersion: promenade/v1
kind: SomeConfigType
metadata:
metadataVersion: deckhand/v1
name: a-unique-config-name-12345
labels:
component: apiserver
hostname: server0
layerDefinition:
layer: global
abstract: True
childSelector:
label: value
substitutions:
- dest:
path: .tls_endpoint.certificate
replacePattern: 'test.pattern'
src:
kind: Certificate
name: some-certificate-asdf-1234
path: .cert
- dest:
path: .tls_endpoint.key
src:
kind: CertificateKey
name: some-certificate-asdf-1234
path: .key
data:
tls_endpoint:
certificate: '.cert'
key: deckhand/v1:some-certificate-asdf-1234

View File

@ -1,5 +1,6 @@
falcon==1.1.0
jsonschema!=2.5.0,<3.0.0,>=2.0.0 # MIT
pbr!=2.1.0,>=2.0.0 # Apache-2.0
six>=1.9.0 # MIT
stevedore>=1.20.0 # Apache-2.0

View File

@ -1 +1,10 @@
falcon==1.1.0
falcon==1.1.0
mock>=2.0
fixtures>=3.0.0 # Apache-2.0/BSD
mox3!=0.19.0,>=0.7.0 # Apache-2.0
python-subunit>=0.0.18 # Apache-2.0/BSD
oslotest>=1.10.0 # Apache-2.0
os-testr>=0.8.0 # Apache-2.0
testrepository>=0.0.18 # Apache-2.0/BSD
testtools>=1.4.0 # MIT

28
tox.ini
View File

@ -1,15 +1,33 @@
[tox]
envlist = py35,py27,pep8
envlist = py{35,27},pep8
[testenv]
usedevelop = True
whitelist_externals = bash
find
rm
env
flake8
setenv = VIRTUAL_ENV={envdir}
OS_TEST_PATH=./deckhand/tests/unit
LANGUAGE=en_US
LC_ALL=en_US.utf-8
deps=
-r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
whitelist_externals = flake8
passenv = OS_STDOUT_CAPTURE OS_STDERR_CAPTURE OS_TEST_TIMEOUT OS_TEST_LOCK_PATH OS_TEST_PATH http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY
deps = -r{toxinidir}/requirements.txt
-r{toxinidir}/test-requirements.txt
commands =
find . -type f -name "*.pyc" -delete
rm -Rf .testrepository/times.dbm
[testenv:py27]
commands =
{[testenv]commands}
ostestr '{posargs}'
[testenv:py35]
commands =
{[testenv]commands}
ostestr '{posargs}'
[testenv:genconfig]
commands = oslo-config-generator --config-file=etc/deckhand/config-generator.conf