From a7d1f0e2f45127bbf8c6c67db72acad12e16da9d Mon Sep 17 00:00:00 2001 From: Felipe Monteiro Date: Sat, 8 Jul 2017 23:34:44 +0100 Subject: [PATCH] Initial engine framework This commit adds the initial engine framework for Deckhand. Included is the logic for parsing YAML files as well as validating them and doing forward substitution as specified by the YAML file. This commit also includes unit tests for the framework changes. --- .gitignore | 1 + .testr.conf | 7 + deckhand/barbican/driver.py | 7 +- deckhand/control/api.py | 2 +- deckhand/control/secrets.py | 28 +++- deckhand/engine/__init__.py | 0 deckhand/engine/secret_substitution.py | 134 ++++++++++++++++++ deckhand/errors.py | 2 +- deckhand/tests/__init__.py | 0 deckhand/tests/unit/__init__.py | 0 deckhand/tests/unit/control/__init__.py | 0 deckhand/tests/unit/control/test_api.py | 36 +++++ deckhand/tests/unit/control/test_base.py | 40 ++++++ deckhand/tests/unit/engine/__init__.py | 0 .../unit/engine/test_secret_substitution.py | 112 +++++++++++++++ deckhand/tests/unit/resources/sample.yaml | 23 +++ test-requirements.txt | 12 +- tox.ini | 28 +++- 18 files changed, 414 insertions(+), 18 deletions(-) create mode 100644 .testr.conf create mode 100644 deckhand/engine/__init__.py create mode 100644 deckhand/engine/secret_substitution.py create mode 100644 deckhand/tests/__init__.py create mode 100644 deckhand/tests/unit/__init__.py create mode 100644 deckhand/tests/unit/control/__init__.py create mode 100644 deckhand/tests/unit/control/test_api.py create mode 100644 deckhand/tests/unit/control/test_base.py create mode 100644 deckhand/tests/unit/engine/__init__.py create mode 100644 deckhand/tests/unit/engine/test_secret_substitution.py create mode 100644 deckhand/tests/unit/resources/sample.yaml diff --git a/.gitignore b/.gitignore index 849c5852..188f0211 100644 --- a/.gitignore +++ b/.gitignore @@ -46,6 +46,7 @@ nosetests.xml coverage.xml *.cover .hypothesis/ +.testrepository/* # Translations *.mo diff --git a/.testr.conf b/.testr.conf new file mode 100644 index 00000000..6d83b3c4 --- /dev/null +++ b/.testr.conf @@ -0,0 +1,7 @@ +[DEFAULT] +test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \ + OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \ + OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-60} \ + ${PYTHON:-python} -m subunit.run discover -t ./ . $LISTOPT $IDOPTION +test_id_option=--load-list $IDFILE +test_list_option=--list diff --git a/deckhand/barbican/driver.py b/deckhand/barbican/driver.py index db8352f3..d34e4d96 100644 --- a/deckhand/barbican/driver.py +++ b/deckhand/barbican/driver.py @@ -20,7 +20,6 @@ class BarbicanDriver(object): def __init__(self): self.barbicanclient = client_wrapper.BarbicanClientWrapper() - def ca_list(self, **kwargs): - # FIXME(felipemonteiro): Testing cas.list endpoint. - ca_list = self.barbicanclient.call("cas.list", **kwargs) - return ca_list + def create_secret(self, **kwargs): + """Create a secret.""" + return self.barbicanclient.call("secrets.create", **kwargs) diff --git a/deckhand/control/api.py b/deckhand/control/api.py index a5ca0606..44f89f6a 100644 --- a/deckhand/control/api.py +++ b/deckhand/control/api.py @@ -28,7 +28,7 @@ def start_api(state_manager=None): control_api = falcon.API(request_type=api_base.DeckhandRequest) v1_0_routes = [ - ('/secrets', secrets.SecretsResource()) + ('secrets', secrets.SecretsResource()) ] for path, res in v1_0_routes: diff --git a/deckhand/control/secrets.py b/deckhand/control/secrets.py index 79c88f16..3e1583f2 100644 --- a/deckhand/control/secrets.py +++ b/deckhand/control/secrets.py @@ -23,8 +23,7 @@ from deckhand.control import base as api_base class SecretsResource(api_base.BaseResource): """API resource for interacting with Barbican. - TODO(felipemonteiro): Once Barbican integration is fully implemented, - implement API endpoints below. + NOTE: Currently only supports Barbican. """ def __init__(self, **kwargs): @@ -32,8 +31,25 @@ class SecretsResource(api_base.BaseResource): self.authorized_roles = ['user'] self.barbican_driver = driver.BarbicanDriver() - def on_get(self, req, resp): - # TODO(felipemonteiro): Implement this API endpoint. - ca_list = self.barbican_driver.ca_list() # Random endpoint to test. - resp.body = json.dumps({'secrets': [c.to_dict() for c in ca_list]}) + def on_post(self, req, resp): + """Create a secret. + + :param name: The name of the secret. Required. + :param type: The type of the secret. Optional. + + For a list of types, please refer to the following API documentation: + https://docs.openstack.org/barbican/latest/api/reference/secret_types.html + """ + secret_name = req.params.get('name', None) + secret_type = req.params.get('type', None) + + if not secret_name: + resp.status = falcon.HTTP_400 + + # Do not allow users to call Barbican with all permitted kwargs. + # Selectively include only what we allow. + kwargs = {'name': secret_name, 'secret_type': secret_type} + secret = self.barbican_driver.create_secret(**kwargs) + + resp.body = json.dumps(secret) resp.status = falcon.HTTP_200 diff --git a/deckhand/engine/__init__.py b/deckhand/engine/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/deckhand/engine/secret_substitution.py b/deckhand/engine/secret_substitution.py new file mode 100644 index 00000000..80ac27c8 --- /dev/null +++ b/deckhand/engine/secret_substitution.py @@ -0,0 +1,134 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import yaml + +from deckhand import errors + + +class SecretSubstitution(object): + """Initialization of class for secret substitution logic for YAML files. + + This class is responsible for parsing, validating and retrieving secret + values for values stored in the YAML file. Afterward, secret values will be + substituted or "forward-repalced" into the YAML file. The end result is a + YAML file containing all necessary secrets to be handed off to other + services. + """ + + def __init__(self, data): + try: + self.data = yaml.safe_load(data) + except yaml.YAMLError: + raise errors.InvalidFormat( + 'The provided YAML file cannot be parsed.') + + self.validate_data() + + def validate_data(self): + """Validate that the YAML file is correctly formatted. + + The YAML file must adhere to the following bare minimum format: + + .. code-block:: yaml + + --- + apiVersion: service/v1 + kind: ConsumerOfCertificateData + metadata: + substitutions: + - dest: .tls_endpoint.certificate + src: + apiVersion: deckhand/v1 + kind: Certificate + name: some-certificate-asdf-1234 + # Forward-reference to specific section under "data" below. + - dest: .tls_endpoint.certificateKey + src: + apiVersion: deckhand/v1 + kind: CertificateKey + name: some-certificate-key-asdf-1234 + data: + tls_endpoint: + certificate: null # Data to be substituted. + certificateKey: null # Data to be substituted. + """ + # Validate that data section exists. + try: + self.data['data'] + except (KeyError, TypeError) as e: + raise errors.InvalidFormat( + 'The provided YAML file has no data section: %s' % e) + # Validate that substitutions section exists. + try: + substitutions = self.data['metadata']['substitutions'] + except (KeyError, TypeError) as e: + raise errors.InvalidFormat( + 'The provided YAML file has no metadata/substitutions ' + 'section: %s' % e) + + # Validate that "src" and "dest" fields exist per substitution entry. + error_message = ('The provided YAML file is missing the "%s" field ' + 'for the %s substition.') + for s in substitutions: + if 'src' not in s: + raise errors.InvalidFormat(error_message % ('src', s)) + elif 'dest' not in s: + raise errors.InvalidFormat(error_message % ('dest', s)) + + # Validate that each "dest" field exists in the YAML data. + destinations = [s['dest'] for s in substitutions] + sub_data = self.data['data'] + for dest in destinations: + result, missing_attr = self._multi_getattr(dest, sub_data) + if not result: + raise errors.InvalidFormat( + 'The attribute "%s" included in the "dest" field "%s" is ' + 'missing from the YAML data: "%s".' % ( + missing_attr, dest, sub_data)) + + def _multi_getattr(self, multi_key, substitutable_data): + """Iteratively check for nested attributes in the YAML data. + + Check for nested attributes included in "dest" attributes in the data + section of the YAML file. For example, a "dest" attribute of + ".foo.bar.baz" should mean that the YAML data adheres to: + + .. code-block:: yaml + + --- + foo: + bar: + baz: + + :param multi_key: A multi-part key that references nested data in the + substitutable part of the YAML data, e.g. ".foo.bar.baz". + :param substitutable_data: The section of data in the YAML data that + is intended to be substituted with secrets. + :returns: Tuple where first value is a boolean indicating that the + nested attribute was found and the second value is the attribute + that was not found, if applicable. + """ + attrs = multi_key.split('.') + # Ignore the first attribute if it is "." as that is a self-reference. + if attrs[0] == '': + attrs = attrs[1:] + + data = substitutable_data + for attr in attrs: + if attr not in data: + return False, attr + data = data.get(attr) + + return True, None diff --git a/deckhand/errors.py b/deckhand/errors.py index 1a1e3c62..ca42be11 100644 --- a/deckhand/errors.py +++ b/deckhand/errors.py @@ -18,4 +18,4 @@ class ApiError(Exception): class InvalidFormat(ApiError): - pass + """The YAML file is incorrectly formatted and cannot be read.""" diff --git a/deckhand/tests/__init__.py b/deckhand/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/deckhand/tests/unit/__init__.py b/deckhand/tests/unit/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/deckhand/tests/unit/control/__init__.py b/deckhand/tests/unit/control/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/deckhand/tests/unit/control/test_api.py b/deckhand/tests/unit/control/test_api.py new file mode 100644 index 00000000..2d327285 --- /dev/null +++ b/deckhand/tests/unit/control/test_api.py @@ -0,0 +1,36 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock + +import testtools + +from deckhand.control import api +from deckhand.control import base as api_base + + +class TestApi(testtools.TestCase): + + @mock.patch.object(api, 'secrets', autospec=True) + @mock.patch.object(api, 'falcon', autospec=True) + def test_start_api(self, mock_falcon, mock_secrets): + mock_falcon_api = mock_falcon.API.return_value + + result = api.start_api() + self.assertEqual(mock_falcon_api, result) + + mock_falcon.API.assert_called_once_with( + request_type=api_base.DeckhandRequest) + mock_falcon_api.add_route.assert_called_once_with( + '/api/v1.0/secrets', mock_secrets.SecretsResource()) diff --git a/deckhand/tests/unit/control/test_base.py b/deckhand/tests/unit/control/test_base.py new file mode 100644 index 00000000..e9e60a28 --- /dev/null +++ b/deckhand/tests/unit/control/test_base.py @@ -0,0 +1,40 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock + +import testtools + +from deckhand.control import base as api_base + + +class TestBaseResource(testtools.TestCase): + + def setUp(self): + super(TestBaseResource, self).setUp() + self.base_resource = api_base.BaseResource() + + def test_on_options(self): + # Override `dir` so that ``dir(self)`` returns `methods`. + expected_methods = ['on_get', 'on_heat', 'on_post', 'on_put', + 'on_delete', 'on_patch'] + api_base.BaseResource.__dir__ = lambda x: expected_methods + + mock_resp = mock.Mock(headers={}) + self.base_resource.on_options(None, mock_resp) + + self.assertIn('Allow', mock_resp.headers) + self.assertEqual('GET,POST,PUT,DELETE,PATCH', + mock_resp.headers['Allow']) + self.assertEqual('200 OK', mock_resp.status) diff --git a/deckhand/tests/unit/engine/__init__.py b/deckhand/tests/unit/engine/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/deckhand/tests/unit/engine/test_secret_substitution.py b/deckhand/tests/unit/engine/test_secret_substitution.py new file mode 100644 index 00000000..b2b9019a --- /dev/null +++ b/deckhand/tests/unit/engine/test_secret_substitution.py @@ -0,0 +1,112 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import testtools + +from oslo_serialization import jsonutils as json +import six + +from deckhand.engine import secret_substitution +from deckhand import errors + + +class TestSecretSubtitution(testtools.TestCase): + + def setUp(self): + super(TestSecretSubtitution, self).setUp() + dir_path = os.path.dirname(os.path.realpath(__file__)) + test_yaml_path = os.path.abspath(os.path.join( + dir_path, os.pardir, 'resources', 'sample.yaml')) + + with open(test_yaml_path, 'r') as yaml_data: + self.yaml_data = yaml_data.read() + + def test_initialization_missing_substitutions_section(self): + expected_err = ( + "The provided YAML file has no metadata/substitutions section") + invalid_data = [ + {"data": []}, + {"data": [], "metadata": None}, + {"data": [], "metadata": {"missing_substitutions": None}} + ] + + for invalid_entry in invalid_data: + invalid_entry = json.dumps(invalid_entry) + with six.assertRaisesRegex(self, errors.InvalidFormat, + expected_err): + secret_substitution.SecretSubstitution(invalid_entry) + + expected_err = ( + "The provided YAML file has no metadata/substitutions section") + invalid_data = [ + {"data": [], "metadata": None}, + ] + + def test_initialization_missing_data_section(self): + expected_err = ( + "The provided YAML file has no data section") + invalid_data = '{"metadata": {"substitutions": []}}' + + with six.assertRaisesRegex(self, errors.InvalidFormat, expected_err): + secret_substitution.SecretSubstitution(invalid_data) + + def test_initialization_missing_src_dest_sections(self): + expected_err = ('The provided YAML file is missing the "%s" field for ' + 'the %s substition.') + invalid_data = [ + {"data": [], "metadata": {"substitutions": [{"dest": "foo"}]}}, + {"data": [], "metadata": {"substitutions": [{"src": "bar"}]}}, + ] + + def _test(invalid_entry, field, substitution): + invalid_entry = json.dumps(invalid_entry) + _expected_err = expected_err % (field, substitution) + + with six.assertRaisesRegex(self, errors.InvalidFormat, + _expected_err): + secret_substitution.SecretSubstitution(invalid_entry) + + _test(invalid_data[0], "src", {"dest": "foo"}) + _test(invalid_data[1], "dest", {"src": "bar"}) + + def test_initialization_bad_substitutions(self): + expected_err = ('The attribute "%s" included in the "dest" field "%s" ' + 'is missing from the YAML data: "%s".') + invalid_data = [ + # Missing attribute. + {"data": {}, "metadata": {"substitutions": [ + {"src": "", "dest": "foo"} + ]}}, + # Missing attribute. + {"data": {"foo": None}, "metadata": {"substitutions": [ + {"src": "", "dest": "bar"} + ]}}, + # Missing nested attribute. + {"data": {"foo": {"baz": None}}, "metadata": {"substitutions": [ + {"src": "", "dest": "foo.bar"} + ]}}, + ] + + def _test(invalid_entry, field, dest, substitution): + invalid_entry = json.dumps(invalid_entry) + _expected_err = expected_err % (field, dest, substitution) + + with six.assertRaisesRegex(self, errors.InvalidFormat, + _expected_err): + secret_substitution.SecretSubstitution(invalid_entry) + + _test(invalid_data[0], "foo", "foo", {}) + _test(invalid_data[1], "bar", "bar", {"foo": None}) + _test(invalid_data[2], "bar", "foo.bar", {'foo': {'baz': None}}) diff --git a/deckhand/tests/unit/resources/sample.yaml b/deckhand/tests/unit/resources/sample.yaml new file mode 100644 index 00000000..0e921f63 --- /dev/null +++ b/deckhand/tests/unit/resources/sample.yaml @@ -0,0 +1,23 @@ +# Sample YAML file for testing forward replacement. +--- +apiVersion: service/v1 +kind: ConsumerOfCertificateData +metadata: + name: asdf-1234 + storage: cleartext + substitutions: + - dest: .tls_endpoint.certificate + src: + apiVersion: deckhand/v1 + kind: Certificate + name: some-certificate-asdf-1234 + - dest: .tls_endpoint.certificateKey + src: + apiVersion: deckhand/v1 + kind: CertificateKey + name: some-certificate-key-asdf-1234 +data: + tls_endpoint: + uri: http://localhost:443 + certificate: null + certificateKey: null \ No newline at end of file diff --git a/test-requirements.txt b/test-requirements.txt index 17d1a8e9..646fffc9 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1 +1,11 @@ -falcon==1.1.0 \ No newline at end of file +falcon==1.1.0 + +mock>=2.0 +fixtures>=3.0.0 # Apache-2.0/BSD +mock>=2.0 # BSD +mox3!=0.19.0,>=0.7.0 # Apache-2.0 +python-subunit>=0.0.18 # Apache-2.0/BSD +oslotest>=1.10.0 # Apache-2.0 +os-testr>=0.8.0 # Apache-2.0 +testrepository>=0.0.18 # Apache-2.0/BSD +testtools>=1.4.0 # MIT diff --git a/tox.ini b/tox.ini index 5a9a54f1..ff5823ac 100644 --- a/tox.ini +++ b/tox.ini @@ -1,15 +1,33 @@ [tox] -envlist = py35,py27,pep8 +envlist = py{35,27},pep8 [testenv] usedevelop = True +whitelist_externals = bash + find + rm + env + flake8 setenv = VIRTUAL_ENV={envdir} + OS_TEST_PATH=./deckhand/tests/unit LANGUAGE=en_US LC_ALL=en_US.utf-8 -deps= - -r{toxinidir}/requirements.txt - -r{toxinidir}/test-requirements.txt -whitelist_externals = flake8 +passenv = OS_STDOUT_CAPTURE OS_STDERR_CAPTURE OS_TEST_TIMEOUT OS_TEST_LOCK_PATH OS_TEST_PATH http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY +deps = -r{toxinidir}/requirements.txt + -r{toxinidir}/test-requirements.txt +commands = + find . -type f -name "*.pyc" -delete + rm -Rf .testrepository/times.dbm + +[testenv:py27] +commands = + {[testenv]commands} + ostestr '{posargs}' + +[testenv:py35] +commands = + {[testenv]commands} + ostestr '{posargs}' [testenv:genconfig] commands = oslo-config-generator --config-file=etc/deckhand/config-generator.conf