Initial engine framework

This commit adds the initial engine framework for Deckhand. Included
is the logic for parsing YAML files as well as validating them and
doing forward substitution as specified by the YAML file.

This commit also includes unit tests for the framework changes.

.gitignore

@@ -46,6 +46,7 @@ nosetests.xml
 coverage.xml
 *.cover
 .hypothesis/
+.testrepository/*
 
 # Translations
 *.mo

.testr.conf

@@ -0,0 +1,7 @@
+[DEFAULT]
+test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \
+             OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \
+             OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-60} \
+             ${PYTHON:-python} -m subunit.run discover -t ./ . $LISTOPT $IDOPTION
+test_id_option=--load-list $IDFILE
+test_list_option=--list

deckhand/barbican/driver.py

@@ -20,7 +20,6 @@ class BarbicanDriver(object):
     def __init__(self):
         self.barbicanclient = client_wrapper.BarbicanClientWrapper()
 
-    def ca_list(self, **kwargs):
-        # FIXME(felipemonteiro): Testing cas.list endpoint.
-        ca_list = self.barbicanclient.call("cas.list", **kwargs)
-        return ca_list
+    def create_secret(self, **kwargs):
+        """Create a secret."""
+        return self.barbicanclient.call("secrets.create", **kwargs)

deckhand/control/api.py

@@ -28,7 +28,7 @@ def start_api(state_manager=None):
     control_api = falcon.API(request_type=api_base.DeckhandRequest)
 
     v1_0_routes = [
-        ('/secrets', secrets.SecretsResource())
+        ('secrets', secrets.SecretsResource())
     ]
 
     for path, res in v1_0_routes:

deckhand/control/secrets.py

@@ -23,8 +23,7 @@ from deckhand.control import base as api_base
 class SecretsResource(api_base.BaseResource):
     """API resource for interacting with Barbican.
 
-    TODO(felipemonteiro): Once Barbican integration is fully implemented,
-    implement API endpoints below.
+    NOTE: Currently only supports Barbican.
     """
 
     def __init__(self, **kwargs):
@@ -32,8 +31,25 @@ class SecretsResource(api_base.BaseResource):
         self.authorized_roles = ['user']
         self.barbican_driver = driver.BarbicanDriver()
 
-    def on_get(self, req, resp):
-        # TODO(felipemonteiro): Implement this API endpoint.
-        ca_list = self.barbican_driver.ca_list()  # Random endpoint to test.
-        resp.body = json.dumps({'secrets': [c.to_dict() for c in ca_list]})
+    def on_post(self, req, resp):
+        """Create a secret.
+
+        :param name: The name of the secret. Required.
+        :param type: The type of the secret. Optional.
+
+        For a list of types, please refer to the following API documentation:
+        https://docs.openstack.org/barbican/latest/api/reference/secret_types.html
+        """
+        secret_name = req.params.get('name', None)
+        secret_type = req.params.get('type', None)
+
+        if not secret_name:
+            resp.status = falcon.HTTP_400
+
+        # Do not allow users to call Barbican with all permitted kwargs.
+        # Selectively include only what we allow.
+        kwargs = {'name': secret_name, 'secret_type': secret_type}
+        secret = self.barbican_driver.create_secret(**kwargs)
+
+        resp.body = json.dumps(secret)
         resp.status = falcon.HTTP_200

deckhand/engine/secret_substitution.py

@@ -0,0 +1,134 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import yaml
+
+from deckhand import errors
+
+
+class SecretSubstitution(object):
+    """Initialization of class for secret substitution logic for YAML files.
+
+    This class is responsible for parsing, validating and retrieving secret
+    values for values stored in the YAML file. Afterward, secret values will be
+    substituted or "forward-repalced" into the YAML file. The end result is a
+    YAML file containing all necessary secrets to be handed off to other
+    services.
+    """
+
+    def __init__(self, data):
+        try:
+            self.data = yaml.safe_load(data)
+        except yaml.YAMLError:
+            raise errors.InvalidFormat(
+                'The provided YAML file cannot be parsed.')
+
+        self.validate_data()
+
+    def validate_data(self):
+        """Validate that the YAML file is correctly formatted.
+
+        The YAML file must adhere to the following bare minimum format:
+
+        .. code-block:: yaml
+
+            ---
+            apiVersion: service/v1
+            kind: ConsumerOfCertificateData
+            metadata:
+              substitutions:
+                - dest: .tls_endpoint.certificate
+                  src:
+                    apiVersion: deckhand/v1
+                    kind: Certificate
+                    name: some-certificate-asdf-1234
+                # Forward-reference to specific section under "data" below.
+                - dest: .tls_endpoint.certificateKey
+                  src:
+                    apiVersion: deckhand/v1
+                    kind: CertificateKey
+                    name: some-certificate-key-asdf-1234
+            data:
+              tls_endpoint:
+                  certificate: null  # Data to be substituted.
+                  certificateKey: null  # Data to be substituted.
+        """
+        # Validate that data section exists.
+        try:
+            self.data['data']
+        except (KeyError, TypeError) as e:
+            raise errors.InvalidFormat(
+                'The provided YAML file has no data section: %s' % e)
+        # Validate that substitutions section exists.
+        try:
+            substitutions = self.data['metadata']['substitutions']
+        except (KeyError, TypeError) as e:
+            raise errors.InvalidFormat(
+                'The provided YAML file has no metadata/substitutions '
+                'section: %s' % e)
+
+        # Validate that "src" and "dest" fields exist per substitution entry.
+        error_message = ('The provided YAML file is missing the "%s" field '
+                         'for the %s substition.')
+        for s in substitutions:
+            if 'src' not in s:
+                raise errors.InvalidFormat(error_message % ('src', s))
+            elif 'dest' not in s:
+                raise errors.InvalidFormat(error_message % ('dest', s))
+
+        # Validate that each "dest" field exists in the YAML data.
+        destinations = [s['dest'] for s in substitutions]
+        sub_data = self.data['data']
+        for dest in destinations:
+            result, missing_attr = self._multi_getattr(dest, sub_data)
+            if not result:
+                raise errors.InvalidFormat(
+                    'The attribute "%s" included in the "dest" field "%s" is '
+                    'missing from the YAML data: "%s".' % (
+                        missing_attr, dest, sub_data))
+
+    def _multi_getattr(self, multi_key, substitutable_data):
+        """Iteratively check for nested attributes in the YAML data.
+
+        Check for nested attributes included in "dest" attributes in the data
+        section of the YAML file. For example, a "dest" attribute of
+        ".foo.bar.baz" should mean that the YAML data adheres to:
+
+        .. code-block:: yaml
+
+           ---
+           foo:
+               bar:
+                   baz: <data_to_be_substituted_here>
+
+        :param multi_key: A multi-part key that references nested data in the
+            substitutable part of the YAML data, e.g. ".foo.bar.baz".
+        :param substitutable_data: The section of data in the YAML data that
+            is intended to be substituted with secrets.
+        :returns: Tuple where first value is a boolean indicating that the
+            nested attribute was found and the second value is the attribute
+            that was not found, if applicable.
+        """
+        attrs = multi_key.split('.')
+        # Ignore the first attribute if it is "." as that is a self-reference.
+        if attrs[0] == '':
+            attrs = attrs[1:]
+
+        data = substitutable_data
+        for attr in attrs:
+            if attr not in data:
+                return False, attr
+            data = data.get(attr)
+
+        return True, None

deckhand/errors.py

@@ -18,4 +18,4 @@ class ApiError(Exception):
 
 
 class InvalidFormat(ApiError):
-    pass
+    """The YAML file is incorrectly formatted and cannot be read."""

deckhand/tests/unit/control/test_api.py

@@ -0,0 +1,36 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import mock
+
+import testtools
+
+from deckhand.control import api
+from deckhand.control import base as api_base
+
+
+class TestApi(testtools.TestCase):
+
+    @mock.patch.object(api, 'secrets', autospec=True)
+    @mock.patch.object(api, 'falcon', autospec=True)
+    def test_start_api(self, mock_falcon, mock_secrets):
+        mock_falcon_api = mock_falcon.API.return_value
+
+        result = api.start_api()
+        self.assertEqual(mock_falcon_api, result)
+
+        mock_falcon.API.assert_called_once_with(
+            request_type=api_base.DeckhandRequest)
+        mock_falcon_api.add_route.assert_called_once_with(
+            '/api/v1.0/secrets', mock_secrets.SecretsResource())

deckhand/tests/unit/control/test_base.py

@@ -0,0 +1,40 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import mock
+
+import testtools
+
+from deckhand.control import base as api_base
+
+
+class TestBaseResource(testtools.TestCase):
+
+    def setUp(self):
+        super(TestBaseResource, self).setUp()
+        self.base_resource = api_base.BaseResource()
+
+    def test_on_options(self):
+        # Override `dir` so that ``dir(self)`` returns `methods`.
+        expected_methods = ['on_get', 'on_heat', 'on_post', 'on_put',
+                            'on_delete', 'on_patch']
+        api_base.BaseResource.__dir__ = lambda x: expected_methods
+
+        mock_resp = mock.Mock(headers={})
+        self.base_resource.on_options(None, mock_resp)
+
+        self.assertIn('Allow', mock_resp.headers)
+        self.assertEqual('GET,POST,PUT,DELETE,PATCH',
+                         mock_resp.headers['Allow'])
+        self.assertEqual('200 OK', mock_resp.status)

deckhand/tests/unit/engine/test_secret_substitution.py

@@ -0,0 +1,112 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import testtools
+
+from oslo_serialization import jsonutils as json
+import six
+
+from deckhand.engine import secret_substitution
+from deckhand import errors
+
+
+class TestSecretSubtitution(testtools.TestCase):
+
+    def setUp(self):
+        super(TestSecretSubtitution, self).setUp()
+        dir_path = os.path.dirname(os.path.realpath(__file__))
+        test_yaml_path = os.path.abspath(os.path.join(
+            dir_path, os.pardir, 'resources', 'sample.yaml'))
+
+        with open(test_yaml_path, 'r') as yaml_data:
+            self.yaml_data = yaml_data.read()
+
+    def test_initialization_missing_substitutions_section(self):
+        expected_err = (
+            "The provided YAML file has no metadata/substitutions section")
+        invalid_data = [
+            {"data": []},
+            {"data": [], "metadata": None},
+            {"data": [], "metadata": {"missing_substitutions": None}}
+        ]
+
+        for invalid_entry in invalid_data:
+            invalid_entry = json.dumps(invalid_entry)
+            with six.assertRaisesRegex(self, errors.InvalidFormat,
+                                       expected_err):
+                secret_substitution.SecretSubstitution(invalid_entry)
+
+        expected_err = (
+            "The provided YAML file has no metadata/substitutions section")
+        invalid_data = [
+            {"data": [], "metadata": None},
+        ]
+
+    def test_initialization_missing_data_section(self):
+        expected_err = (
+            "The provided YAML file has no data section")
+        invalid_data = '{"metadata": {"substitutions": []}}'
+
+        with six.assertRaisesRegex(self, errors.InvalidFormat, expected_err):
+            secret_substitution.SecretSubstitution(invalid_data)
+        
+    def test_initialization_missing_src_dest_sections(self):
+        expected_err = ('The provided YAML file is missing the "%s" field for '
+                        'the %s substition.')
+        invalid_data = [
+            {"data": [], "metadata": {"substitutions": [{"dest": "foo"}]}},
+            {"data": [], "metadata": {"substitutions": [{"src": "bar"}]}},
+        ]
+
+        def _test(invalid_entry, field, substitution):
+            invalid_entry = json.dumps(invalid_entry)
+            _expected_err = expected_err % (field, substitution)
+
+            with six.assertRaisesRegex(self, errors.InvalidFormat,
+                                       _expected_err):
+                secret_substitution.SecretSubstitution(invalid_entry)
+
+        _test(invalid_data[0], "src", {"dest": "foo"})
+        _test(invalid_data[1], "dest", {"src": "bar"})
+
+    def test_initialization_bad_substitutions(self):
+        expected_err = ('The attribute "%s" included in the "dest" field "%s" '
+                        'is missing from the YAML data: "%s".')
+        invalid_data = [
+            # Missing attribute.
+            {"data": {}, "metadata": {"substitutions": [
+                {"src": "", "dest": "foo"}
+            ]}},
+            # Missing attribute.
+            {"data": {"foo": None}, "metadata": {"substitutions": [
+                {"src": "", "dest": "bar"}
+            ]}},
+            # Missing nested attribute.
+            {"data": {"foo": {"baz": None}}, "metadata": {"substitutions": [
+                {"src": "", "dest": "foo.bar"}
+            ]}},
+        ]
+
+        def _test(invalid_entry, field, dest, substitution):
+            invalid_entry = json.dumps(invalid_entry)
+            _expected_err = expected_err % (field, dest, substitution)
+
+            with six.assertRaisesRegex(self, errors.InvalidFormat,
+                                       _expected_err):
+                secret_substitution.SecretSubstitution(invalid_entry)
+
+        _test(invalid_data[0], "foo", "foo", {})
+        _test(invalid_data[1], "bar", "bar", {"foo": None})
+        _test(invalid_data[2], "bar", "foo.bar", {'foo': {'baz': None}})

deckhand/tests/unit/resources/sample.yaml

@@ -0,0 +1,23 @@
+# Sample YAML file for testing forward replacement.
+---
+apiVersion: service/v1
+kind: ConsumerOfCertificateData
+metadata:
+  name: asdf-1234
+  storage: cleartext
+  substitutions:
+    - dest: .tls_endpoint.certificate
+      src:
+        apiVersion: deckhand/v1
+        kind: Certificate
+        name: some-certificate-asdf-1234
+    - dest: .tls_endpoint.certificateKey
+      src:
+        apiVersion: deckhand/v1
+        kind: CertificateKey
+        name: some-certificate-key-asdf-1234
+data:
+  tls_endpoint:
+      uri: http://localhost:443
+      certificate: null
+      certificateKey: null

test-requirements.txt

@@ -1 +1,11 @@
-falcon==1.1.0
+falcon==1.1.0
+
+mock>=2.0
+fixtures>=3.0.0 # Apache-2.0/BSD
+mock>=2.0 # BSD
+mox3!=0.19.0,>=0.7.0 # Apache-2.0
+python-subunit>=0.0.18 # Apache-2.0/BSD
+oslotest>=1.10.0 # Apache-2.0
+os-testr>=0.8.0 # Apache-2.0
+testrepository>=0.0.18 # Apache-2.0/BSD
+testtools>=1.4.0 # MIT

tox.ini

@@ -1,15 +1,33 @@
 [tox]
-envlist = py35,py27,pep8
+envlist = py{35,27},pep8
 
 [testenv]
 usedevelop = True
+whitelist_externals = bash
+                      find
+                      rm
+                      env
+                      flake8
 setenv = VIRTUAL_ENV={envdir}
+         OS_TEST_PATH=./deckhand/tests/unit
          LANGUAGE=en_US
          LC_ALL=en_US.utf-8
-deps=
-  -r{toxinidir}/requirements.txt
-  -r{toxinidir}/test-requirements.txt
-whitelist_externals = flake8
+passenv = OS_STDOUT_CAPTURE OS_STDERR_CAPTURE OS_TEST_TIMEOUT OS_TEST_LOCK_PATH OS_TEST_PATH http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY
+deps = -r{toxinidir}/requirements.txt
+       -r{toxinidir}/test-requirements.txt
+commands =
+  find . -type f -name "*.pyc" -delete
+  rm -Rf .testrepository/times.dbm
+
+[testenv:py27]
+commands =
+    {[testenv]commands}
+    ostestr '{posargs}'
+
+[testenv:py35]
+commands =
+    {[testenv]commands}
+    ostestr '{posargs}'
 
 [testenv:genconfig]
 commands = oslo-config-generator --config-file=etc/deckhand/config-generator.conf