From c9cdd7514c17bdf237be4ed32d84273546660bcb Mon Sep 17 00:00:00 2001 From: Felipe Monteiro Date: Sun, 13 Aug 2017 21:53:55 -0400 Subject: [PATCH] [feat] DECKHAND-38: Secrets DB model and secrets manager. This commit adds a DocumentSecret model to the DB for storing secrets directly in Deckhand as well as references to secrets stored in Barbican if the encryption type for the secret is encrypted. This commit also adds a new class called SecretsManager for managing the lifecycle of secrets from a higher level. This commit also adds Postgres compliance. So now all the DB models should work with Postgres. Also includes unit tests. Change-Id: Id7c4be8de2e70735f42b1f6710139d553ab4bea2 --- deckhand/barbican/client_wrapper.py | 13 +- deckhand/barbican/driver.py | 25 ++- deckhand/control/api.py | 5 +- deckhand/control/buckets.py | 10 +- deckhand/control/common.py | 11 -- deckhand/control/secrets.py | 55 ------ deckhand/control/views/revision.py | 3 +- deckhand/db/sqlalchemy/api.py | 19 +- deckhand/db/sqlalchemy/models.py | 31 ++-- deckhand/engine/secrets_manager.py | 86 ++++++++++ deckhand/errors.py | 6 + deckhand/factories.py | 60 ++++++- deckhand/tests/test_utils.py | 44 +++-- deckhand/tests/unit/base.py | 31 ++++ deckhand/tests/unit/control/test_api.py | 8 +- deckhand/tests/unit/db/test_documents.py | 162 +++++++++++++++--- deckhand/tests/unit/db/test_revision_tags.py | 3 +- .../tests/unit/engine/test_secrets_manager.py | 73 ++++++++ .../tests/unit/views/test_document_views.py | 3 + deckhand/types.py | 19 +- deckhand/utils.py | 15 ++ 21 files changed, 524 insertions(+), 158 deletions(-) delete mode 100644 deckhand/control/secrets.py create mode 100644 deckhand/engine/secrets_manager.py create mode 100644 deckhand/tests/unit/engine/test_secrets_manager.py diff --git a/deckhand/barbican/client_wrapper.py b/deckhand/barbican/client_wrapper.py index 5f418829..c8df3db4 100644 --- a/deckhand/barbican/client_wrapper.py +++ b/deckhand/barbican/client_wrapper.py @@ -46,16 +46,13 @@ class BarbicanClientWrapper(object): # TODO(fmontei): Deckhand's configuration file needs to be populated # with correct Keysone authentication values as well as the Barbican # endpoint URL automatically. - barbican_url = (CONF.barbican.api_endpoint - if CONF.barbican.api_endpoint - else 'http://127.0.0.1:9311') + barbican_url = CONF.barbican.api_endpoint keystone_auth = dict(CONF.keystone_authtoken) auth = v3.Password(**keystone_auth) sess = session.Session(auth=auth) try: - # TODO(fmontei): replace with ``barbican_url``. cli = barbican.client.Client(endpoint=barbican_url, session=sess) # Cache the client so we don't have to reconstruct and @@ -63,10 +60,10 @@ class BarbicanClientWrapper(object): if retry_on_conflict: self._cached_client = cli - except barbican_exc.HTTPAuthError: - msg = _("Unable to authenticate Barbican client.") - # TODO(fmontei): Log the error. - raise errors.ApiError(msg) + except barbican_exc.HTTPAuthError as e: + LOG.exception(e.message) + raise errors.BarbicanException(message=e.message, + code=e.status_code) return cli diff --git a/deckhand/barbican/driver.py b/deckhand/barbican/driver.py index d34e4d96..edef7eab 100644 --- a/deckhand/barbican/driver.py +++ b/deckhand/barbican/driver.py @@ -12,7 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. +import barbicanclient +from oslo_log import log as logging + from deckhand.barbican import client_wrapper +from deckhand import errors +from deckhand import utils + +LOG = logging.getLogger(__name__) class BarbicanDriver(object): @@ -22,4 +29,20 @@ class BarbicanDriver(object): def create_secret(self, **kwargs): """Create a secret.""" - return self.barbicanclient.call("secrets.create", **kwargs) + secret = self.barbicanclient.call("secrets.create", **kwargs) + + try: + secret.store() + except (barbicanclient.exceptions.HTTPAuthError, + barbicanclient.exceptions.HTTPClientError, + barbicanclient.exceptions.HTTPServerError) as e: + LOG.exception(e.message) + raise errors.BarbicanException(message=e.message, + code=e.status_code) + + # NOTE(fmontei): The dictionary representation of the Secret object by + # default has keys that are not snake case -- so make them snake case. + resp = secret.to_dict() + for key in resp.keys(): + resp[utils.to_snake_case(key)] = resp.pop(key) + return resp diff --git a/deckhand/control/api.py b/deckhand/control/api.py index fc2b7559..eaf3dd24 100644 --- a/deckhand/control/api.py +++ b/deckhand/control/api.py @@ -25,7 +25,6 @@ from deckhand.control import middleware from deckhand.control import revision_documents from deckhand.control import revision_tags from deckhand.control import revisions -from deckhand.control import secrets from deckhand.db.sqlalchemy import api as db_api CONF = cfg.CONF @@ -102,9 +101,7 @@ def start_api(state_manager=None): revision_documents.RevisionDocumentsResource()), ('revisions/{revision_id}/tags', revision_tags.RevisionTagsResource()), ('revisions/{revision_id}/tags/{tag}', - revision_tags.RevisionTagsResource()), - # TODO(fmontei): remove in follow-up commit. - ('secrets', secrets.SecretsResource()) + revision_tags.RevisionTagsResource()) ] for path, res in v1_0_routes: diff --git a/deckhand/control/buckets.py b/deckhand/control/buckets.py index 578e0a3d..05ccad1e 100644 --- a/deckhand/control/buckets.py +++ b/deckhand/control/buckets.py @@ -23,7 +23,9 @@ from deckhand.control import base as api_base from deckhand.control.views import document as document_view from deckhand.db.sqlalchemy import api as db_api from deckhand.engine import document_validation +from deckhand.engine import secrets_manager from deckhand import errors as deckhand_errors +from deckhand import types LOG = logging.getLogger(__name__) @@ -32,10 +34,10 @@ class BucketsResource(api_base.BaseResource): """API resource for realizing CRUD operations for buckets.""" view_builder = document_view.ViewBuilder() + secrets_mgr = secrets_manager.SecretsManager() def on_put(self, req, resp, bucket_name=None): document_data = req.stream.read(req.content_length or 0) - try: documents = list(yaml.safe_load_all(document_data)) except yaml.YAMLError as e: @@ -52,6 +54,12 @@ class BucketsResource(api_base.BaseResource): except (deckhand_errors.InvalidDocumentFormat) as e: raise falcon.HTTPBadRequest(description=e.format_message()) + for document in documents: + if any([document['schema'].startswith(t) + for t in types.DOCUMENT_SECRET_TYPES]): + secret_data = self.secrets_mgr.create(document) + document['data'] = secret_data + try: documents.extend(validation_policies) created_documents = db_api.documents_create(bucket_name, documents) diff --git a/deckhand/control/common.py b/deckhand/control/common.py index 711d00f1..ba8dbbfc 100644 --- a/deckhand/control/common.py +++ b/deckhand/control/common.py @@ -13,17 +13,6 @@ # limitations under the License. import functools -import string - -from oslo_log import log as logging - - -LOG = logging.getLogger(__name__) - - -def to_camel_case(s): - return (s[0].lower() + string.capwords(s, sep='_').replace('_', '')[1:] - if s else s) class ViewBuilder(object): diff --git a/deckhand/control/secrets.py b/deckhand/control/secrets.py deleted file mode 100644 index d5207331..00000000 --- a/deckhand/control/secrets.py +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright 2017 AT&T Intellectual Property. All other rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import falcon - -from oslo_serialization import jsonutils as json - -from deckhand.barbican import driver -from deckhand.control import base as api_base - - -class SecretsResource(api_base.BaseResource): - """API resource for interacting with Barbican. - - NOTE: Currently only supports Barbican. - """ - - def __init__(self, **kwargs): - super(SecretsResource, self).__init__(**kwargs) - self.authorized_roles = ['user'] - self.barbican_driver = driver.BarbicanDriver() - - def on_post(self, req, resp): - """Create a secret. - - :param name: The name of the secret. Required. - :param type: The type of the secret. Optional. - - For a list of types, please refer to the following API documentation: - https://docs.openstack.org/barbican/latest/api/reference/secret_types.html - """ - secret_name = req.params.get('name') - secret_type = req.params.get('type') - - if not secret_name: - resp.status = falcon.HTTP_400 - - # Do not allow users to call Barbican with all permitted kwargs. - # Selectively include only what we allow. - kwargs = {'name': secret_name, 'secret_type': secret_type} - secret = self.barbican_driver.create_secret(**kwargs) - - resp.body = json.dumps(secret) - resp.status = falcon.HTTP_200 diff --git a/deckhand/control/views/revision.py b/deckhand/control/views/revision.py index 8406e7b0..2b735bc4 100644 --- a/deckhand/control/views/revision.py +++ b/deckhand/control/views/revision.py @@ -14,6 +14,7 @@ from deckhand.control import common from deckhand import types +from deckhand import utils class ViewBuilder(common.ViewBuilder): @@ -32,7 +33,7 @@ class ViewBuilder(common.ViewBuilder): rev_documents = revision.pop('documents') for attr in ('id', 'created_at'): - body[common.to_camel_case(attr)] = revision[attr] + body[utils.to_camel_case(attr)] = revision[attr] body['tags'].update([t['tag'] for t in revision['tags']]) body['buckets'].update( diff --git a/deckhand/db/sqlalchemy/api.py b/deckhand/db/sqlalchemy/api.py index 788bc603..514c841f 100644 --- a/deckhand/db/sqlalchemy/api.py +++ b/deckhand/db/sqlalchemy/api.py @@ -80,14 +80,16 @@ def clear_db_env(): _FACADE = None -def setup_db(): - models.register_models(get_engine()) - - def drop_db(): models.unregister_models(get_engine()) +def setup_db(): + # Ensure the DB doesn't exist before creation. + drop_db() + models.register_models(get_engine()) + + def documents_create(bucket_name, documents, session=None): session = session or get_session() documents_created = _documents_create(documents, session) @@ -138,14 +140,7 @@ def _documents_create(values_list, session=None): for values in values_list: values['_metadata'] = values.pop('metadata') values['name'] = values['_metadata']['name'] - - # NOTE(fmontei): Database requires that the 'data' column be a dict, so - # coerce the secret into a dictionary if it already isn't one. - if values['schema'] in (types.CERTIFICATE_SCHEMA, - types.CERTIFICATE_KEY_SCHEMA, - types.PASSPHRASE_SCHEMA): - if not isinstance(values['data'], dict): - values['data'] = {'secret': values['data']} + values['is_secret'] = 'secret' in values['data'] try: existing_document = document_get( diff --git a/deckhand/db/sqlalchemy/models.py b/deckhand/db/sqlalchemy/models.py index 3a5b3348..1ec31bba 100644 --- a/deckhand/db/sqlalchemy/models.py +++ b/deckhand/db/sqlalchemy/models.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import uuid - from oslo_db.sqlalchemy import models from oslo_db.sqlalchemy import types as oslo_types from oslo_utils import timeutils @@ -26,7 +24,6 @@ from sqlalchemy import Integer from sqlalchemy.orm import relationship from sqlalchemy import schema from sqlalchemy import String -from sqlalchemy import Unicode # Declarative base class which maintains a catalog of classes and tables @@ -90,12 +87,12 @@ class DeckhandBase(models.ModelBase, models.TimestampMixin): return d - @staticmethod - def gen_unqiue_contraint(*fields): - constraint_name = 'ix_' + DeckhandBase.__name__.lower() + '_' - for field in fields: - constraint_name = constraint_name + '_%s' % field - return schema.UniqueConstraint(*fields, name=constraint_name) + +def gen_unique_constraint(table_name, *fields): + constraint_name = 'ix_' + table_name.lower() + for field in fields: + constraint_name = constraint_name + '_%s' % field + return schema.UniqueConstraint(*fields, name=constraint_name) class Bucket(BASE, DeckhandBase): @@ -108,8 +105,7 @@ class Bucket(BASE, DeckhandBase): class Revision(BASE, DeckhandBase): __tablename__ = 'revisions' - id = Column(String(36), primary_key=True, - default=lambda: str(uuid.uuid4())) + id = Column(Integer, primary_key=True) documents = relationship("Document") tags = relationship("RevisionTag") @@ -123,9 +119,10 @@ class Revision(BASE, DeckhandBase): class RevisionTag(BASE, DeckhandBase): UNIQUE_CONSTRAINTS = ('tag', 'revision_id') __tablename__ = 'revision_tags' - __table_args__ = (DeckhandBase.gen_unqiue_contraint(*UNIQUE_CONSTRAINTS),) + __table_args__ = ( + gen_unique_constraint(__tablename__, *UNIQUE_CONSTRAINTS),) - tag = Column(Unicode(80), primary_key=True, nullable=False) + tag = Column(String(64), primary_key=True, nullable=False) data = Column(oslo_types.JsonEncodedDict(), nullable=True, default={}) revision_id = Column( Integer, ForeignKey('revisions.id', ondelete='CASCADE'), @@ -135,17 +132,17 @@ class RevisionTag(BASE, DeckhandBase): class Document(BASE, DeckhandBase): UNIQUE_CONSTRAINTS = ('schema', 'name', 'revision_id') __tablename__ = 'documents' - __table_args__ = (DeckhandBase.gen_unqiue_contraint(*UNIQUE_CONSTRAINTS),) + __table_args__ = (gen_unique_constraint(*UNIQUE_CONSTRAINTS),) - id = Column(String(36), primary_key=True, - default=lambda: str(uuid.uuid4())) + id = Column(Integer, primary_key=True) name = Column(String(64), nullable=False) schema = Column(String(64), nullable=False) # NOTE: Do not define a maximum length for these JSON data below. However, # this approach is not compatible with all database types. # "metadata" is reserved, so use "_metadata" instead. _metadata = Column(oslo_types.JsonEncodedDict(), nullable=False) - data = Column(oslo_types.JsonEncodedDict(), nullable=False) + data = Column(oslo_types.JsonEncodedDict(), nullable=True) + is_secret = Column(Boolean, nullable=False, default=False) bucket_id = Column(Integer, ForeignKey('buckets.name', ondelete='CASCADE'), nullable=False) diff --git a/deckhand/engine/secrets_manager.py b/deckhand/engine/secrets_manager.py new file mode 100644 index 00000000..a4179fce --- /dev/null +++ b/deckhand/engine/secrets_manager.py @@ -0,0 +1,86 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from deckhand.barbican import driver + +CLEARTEXT = 'cleartext' +ENCRYPTED = 'encrypted' + + +class SecretsManager(object): + """Internal API resource for interacting with Barbican. + + Currently only supports Barbican. + """ + + barbican_driver = driver.BarbicanDriver() + + def create(self, secret_doc): + """Securely store secrets contained in ``secret_doc``. + + Ordinarily, Deckhand documents are stored directly in Deckhand's + database. However, secret data (contained in the data section for the + documents with the schemas enumerated below) must be stored using a + secure storage service like Barbican. + + Documents with metadata.storagePolicy == "clearText" have their secrets + stored directly in Deckhand. + + Documents with metadata.storagePolicy == "encrypted" are stored in + Barbican directly. Deckhand in turn stores the reference returned + by Barbican in Deckhand. + + :param secret_doc: A Deckhand document with one of the following + schemas: + + * deckhand/Certificate/v1 + * deckhand/CertificateKey/v1 + * deckhand/Passphrase/v1 + + :returns: Dictionary representation of + `deckhand.db.sqlalchemy.models.DocumentSecret`. + """ + encryption_type = secret_doc['metadata']['storagePolicy'] + secret_type = self._get_secret_type(secret_doc['schema']) + + if encryption_type == ENCRYPTED: + # Store secret_ref in database for `secret_doc`. + kwargs = { + 'name': secret_doc['metadata']['name'], + 'secret_type': secret_type, + 'payload': secret_doc['data'] + } + resp = self.barbican_driver.create_secret(**kwargs) + + secret_ref = resp['secret_href'] + created_secret = {'secret': secret_ref} + elif encryption_type == CLEARTEXT: + created_secret = {'secret': secret_doc['data']} + + return created_secret + + def _get_secret_type(self, schema): + """Get the Barbican secret type based on the following mapping: + + deckhand/Certificate/v1 => certificate + deckhand/CertificateKey/v1 => private + deckhand/Passphrase/v1 => passphrase + + :param schema: The document's schema. + :returns: The value corresponding to the mapping above. + """ + _schema = schema.split('/')[1].lower().strip() + if _schema == 'certificatekey': + return 'private' + return _schema diff --git a/deckhand/errors.py b/deckhand/errors.py index 2cbac3db..2fe9c3cb 100644 --- a/deckhand/errors.py +++ b/deckhand/errors.py @@ -125,3 +125,9 @@ class RevisionTagBadFormat(DeckhandException): msg_fmt = ("The requested tag data %(data)s must either be null or " "dictionary.") code = 400 + + +class BarbicanException(DeckhandException): + + def __init__(self, message, code): + super(BarbicanException, self).__init__(message=message, code=code) diff --git a/deckhand/factories.py b/deckhand/factories.py index 8e0b5ac2..73ab1b3c 100644 --- a/deckhand/factories.py +++ b/deckhand/factories.py @@ -240,6 +240,65 @@ class DocumentFactory(DeckhandFactory): return rendered_template +class DocumentSecretFactory(DeckhandFactory): + """Class for auto-generating document secrets templates for testing. + + Returns formats that adhere to the following supported schemas: + + * deckhand/Certificate/v1 + * deckhand/CertificateKey/v1 + * deckhand/Passphrase/v1 + """ + + DOCUMENT_SECRET_TEMPLATE = { + "data": { + }, + "metadata": { + "schema": "metadata/Document/v1", + "name": "application-api", + "storagePolicy": "" + }, + "schema": "deckhand/%s/v1" + } + + def __init__(self): + """Constructor for ``DocumentSecretFactory``. + + Returns a template whose YAML representation is of the form:: + + --- + schema: deckhand/Certificate/v1 + metadata: + schema: metadata/Document/v1 + name: application-api + storagePolicy: cleartext + data: |- + -----BEGIN CERTIFICATE----- + MIIDYDCCAkigAwIBAgIUKG41PW4VtiphzASAMY4/3hL8OtAwDQYJKoZIhvcNAQEL + ...snip... + P3WT9CfFARnsw2nKjnglQcwKkKLYip0WY2wh3FE7nrQZP6xKNaSRlh6p2pCGwwwH + HkvVwA== + -----END CERTIFICATE----- + ... + """ + + def gen(self): + pass + + def gen_test(self, schema, storage_policy, data=None): + if data is None: + data = test_utils.rand_password() + + document_secret_template = copy.deepcopy(self.DOCUMENT_SECRET_TEMPLATE) + + document_secret_template['metadata']['storagePolicy'] = storage_policy + document_secret_template['schema'] = ( + document_secret_template['schema'] % schema) + document_secret_template['data'] = data + + return document_secret_template + + class ValidationPolicyFactory(DeckhandFactory): """Class for auto-generating validation policy templates for testing.""" @@ -274,7 +333,6 @@ class ValidationPolicyFactory(DeckhandFactory): - name: armada-deployability-validation ... """ - pass def gen(self, validation_type, status): if validation_type not in types.DECKHAND_VALIDATION_TYPES: diff --git a/deckhand/tests/test_utils.py b/deckhand/tests/test_utils.py index 8a0e9f39..3f0b123f 100644 --- a/deckhand/tests/test_utils.py +++ b/deckhand/tests/test_utils.py @@ -1,18 +1,20 @@ -# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# Copyright 2016 OpenStack Foundation +# All Rights Reserved. # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. import random +import string import uuid @@ -60,3 +62,25 @@ def rand_int(min, max): :rtype: integer """ return random.randint(min, max) + + +def rand_password(length=15): + """Generate a random password + :param int length: The length of password that you expect to set + (If it's smaller than 3, it's same as 3.) + :return: a random password. The format is + '-- + -' + (e.g. 'G2*ac8&lKFFgh%2') + :rtype: string + """ + upper = random.choice(string.ascii_uppercase) + ascii_char = string.ascii_letters + digits = string.digits + digit = random.choice(string.digits) + puncs = '~!@#%^&*_=+' + punc = random.choice(puncs) + seed = ascii_char + digits + puncs + pre = upper + digit + punc + password = pre + ''.join(random.choice(seed) for x in range(length - 3)) + return password diff --git a/deckhand/tests/unit/base.py b/deckhand/tests/unit/base.py index 0f4db506..92e3e860 100644 --- a/deckhand/tests/unit/base.py +++ b/deckhand/tests/unit/base.py @@ -13,6 +13,7 @@ # limitations under the License. import fixtures +import mock from oslo_config import cfg from oslo_log import log as logging import testtools @@ -40,6 +41,36 @@ class DeckhandTestCase(testtools.TestCase): elif isinstance(collection, dict): self.assertEqual(0, len(collection.keys())) + def patch(self, target, autospec=True, **kwargs): + """Returns a started `mock.patch` object for the supplied target. + + The caller may then call the returned patcher to create a mock object. + + The caller does not need to call stop() on the returned + patcher object, as this method automatically adds a cleanup + to the test class to stop the patcher. + + :param target: String module.class or module.object expression to patch + :param **kwargs: Passed as-is to `mock.patch`. See mock documentation + for details. + """ + p = mock.patch(target, autospec=autospec, **kwargs) + m = p.start() + self.addCleanup(p.stop) + return m + + def patchobject(self, target, attribute, new=mock.DEFAULT, autospec=True): + """Convenient wrapper around `mock.patch.object` + + Returns a started mock that will be automatically stopped after the + test ran. + """ + + p = mock.patch.object(target, attribute, new, autospec=autospec) + m = p.start() + self.addCleanup(p.stop) + return m + class DeckhandWithDBTestCase(DeckhandTestCase): diff --git a/deckhand/tests/unit/control/test_api.py b/deckhand/tests/unit/control/test_api.py index c63aa286..0c1fa765 100644 --- a/deckhand/tests/unit/control/test_api.py +++ b/deckhand/tests/unit/control/test_api.py @@ -20,7 +20,6 @@ from deckhand.control import buckets from deckhand.control import revision_documents from deckhand.control import revision_tags from deckhand.control import revisions -from deckhand.control import secrets from deckhand.tests.unit import base as test_base @@ -28,8 +27,8 @@ class TestApi(test_base.DeckhandTestCase): def setUp(self): super(TestApi, self).setUp() - for resource in (buckets, revision_documents, revision_tags, revisions, - secrets): + for resource in (buckets, revision_documents, revision_tags, + revisions): resource_name = resource.__name__.split('.')[-1] resource_obj = mock.patch.object( resource, '%sResource' % resource_name.title().replace( @@ -59,8 +58,7 @@ class TestApi(test_base.DeckhandTestCase): mock.call('/api/v1.0/revisions/{revision_id}/tags', self.revision_tags_resource()), mock.call('/api/v1.0/revisions/{revision_id}/tags/{tag}', - self.revision_tags_resource()), - mock.call('/api/v1.0/secrets', self.secrets_resource()) + self.revision_tags_resource()) ]) mock_config.parse_args.assert_called_once_with() mock_db_api.setup_db.assert_called_once_with() diff --git a/deckhand/tests/unit/db/test_documents.py b/deckhand/tests/unit/db/test_documents.py index a34d6271..dcbc02d7 100644 --- a/deckhand/tests/unit/db/test_documents.py +++ b/deckhand/tests/unit/db/test_documents.py @@ -12,52 +12,67 @@ # See the License for the specific language governing permissions and # limitations under the License. +from deckhand import factories from deckhand.tests import test_utils from deckhand.tests.unit.db import base class TestDocuments(base.TestDbBase): + def setUp(self): + super(TestDocuments, self).setUp() + # Will create 3 documents: layering policy, plus a global and site + # document. + self.secrets_factory = factories.DocumentSecretFactory() + self.documents_factory = factories.DocumentFactory(2, [1, 1]) + self.document_mapping = { + "_GLOBAL_DATA_1_": {"data": {"a": {"x": 1, "y": 2}}}, + "_SITE_DATA_1_": {"data": {"a": {"x": 7, "z": 3}, "b": 4}}, + "_SITE_ACTIONS_1_": { + "actions": [{"method": "merge", "path": "."}]} + } + def test_create_and_show_bucket(self): - payload = base.DocumentFixture.get_minimal_fixture() + payload = self.documents_factory.gen_test( + self.document_mapping) bucket_name = test_utils.rand_name('bucket') documents = self.create_documents(bucket_name, payload) self.assertIsInstance(documents, list) - self.assertEqual(1, len(documents)) + self.assertEqual(3, len(documents)) - retrieved_document = self.show_document(id=documents[0]['id']) - self.assertEqual(documents[0], retrieved_document) + for idx in range(len(documents)): + retrieved_document = self.show_document(id=documents[idx]['id']) + self.assertEqual(documents[idx], retrieved_document) + + def test_create_and_get_multiple_document(self): + payload = self.documents_factory.gen_test( + self.document_mapping) + bucket_name = test_utils.rand_name('bucket') + created_documents = self.create_documents(bucket_name, payload) + + self.assertIsInstance(created_documents, list) + self.assertEqual(3, len(created_documents)) def test_create_document_conflict(self): - payload = base.DocumentFixture.get_minimal_fixture() + payload = self.documents_factory.gen_test( + self.document_mapping) bucket_name = test_utils.rand_name('bucket') + self.create_documents(bucket_name, payload) - documents = self.create_documents(bucket_name, payload) + unchanged_documents = self.create_documents(bucket_name, payload) - self.assertIsInstance(documents, list) - self.assertEmpty(documents) - - def test_create_document_and_show_revision(self): - payload = base.DocumentFixture.get_minimal_fixture() - bucket_name = test_utils.rand_name('bucket') - documents = self.create_documents(bucket_name, payload) - - self.assertIsInstance(documents, list) - self.assertEqual(1, len(documents)) - - for document in documents: - revision = self.show_revision(document['revision_id']) - self.validate_revision(revision) - self.assertEqual(document['revision_id'], revision['id']) + self.assertIsInstance(unchanged_documents, list) + self.assertEmpty(unchanged_documents) def test_list_documents_by_revision_id(self): - payload = base.DocumentFixture.get_minimal_fixture() + payload = self.documents_factory.gen_test( + self.document_mapping) bucket_name = test_utils.rand_name('bucket') documents = self.create_documents(bucket_name, payload) revision = self.show_revision(documents[0]['revision_id']) - self.assertEqual(1, len(revision['documents'])) + self.assertEqual(3, len(revision['documents'])) self.assertEqual(documents[0], revision['documents'][0]) def test_list_multiple_documents_by_revision_id(self): @@ -74,20 +89,113 @@ class TestDocuments(base.TestDbBase): self.assertEqual(document['revision_id'], revision['id']) def test_list_documents_by_revision_id_and_filters(self): - payload = base.DocumentFixture.get_minimal_fixture() + payload = self.documents_factory.gen_test( + self.document_mapping) bucket_name = test_utils.rand_name('bucket') - document = self.create_documents(bucket_name, payload)[0] + document = self.create_documents(bucket_name, payload)[1] + filters = { 'schema': document['schema'], 'metadata.name': document['metadata']['name'], 'metadata.layeringDefinition.abstract': document['metadata']['layeringDefinition']['abstract'], 'metadata.layeringDefinition.layer': - document['metadata']['layeringDefinition']['layer'], - 'metadata.label': document['metadata']['label'] + document['metadata']['layeringDefinition']['layer'] } documents = self.list_revision_documents( document['revision_id'], **filters) self.assertEqual(1, len(documents)) self.assertEqual(document, documents[0]) + + def test_create_multiple_documents_and_get_revision(self): + payload = self.documents_factory.gen_test( + self.document_mapping) + bucket_name = test_utils.rand_name('bucket') + created_documents = self.create_documents(bucket_name, payload) + + self.assertIsInstance(created_documents, list) + self.assertEqual(3, len(created_documents)) + + # Validate that each document references the same revision. + revisions = set(d['revision_id'] for d in created_documents) + self.assertEqual(1, len(revisions)) + + # Validate that the revision is valid. + for document in created_documents: + revision = self.show_revision(document['revision_id']) + self.assertEqual(3, len(revision['documents'])) + self.assertIn(document, revision['documents']) + self.assertEqual(document['revision_id'], revision['id']) + + def test_get_documents_by_revision_id_and_filters(self): + payload = self.documents_factory.gen_test( + self.document_mapping) + bucket_name = test_utils.rand_name('bucket') + created_documents = self.create_documents(bucket_name, payload) + + for document in created_documents[1:]: + filters = { + 'schema': document['schema'], + 'metadata.name': document['metadata']['name'], + 'metadata.layeringDefinition.abstract': + document['metadata']['layeringDefinition']['abstract'], + 'metadata.layeringDefinition.layer': + document['metadata']['layeringDefinition']['layer'] + } + filtered_documents = self.list_revision_documents( + document['revision_id'], **filters) + + self.assertEqual(1, len(filtered_documents)) + self.assertEqual(document, filtered_documents[0]) + + def test_create_certificate(self): + rand_secret = {'secret': test_utils.rand_password()} + bucket_name = test_utils.rand_name('bucket') + + for storage_policy in ('encrypted', 'cleartext'): + secret_doc_payload = self.secrets_factory.gen_test( + 'Certificate', storage_policy, rand_secret) + created_documents = self.create_documents( + bucket_name, secret_doc_payload) + + self.assertEqual(1, len(created_documents)) + self.assertIn('Certificate', created_documents[0]['schema']) + self.assertEqual(storage_policy, created_documents[0][ + 'metadata']['storagePolicy']) + self.assertTrue(created_documents[0]['is_secret']) + self.assertEqual(rand_secret, created_documents[0]['data']) + + def test_create_certificate_key(self): + rand_secret = {'secret': test_utils.rand_password()} + bucket_name = test_utils.rand_name('bucket') + + for storage_policy in ('encrypted', 'cleartext'): + secret_doc_payload = self.secrets_factory.gen_test( + 'CertificateKey', storage_policy, rand_secret) + created_documents = self.create_documents( + bucket_name, secret_doc_payload) + + self.assertEqual(1, len(created_documents)) + self.assertIn('CertificateKey', created_documents[0]['schema']) + self.assertEqual(storage_policy, created_documents[0][ + 'metadata']['storagePolicy']) + self.assertTrue(created_documents[0]['is_secret']) + self.assertEqual(rand_secret, created_documents[0]['data']) + + def test_create_passphrase(self): + rand_secret = {'secret': test_utils.rand_password()} + bucket_name = test_utils.rand_name('bucket') + + for storage_policy in ('encrypted', 'cleartext'): + secret_doc_payload = self.secrets_factory.gen_test( + 'Passphrase', storage_policy, rand_secret) + created_documents = self.create_documents( + bucket_name, secret_doc_payload) + + self.assertEqual(1, len(created_documents)) + self.assertIn('Passphrase', created_documents[0]['schema']) + self.assertEqual(storage_policy, created_documents[0][ + 'metadata']['storagePolicy']) + self.assertTrue(created_documents[0]['is_secret']) + self.assertEqual(rand_secret, created_documents[0]['data']) diff --git a/deckhand/tests/unit/db/test_revision_tags.py b/deckhand/tests/unit/db/test_revision_tags.py index 69cb6e64..459c043e 100644 --- a/deckhand/tests/unit/db/test_revision_tags.py +++ b/deckhand/tests/unit/db/test_revision_tags.py @@ -77,8 +77,7 @@ class TestRevisionTags(base.TestDbBase): for idx, tag in enumerate(tags): expected_tag_names = sorted(tags[idx + 1:]) - result = db_api.revision_tag_delete(self.revision_id, tag) - self.assertIsNone(result) + db_api.revision_tag_delete(self.revision_id, tag) retrieved_tags = db_api.revision_tag_get_all(self.revision_id) retrieved_tag_names = [t['tag'] for t in retrieved_tags] diff --git a/deckhand/tests/unit/engine/test_secrets_manager.py b/deckhand/tests/unit/engine/test_secrets_manager.py new file mode 100644 index 00000000..8c5f59a6 --- /dev/null +++ b/deckhand/tests/unit/engine/test_secrets_manager.py @@ -0,0 +1,73 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from deckhand.engine import secrets_manager +from deckhand import factories +from deckhand.tests import test_utils +from deckhand.tests.unit.db import base + + +class TestSecretsManager(base.TestDbBase): + + def setUp(self): + super(TestSecretsManager, self).setUp() + self.mock_barbican_driver = self.patchobject( + secrets_manager.SecretsManager, 'barbican_driver') + self.secret_ref = 'https://path/to/fake_secret' + self.mock_barbican_driver.create_secret.return_value = ( + {'secret_href': self.secret_ref}) + + self.secrets_manager = secrets_manager.SecretsManager() + self.factory = factories.DocumentSecretFactory() + + def _test_create_secret(self, encryption_type, secret_type): + secret_data = test_utils.rand_password() + secret_doc = self.factory.gen_test( + secret_type, encryption_type, secret_data) + + created_secret = self.secrets_manager.create(secret_doc) + + if encryption_type == 'cleartext': + self.assertIn('secret', created_secret) + self.assertEqual(secret_data, created_secret['secret']) + elif encryption_type == 'encrypted': + expected_kwargs = { + 'name': secret_doc['metadata']['name'], + 'secret_type': ('private' if secret_type == 'CertificateKey' + else secret_type.lower()), + 'payload': secret_doc['data'] + } + self.mock_barbican_driver.create_secret.assert_called_once_with( + **expected_kwargs) + + self.assertIn('secret', created_secret) + self.assertEqual(self.secret_ref, created_secret['secret']) + + def test_create_cleartext_certificate(self): + self._test_create_secret('cleartext', 'Certificate') + + def test_create_cleartext_certificate_key(self): + self._test_create_secret('cleartext', 'CertificateKey') + + def test_create_cleartext_passphrase(self): + self._test_create_secret('cleartext', 'Passphrase') + + def test_create_encrypted_certificate(self): + self._test_create_secret('encrypted', 'Certificate') + + def test_create_encrypted_certificate_key(self): + self._test_create_secret('encrypted', 'CertificateKey') + + def test_create_encrypted_passphrase(self): + self._test_create_secret('encrypted', 'Passphrase') diff --git a/deckhand/tests/unit/views/test_document_views.py b/deckhand/tests/unit/views/test_document_views.py index 82006db7..af2f873a 100644 --- a/deckhand/tests/unit/views/test_document_views.py +++ b/deckhand/tests/unit/views/test_document_views.py @@ -44,6 +44,9 @@ class TestRevisionViews(base.TestDbBase): for attr in ('bucket', 'revision'): self.assertIn(attr, document_view[idx]['status']) + revision_ids = set([v['status']['revision'] for v in document_view]) + self.assertEqual([1], list(revision_ids)) + def test_create_single_document(self): self._test_document_creation_view(1) diff --git a/deckhand/types.py b/deckhand/types.py index 5d37f8e3..6b057ced 100644 --- a/deckhand/types.py +++ b/deckhand/types.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +# TODO(fmontei): Make all these version-less. DOCUMENT_SCHEMA_TYPES = ( CERTIFICATE_SCHEMA, CERTIFICATE_KEY_SCHEMA, @@ -19,13 +20,25 @@ DOCUMENT_SCHEMA_TYPES = ( PASSPHRASE_SCHEMA, VALIDATION_POLICY_SCHEMA, ) = ( - 'deckhand/Certificate/v1', - 'deckhand/CertificateKey/v1', + 'deckhand/Certificate', + 'deckhand/CertificateKey', 'deckhand/LayeringPolicy/v1', - 'deckhand/Passphrase/v1', + 'deckhand/Passphrase', 'deckhand/ValidationPolicy/v1', ) + +DOCUMENT_SECRET_TYPES = ( + CERTIFICATE_KEY_SCHEMA, + CERTIFICATE_SCHEMA, + PASSPHRASE_SCHEMA +) = ( + 'deckhand/Certificate', + 'deckhand/CertificateKey', + 'deckhand/Passphrase' +) + + DECKHAND_VALIDATION_TYPES = ( DECKHAND_SCHEMA_VALIDATION, ) = ( diff --git a/deckhand/utils.py b/deckhand/utils.py index 0e44eefd..c3b51d49 100644 --- a/deckhand/utils.py +++ b/deckhand/utils.py @@ -12,6 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. +import re +import string + + +def to_camel_case(s): + """Convert string to camel case.""" + return (s[0].lower() + string.capwords(s, sep='_') + .replace('_', '')[1:] if s else s) + + +def to_snake_case(name): + """Convert string to snake case.""" + s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name) + return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower() + def multi_getattr(multi_key, dict_data): """Iteratively check for nested attributes in the YAML data.