From bbd0292455e693c6c6128dc218cc7ce043a701a5 Mon Sep 17 00:00:00 2001 From: Felipe Monteiro Date: Fri, 21 Jul 2017 04:22:09 +0100 Subject: [PATCH] Remove oslo_versionedobjects This commit also updates documentation for launching test app. --- ChangeLog | 10 ++ README.rst | 29 +++++- deckhand/control/README.rst | 2 +- deckhand/control/documents.py | 17 ++-- deckhand/db/sqlalchemy/api.py | 5 +- deckhand/db/sqlalchemy/models.py | 1 + deckhand/objects/__init__.py | 0 deckhand/objects/base.py | 165 ------------------------------- deckhand/objects/documents.py | 89 ----------------- deckhand/objects/fields.py | 33 ------- 10 files changed, 50 insertions(+), 301 deletions(-) delete mode 100644 deckhand/objects/__init__.py delete mode 100644 deckhand/objects/base.py delete mode 100644 deckhand/objects/documents.py delete mode 100644 deckhand/objects/fields.py diff --git a/ChangeLog b/ChangeLog index c96ebc7d..dbd52767 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,6 +1,16 @@ CHANGES ======= +* Change application/yaml to application/x-yaml +* Cleaned up some logic, added exception handling to document creation +* Add currently necessary oslo namespaces to oslo-config-generator conf file +* Successfully creating document +* Added logic for establishing DB connection +* Refactor database sqlalchemy api/models +* Added oslo\_context-based context for oslo\_db compatibility +* Update database documents schema +* Helper for generating versioned object automatically from dictionary payload +* Update README * Temporary change - do not commit * Initial DB API models implementation * Added control (API) readme diff --git a/README.rst b/README.rst index b24db09d..e62d2b16 100644 --- a/README.rst +++ b/README.rst @@ -3,11 +3,36 @@ Deckhand A foundational python REST YAML processing engine providing data and secrets management to other platform services. -To run:: +To generate a configuration file automatically:: + + $ tox -e genconfig + +Resulting deckhand.conf.sample file is output to +:path:etc/deckhand/deckhand.conf.sample + +Copy the config file to a directory discoverably by ``oslo.conf``:: + + $ cp etc/deckhand/deckhand.conf.sample ~/deckhand.conf + +To setup an in-memory database for testing: + +.. code-block:: ini + + [database] + + # + # From oslo.db + # + + # The SQLAlchemy connection string to use to connect to the database. + # (string value) + connection = sqlite:///:memory: + +To run locally in a development environment:: $ sudo pip install uwsgi $ virtualenv -p python3 /var/tmp/deckhand $ . /var/tmp/deckhand/bin/activate $ sudo pip install . - $ python setup.py install + $ sudo python setup.py install $ uwsgi --http :9000 -w deckhand.deckhand --callable deckhand_callable --enable-threads -L diff --git a/deckhand/control/README.rst b/deckhand/control/README.rst index 4b138d7d..a8b49c57 100644 --- a/deckhand/control/README.rst +++ b/deckhand/control/README.rst @@ -18,7 +18,7 @@ revision number will be returned. Testing ------- -Document creation can be tested locally using: +Document creation can be tested locally using (from root deckhand directory): .. code-block:: console diff --git a/deckhand/control/documents.py b/deckhand/control/documents.py index 8e3880f9..99476abc 100644 --- a/deckhand/control/documents.py +++ b/deckhand/control/documents.py @@ -12,16 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. +import copy import yaml import falcon +from oslo_db import exception as db_exc from oslo_log import log as logging from deckhand.control import base as api_base +from deckhand.db.sqlalchemy import api as db_api from deckhand.engine import document_validation from deckhand import errors as deckhand_errors -from deckhand.objects import documents LOG = logging.getLogger(__name__) @@ -61,17 +63,12 @@ class DocumentsResource(api_base.BaseResource): return self.return_error(resp, falcon.HTTP_400, message=e) try: - LOG.debug('Calling Document.create()') - documents.Document().create(document) + db_api.document_create(document) + except db_exc.DBDuplicateEntry as e: + return self.return_error(resp, falcon.HTTP_409, message=e) except Exception as e: - LOG.exception(e) - raise + return self.return_error(resp, falcon.HTTP_500, message=e) - # Check if a document with the specified name already exists. If so, - # treat this request as an update. - doc_name = doc_validation.doc_name - - resp.data = doc_name resp.status = falcon.HTTP_201 def _check_document_exists(self): diff --git a/deckhand/db/sqlalchemy/api.py b/deckhand/db/sqlalchemy/api.py index ea093ec2..e1d1ce7e 100644 --- a/deckhand/db/sqlalchemy/api.py +++ b/deckhand/db/sqlalchemy/api.py @@ -95,9 +95,12 @@ def clear_db_env(): _FACADE = None -def document_create(context, values, session=None): +def document_create(values, session=None): """Create a document.""" values = values.copy() + values['doc_metadata'] = values.pop('metadata') + values['schema_version'] = values.pop('schemaVersion') + session = session or get_session() with session.begin(): document = models.Document() diff --git a/deckhand/db/sqlalchemy/models.py b/deckhand/db/sqlalchemy/models.py index 3f95d7a3..f0d4d4f6 100644 --- a/deckhand/db/sqlalchemy/models.py +++ b/deckhand/db/sqlalchemy/models.py @@ -106,6 +106,7 @@ class Document(BASE, DeckhandBase): id = Column(String(36), primary_key=True, default=lambda: str(uuid.uuid4())) + # TODO: the revision_index will be a foreign key to a Revision table. revision_index = Column(String(36), nullable=False, default=lambda: str(uuid.uuid4())) schema_version = Column(String(64), nullable=False) diff --git a/deckhand/objects/__init__.py b/deckhand/objects/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/deckhand/objects/base.py b/deckhand/objects/base.py deleted file mode 100644 index 4977664f..00000000 --- a/deckhand/objects/base.py +++ /dev/null @@ -1,165 +0,0 @@ -# Copyright 2017 AT&T Intellectual Property. All other rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import datetime - -from oslo_log import log as logging -from oslo_versionedobjects import base -from oslo_versionedobjects import exception as ovo_exception -from oslo_versionedobjects import fields as ovo_fields - -from deckhand import objects - -LOG = logging.getLogger(__name__) - - -class DeckhandObjectRegistry(base.VersionedObjectRegistry): - - # Steal this from Cinder to bring all registered objects - # into the Deckhand_provisioner.objects namespace. - def registration_hook(self, cls, index): - setattr(objects, cls.obj_name(), cls) - - -class DeckhandObject(base.VersionedObject): - - # Version 1.0: Initial version - VERSION = '1.0' - - OBJ_PROJECT_NAMESPACE = 'deckhand.objects' - - def obj_load_attr(self, attrname): - if attrname in self.fields.keys(): - setattr(self, attrname, None) - else: - raise ValueError("Unknown field %s." % (attrname)) - - def obj_to_simple(self): - """ - Create a simple primitive representation of this object excluding - all the versioning stuff. Used to serialize an object for public - consumption, not intended to be deserialized by OVO. - """ - - primitive = dict() - - primitive['model_type'] = self.__class__.__name__ - primitive['model_version'] = self.VERSION - - for name, field in self.fields.items(): - if self.obj_attr_is_set(name): - value = getattr(self, name) - if (hasattr(value, 'obj_to_simple') and - callable(value.obj_to_simple)): - primitive[name] = value.obj_to_simple() - else: - value = field.to_primitive(self, name, value) - if value is not None: - primitive[name] = value - - return primitive - - -class DeckhandPersistentObject(base.VersionedObject): - - fields = { - 'created_at': ovo_fields.DateTimeField(nullable=False), - 'created_by': ovo_fields.StringField(nullable=False), - 'updated_at': ovo_fields.DateTimeField(nullable=True), - 'updated_by': ovo_fields.StringField(nullable=True), - } - - def set_create_fields(self, context): - self.created_at = datetime.datetime.now() - self.created_by = context.user - - def set_update_fields(self, context): - self.updated_at = datetime.datetime.now() - self.updated_by = context.user - - -class DeckhandPayloadBase(DeckhandPersistentObject): - """Base class for the payload of versioned notifications.""" - # SCHEMA defines how to populate the payload fields. It is a dictionary - # where every key value pair has the following format: - # : (, - # ) - # The is the name where the data will be stored in the - # payload object, this field has to be defined as a field of the payload. - # The shall refer to name of the parameter passed as - # kwarg to the payload's populate_schema() call and this object will be - # used as the source of the data. The shall be - # a valid field of the passed argument. - # The SCHEMA needs to be applied with the populate_schema() call before the - # notification can be emitted. - # The value of the payload. field will be set by the - # . field. The - # will not be part of the payload object internal or - # external representation. - # Payload fields that are not set by the SCHEMA can be filled in the same - # way as in any versioned object. - SCHEMA = {} - # Version 1.0: Initial version - VERSION = '1.0' - - def __init__(self): - super(DeckhandPayloadBase, self).__init__() - self.populated = not self.SCHEMA - - def populate_schema(self, **kwargs): - """Populate the object based on the SCHEMA and the source objects - - :param kwargs: A dict contains the source object at the key defined in - the SCHEMA - """ - for key, (obj, field) in self.SCHEMA.items(): - source = kwargs[obj] - if isinstance(source, dict): - source = self._dict_to_obj(source) - try: - setattr(self, key, getattr(source, field)) - # ObjectActionError - not lazy loadable field - # NotImplementedError - obj_load_attr() is not even defined - # OrphanedObjectError - lazy loadable field but context is None - except (#exception.ObjectActionError, - NotImplementedError, - #exception.OrphanedObjectError, - ovo_exception.OrphanedObjectError) as e: - LOG.debug(("Defaulting the value of the field '%(field)s' " - "to None in %(payload)s due to '%(exception)s'"), - {'field': key, - 'payload': self.__class__.__name__, - 'exception': e}) - # NOTE: This will fail if the payload field is not - # nullable, but that means that either the source object is not - # properly initialized or the payload field needs to be defined - # as nullable - setattr(self, key, None) - self.populated = True - - # the schema population will create changed fields but we don't need - # this information in the notification - self.obj_reset_changes(recursive=True) - - def _dict_to_obj(self, d): - """Converts dictionary to object. - - :param d: The dictionary to convert into an object. - :returns: The object representation of the dictionary passed in. - """ - class Object: - def __init__(self, **entries): - self.__dict__.update(entries) - - return Object(**dict(d.items())) diff --git a/deckhand/objects/documents.py b/deckhand/objects/documents.py deleted file mode 100644 index f1dd92bc..00000000 --- a/deckhand/objects/documents.py +++ /dev/null @@ -1,89 +0,0 @@ -# Copyright 2017 AT&T Intellectual Property. All other rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# -# Models for deckhand -# - -from oslo_db import exception as db_exc -from oslo_log import log as logging -from oslo_versionedobjects import fields as ovo_fields - -from deckhand.db.sqlalchemy import api as db_api -from deckhand import errors -from deckhand import objects -from deckhand.objects import base -from deckhand.objects import fields as deckhand_fields - -LOG = logging.getLogger(__name__) - - -class DocumentPayload(base.DeckhandPayloadBase): - - SCHEMA = { - 'schema_version': ('document', 'schemaVersion'), - 'kind': ('document', 'kind'), - 'metadata': ('document', 'metadata'), - 'data': ('document', 'data') - } - - # Version 1.0: Initial version - VERSION = '1.0' - - fields = { - 'schema_version': ovo_fields.StringField(nullable=False), - 'kind': ovo_fields.StringField(nullable=False), - 'metadata': ovo_fields.DictOfStringsField(nullable=False), - 'data': ovo_fields.DictOfStringsField(nullable=False) - } - - def __init__(self, document): - super(DocumentPayload, self).__init__() - self.populate_schema(document=document) - - -@base.DeckhandObjectRegistry.register -class Document(base.DeckhandPersistentObject, base.DeckhandObject): - - # Version 1.0: Initial version - VERSION = '1.0' - - fields = { - 'id': ovo_fields.IntegerField(nullable=False, read_only=True), - 'document': ovo_fields.ObjectField('DocumentPayload', nullable=False), - 'revision_index': ovo_fields.NonNegativeIntegerField(nullable=False) - } - - def __init__(self, *args, **kwargs): - super(Document, self).__init__(*args, **kwargs) - # Set up defaults. - self.obj_reset_changes() - - def create(self, document): - document_obj = DocumentPayload(document) - - values = { - 'schema_version': document_obj.schema_version, - 'kind': document_obj.kind, - 'doc_metadata': document_obj.metadata, - 'data': document_obj.data - } - - try: - db_api.document_create(None, values) - except db_exc.DBDuplicateEntry as e: - raise errors.DocumentExists( - kind=values['kind'], schema_version=values['schema_version']) - except Exception as e: - raise db_exc.DBError(e) diff --git a/deckhand/objects/fields.py b/deckhand/objects/fields.py deleted file mode 100644 index 0f99f27c..00000000 --- a/deckhand/objects/fields.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2017 AT&T Intellectual Property. All other rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from oslo_versionedobjects import fields - - -class BaseDeckhandEnum(fields.Enum): - def __init__(self): - super(BaseDeckhandEnum, self).__init__(valid_values=self.__class__.ALL) - - -class DocumentField(BaseDeckhandEnum): - - # A list of potentially valid statuses for a Document to have. This list - # will evolve over time. - active = 'ACTIVE' - conflict_error = 'CONFLICT_ERROR' - merge_error = 'MERGE_ERROR' - substitute_error = 'SUBSTITUTE_ERROR' - warning = 'WARNING' - - ALL = (active, conflict_error, merge_error, substitute_error, warning)