Refactor database sqlalchemy api/models.

deckhand/conf/config.py

@@ -17,6 +17,18 @@ from oslo_config import cfg
 CONF = cfg.CONF
 
 
+database_group = cfg.OptGroup(
+    name='database',
+    title='Deckhand Database Options'
+)
+
+
+database_opts = [
+    cfg.StrOpt(name='connection',
+               default='')
+]
+
+
 keystone_auth_group = cfg.OptGroup(
     name='keystone_authtoken',
     title='Keystone Authentication Options'
@@ -66,8 +78,13 @@ logging_opts = [
 def register_opts(conf):
     conf.register_group(barbican_group)
     conf.register_opts(barbican_opts, group=barbican_group)
+
+    conf.register_group(database_group)
+    conf.register_opts(database_opts, group=database_group)
+
     conf.register_group(keystone_auth_group)
     conf.register_opts(keystone_auth_opts, group=keystone_auth_group)
+
     conf.register_group(logging_group)
     conf.register_opts(logging_opts, group=logging_group)
 

deckhand/context.py

@@ -34,13 +34,9 @@ class RequestContext(context.RequestContext):
                  timestamp=None, **kwargs):
         if user_id:
             kwargs['user'] = user_id
-        if project_id:
-            kwargs['tenant'] = project_id
 
         super(RequestContext, self).__init__(is_admin=is_admin, **kwargs)
 
-        self.read_deleted = read_deleted
-        self.remote_address = remote_address
         if not timestamp:
             timestamp = timeutils.utcnow()
         if isinstance(timestamp, six.string_types):

deckhand/control/api.py

@@ -22,6 +22,7 @@ from deckhand.conf import config
 from deckhand.control import base as api_base
 from deckhand.control import documents
 from deckhand.control import secrets
+from deckhand.db.sqlalchemy import api as db_api
 
 CONF = cfg.CONF
 
@@ -55,6 +56,8 @@ def start_api(state_manager=None):
     """
     config.register_opts(CONF)
     __setup_logging()
+    engine = db_api.get_engine()
+    assert engine.engine.name == 'postgres'
 
     control_api = falcon.API(request_type=api_base.DeckhandRequest)
 

deckhand/db/session.py

@@ -1,20 +0,0 @@
-# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from sqlalchemy.orm import sessionmaker
-from sqlalchemy import create_engine
-
-
-	engine = create_engine('sqlite:///:memory:', echo=True)
-	session = sessionmaker(bind=engine)

deckhand/db/sqlalchemy/api.py

@@ -0,0 +1,100 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+"""Defines interface for DB access."""
+
+import datetime
+import threading
+
+from oslo_config import cfg
+from oslo_db import exception as db_exception
+from oslo_db.sqlalchemy import session
+from oslo_log import log as logging
+from oslo_utils import excutils
+import osprofiler.sqlalchemy
+from retrying import retry
+import six
+from six.moves import range
+import sqlalchemy
+from sqlalchemy.ext.compiler import compiles
+from sqlalchemy import MetaData, Table
+import sqlalchemy.orm as sa_orm
+from sqlalchemy import sql
+import sqlalchemy.sql as sa_sql
+
+
+sa_logger = None
+LOG = logging.getLogger(__name__)
+
+CONF = cfg.CONF
+
+_FACADE = None
+_LOCK = threading.Lock()
+
+
+def _retry_on_deadlock(exc):
+    """Decorator to retry a DB API call if Deadlock was received."""
+
+    if isinstance(exc, db_exception.DBDeadlock):
+        LOG.warn("Deadlock detected. Retrying...")
+        return True
+    return False
+
+
+def _create_facade_lazily():
+    global _LOCK, _FACADE
+    if _FACADE is None:
+        with _LOCK:
+            if _FACADE is None:
+                _FACADE = session.EngineFacade.from_config(CONF)
+    return _FACADE
+
+
+def get_engine():
+    facade = _create_facade_lazily()
+    return facade.get_engine()
+
+
+def get_session(autocommit=True, expire_on_commit=False):
+    facade = _create_facade_lazily()
+    return facade.get_session(autocommit=autocommit,
+                              expire_on_commit=expire_on_commit)
+
+
+def _validate_db_int(**kwargs):
+    """Make sure that all arguments are less than or equal to 2 ** 31 - 1.
+    This limitation is introduced because databases stores INT in 4 bytes.
+    If the validation fails for some argument, exception. Invalid is raised
+    with appropriate information.
+    """
+    max_int = (2 ** 31) - 1
+
+    for param_key, param_value in kwargs.items():
+        if param_value and param_value > max_int:
+            msg = _("'%(param)s' value out of range, "
+                    "must not exceed %(max)d.") % {"param": param_key,
+                                                   "max": max_int}
+            raise exception.Invalid(msg)
+
+
+def clear_db_env():
+    """Unset global configuration variables for database."""
+    global _FACADE
+    _FACADE = None
+
+
+def image_create(context):
+    """Create a document."""
+    pass

deckhand/db/sqlalchemy/models.py

@@ -21,16 +21,12 @@ from sqlalchemy import Integer
 from sqlalchemy import orm
 from sqlalchemy import schema
 from sqlalchemy import String
-from sqlalchemy import types
-
-
-class _DeckhandBase(models.ModelBase, models.TimestampMixin):
-    pass
+from sqlalchemy import Text
 
 
 # Declarative base class which maintains a catalog of classes and tables
 # relative to that base.
-API_BASE = declarative.declarative_base(cls=_DeckhandBase)
+BASE = declarative.declarative_base()
 
 
 class JSONEncodedDict(types.TypeDecorator):
@@ -42,7 +38,7 @@ class JSONEncodedDict(types.TypeDecorator):
 
     """
 
-    impl = types.VARCHAR
+    impl = Text
 
     def process_bind_param(self, value, dialect):
         if value is not None:
@@ -56,7 +52,49 @@ class JSONEncodedDict(types.TypeDecorator):
         return value
 
 
-class Document(API_BASE):
+class DeckhandBase(models.ModelBase, models.TimestampMixin):
+    """Base class for Deckhand Models."""
+
+    __table_args__ = {'mysql_engine': 'InnoDB', 'mysql_charset': 'utf8'}
+    __table_initialized__ = False
+    __protected_attributes__ = set([
+        "created_at", "updated_at", "deleted_at", "deleted"])
+
+    def save(self, session=None):
+        from deckhand.db.sqlalchemy import api as db_api
+        super(DeckhandBase, self).save(session or db_api.get_session())
+
+    created_at = Column(DateTime, default=lambda: timeutils.utcnow(),
+                        nullable=False)
+    updated_at = Column(DateTime, default=lambda: timeutils.utcnow(),
+                        nullable=True, onupdate=lambda: timeutils.utcnow())
+    deleted_at = Column(DateTime)
+    deleted = Column(Boolean, nullable=False, default=False)
+
+    def delete(self, session=None):
+        """Delete this object."""
+        self.deleted = True
+        self.deleted_at = timeutils.utcnow()
+        self.save(session=session)
+
+    def keys(self):
+        return self.__dict__.keys()
+
+    def values(self):
+        return self.__dict__.values()
+
+    def items(self):
+        return self.__dict__.items()
+
+    def to_dict(self):
+        d = self.__dict__.copy()
+        # Remove private state instance, as it is not serializable and causes
+        # CircularReference.
+        d.pop("_sa_instance_state")
+        return d
+
+
+class Document(BASE, DeckhandBase):
     __tablename__ = 'document'
     __table_args__ = (schema.UniqueConstraint('schema_version', 'kind',
                       name='uniq_schema_version_kinds0schema_version0kind'),)

deckhand/objects/documents.py

@@ -87,8 +87,9 @@ class Document(base.DeckhandPersistentObject, base.DeckhandObject):
         db_document = api_models.Document()
         db_document.update(payload)
 
-        try:
-            # Need to pass session context
-            db_document.save()
-        except Exception as e:
-            LOG.exception(e)
+
+        # deckhand_context = context.RequestContext()
+        # try:
+        #     deckhand_context.session.add(db_document)
+        # except Exception as e:
+        #     LOG.exception(e)