Deckhand postgresql compatibility.

Currently, Deckhand is not fully compatible with postgresql as
it uses sqlite for all of its testing, including functional testing.
Since postgresql will be used in prod, Deckhand obviously must
support it, in addition to sqlite, needed for unit testing.

This commit alters the functional testing script to use postgresql
as well as makes necessary back-end changes to support postgresql.

Included in this commit:

  - alter tools/functional-tests.sh so that it uses postgresql
    as the db connection
  - modifies primary key for Bucket DB model to be an Integer rather
    than a String
  - updates foreign key to point to new primary key
  - updates necessary integration logic so that the bucket name
    is still known by the Document DB model and returned in
    appropriate response bodies

Change-Id: I7bc806fb18f7b47c13978dcd806d422a573a06b3

deckhand/control/views/document.py

@@ -34,7 +34,7 @@ class ViewBuilder(common.ViewBuilder):
         # need to return bucket_id and revision_id.
         if len(documents) == 1 and documents[0]['deleted']:
             resp_obj = {'status': {}}
-            resp_obj['status']['bucket'] = documents[0]['bucket_id']
+            resp_obj['status']['bucket'] = documents[0]['bucket_name']
             resp_obj['status']['revision'] = documents[0]['revision_id']
             return [resp_obj]
 
@@ -47,7 +47,7 @@ class ViewBuilder(common.ViewBuilder):
 
             resp_obj = {x: document[x] for x in attrs}
             resp_obj.setdefault('status', {})
-            resp_obj['status']['bucket'] = document['bucket_id']
+            resp_obj['status']['bucket'] = document['bucket_name']
             resp_obj['status']['revision'] = document['revision_id']
             resp_list.append(resp_obj)
 

deckhand/control/views/revision.py

@@ -37,7 +37,7 @@ class ViewBuilder(common.ViewBuilder):
 
             body['tags'].update([t['tag'] for t in revision['tags']])
             body['buckets'].update(
-                [d['bucket_id'] for d in rev_documents])
+                [d['bucket_name'] for d in rev_documents])
 
             body['tags'] = sorted(body['tags'])
             body['buckets'] = sorted(body['buckets'])
@@ -77,7 +77,8 @@ class ViewBuilder(common.ViewBuilder):
         for tag in revision['tags']:
             tags.setdefault(tag['tag'], {'name': tag['tag']})
 
-        buckets = sorted(set([d['bucket_id'] for d in revision['documents']]))
+        buckets = sorted(
+            set([d['bucket_name'] for d in revision['documents']]))
 
         return {
             'id': revision.get('id'),

deckhand/db/sqlalchemy/api.py

@@ -114,7 +114,7 @@ def documents_create(bucket_name, documents, session=None):
     # `documents`: the difference between the former and the latter.
     document_history = [(d['schema'], d['name'])
                         for d in revision_get_documents(
-                            bucket_id=bucket_name)]
+                            bucket_name=bucket_name)]
     documents_to_delete = [
         h for h in document_history if h not in
         [(d['schema'], d['metadata']['name']) for d in documents]]
@@ -136,7 +136,7 @@ def documents_create(bucket_name, documents, session=None):
                 doc['name'] = d[1]
                 doc['data'] = {}
                 doc['_metadata'] = {}
-                doc['bucket_id'] = bucket['name']
+                doc['bucket_id'] = bucket['id']
                 doc['revision_id'] = revision['id']
 
                 # Save and mark the document as `deleted` in the database.
@@ -151,9 +151,10 @@ def documents_create(bucket_name, documents, session=None):
                   [(d['schema'], d['name']) for d in documents_to_create])
         for doc in documents_to_create:
             with session.begin():
-                doc['bucket_id'] = bucket['name']
+                doc['bucket_id'] = bucket['id']
                 doc['revision_id'] = revision['id']
                 doc.save(session=session)
+
         # NOTE(fmontei): The orig_revision_id is not copied into the
         # revision_id for each created document, because the revision_id here
         # should reference the just-created revision. In case the user needs
@@ -200,12 +201,12 @@ def _documents_create(bucket_name, values_list, session=None):
             # If the document already exists in another bucket, raise an error.
             # Ignore redundant validation policies as they are allowed to exist
             # in multiple buckets.
-            if (existing_document['bucket_id'] != bucket_name and
+            if (existing_document['bucket_name'] != bucket_name and
                 existing_document['schema'] != types.VALIDATION_POLICY_SCHEMA):
                 raise errors.DocumentExists(
                     schema=existing_document['schema'],
                     name=existing_document['name'],
-                    bucket=existing_document['bucket_id'])
+                    bucket=existing_document['bucket_name'])
 
             if not _document_changed(existing_document):
                 # Since the document has not changed, reference the original

deckhand/db/sqlalchemy/models.py

@@ -19,6 +19,7 @@ from sqlalchemy import Boolean
 from sqlalchemy import Column
 from sqlalchemy import DateTime
 from sqlalchemy.ext import declarative
+from sqlalchemy.ext.hybrid import hybrid_property
 from sqlalchemy import ForeignKey
 from sqlalchemy import Integer
 from sqlalchemy.orm import relationship
@@ -64,10 +65,8 @@ class DeckhandBase(models.ModelBase, models.TimestampMixin):
     def items(self):
         return self.__dict__.items()
 
-    def to_dict(self, raw_dict=False):
+    def to_dict(self):
         """Convert the object into dictionary format.
-
-        :param raw_dict: Renames the key "_metadata" to "metadata".
         """
         d = self.__dict__.copy()
         # Remove private state instance, as it is not serializable and causes
@@ -93,8 +92,9 @@ def gen_unique_constraint(table_name, *fields):
 class Bucket(BASE, DeckhandBase):
     __tablename__ = 'buckets'
 
-    name = Column(String(36), primary_key=True)
+    id = Column(Integer, primary_key=True)
-    documents = relationship("Document")
+    name = Column(String(36), unique=True)
+    documents = relationship("Document", backref="bucket")
 
 
 class Revision(BASE, DeckhandBase):
@@ -141,8 +141,7 @@ class Document(BASE, DeckhandBase):
     _metadata = Column(oslo_types.JsonEncodedDict(), nullable=False)
     data = Column(oslo_types.JsonEncodedDict(), nullable=True)
     is_secret = Column(Boolean, nullable=False, default=False)
-
+    bucket_id = Column(Integer, ForeignKey('buckets.id', ondelete='CASCADE'),
-    bucket_id = Column(Integer, ForeignKey('buckets.name', ondelete='CASCADE'),
                        nullable=False)
     revision_id = Column(
         Integer, ForeignKey('revisions.id', ondelete='CASCADE'),
@@ -160,11 +159,23 @@ class Document(BASE, DeckhandBase):
         Integer, ForeignKey('revisions.id', ondelete='CASCADE'),
                             nullable=True)
 
+    @hybrid_property
+    def bucket_name(self):
+        if hasattr(self, 'bucket') and self.bucket:
+            return self.bucket.name
+        return None
+
     def to_dict(self, raw_dict=False):
+        """Convert the object into dictionary format.
+
+        :param raw_dict: Renames the key "_metadata" to "metadata".
+        """
         d = super(Document, self).to_dict()
+        d['bucket_name'] = self.bucket_name
+
         # NOTE(fmontei): ``metadata`` is reserved by the DB, so ``_metadata``
         # must be used to store document metadata information in the DB.
-        if not raw_dict and '_metadata' in self.keys():
+        if not raw_dict:
             d['metadata'] = d.pop('_metadata')
 
         return d

deckhand/tests/unit/db/base.py

@@ -69,7 +69,7 @@ class TestDbBase(base.DeckhandWithDBTestCase):
         if do_validation:
             for idx, doc in enumerate(docs):
                 self.validate_document(expected=documents[idx], actual=doc)
-                self.assertEqual(bucket_name, doc['bucket_id'])
+                self.assertEqual(bucket_name, doc['bucket_name'])
 
         return docs
 

requirements.txt

@@ -18,6 +18,8 @@ stevedore>=1.20.0 # Apache-2.0
 jsonschema!=2.5.0,<3.0.0,>=2.0.0 # MIT
 python-keystoneclient>=3.8.0 # Apache-2.0
 keystonemiddleware>=4.12.0 # Apache-2.0
+psycopg2==2.7.3.1
+uwsgi==2.0.15
 
 oslo.cache>=1.5.0 # Apache-2.0
 oslo.concurrency>=3.8.0 # Apache-2.0
@@ -33,4 +35,3 @@ oslo.serialization!=2.19.1,>=1.10.0 # Apache-2.0
 oslo.utils>=3.20.0 # Apache-2.0
 
 python-barbicanclient>=4.0.0  # Apache-2.0
-uwsgi==2.0.15

tools/functional-tests.sh

@@ -34,17 +34,12 @@ POSTGRES_IP=$(
         --format='{{ .NetworkSettings.Networks.bridge.IPAddress }}' \
             $POSTGRES_ID
 )
-POSTGRES_PORT=$(
-    sudo docker inspect \
-        --format='{{(index (index .NetworkSettings.Ports "5432/tcp") 0).HostPort}}' \
-            $POSTGRES_ID
-)
 
 log_section Creating config file
 CONF_DIR=$(mktemp -d)
 
 export DECKHAND_TEST_URL=http://localhost:9000
-export DATABASE_URL=postgres://deckhand:password@$POSTGRES_IP:$POSTGRES_PORT/deckhand
+export DATABASE_URL=postgresql+psycopg2://deckhand:password@$POSTGRES_IP:5432/deckhand
 # Used by Deckhand's initialization script to search for config files.
 export OS_DECKHAND_CONFIG_DIR=$CONF_DIR
 
@@ -61,9 +56,7 @@ use_stderr = true
 [barbican]
 
 [database]
-# XXX For now, connection to postgres is not setup.
+connection = $DATABASE_URL
-#connection = $DATABASE_URL
-connection = sqlite://
 
 [keystone_authtoken]
 EOCONF