Support links for task status

- Some status changes in a task may have additional information that
  is referenced by a URI link. Support describing these links and
  returning them via API.
- Refactor alembic stuff to better handle table schema updates
- Add unit tests

Change-Id: Iae63a9716f2522578be0244925fc274a4338eac4

alembic/versions/4713e7ebca9_add_task_status_links.py

@@ -0,0 +1,29 @@
+"""add task status links
+
+Revision ID: 4713e7ebca9
+Revises: 4a5bef3702b
+Create Date: 2018-07-05 14:54:18.381988
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '4713e7ebca9'
+down_revision = '4a5bef3702b'
+branch_labels = None
+depends_on = None
+
+from alembic import op
+import sqlalchemy as sa
+
+from drydock_provisioner.statemgmt.db import tables
+
+
+def upgrade():
+    for c in tables.Tasks.__add_result_links__:
+        op.add_column(tables.Tasks.__tablename__, c)
+
+
+def downgrade():
+    for c in tables.Tasks.__add_result_links__:
+        op.drop_column(tables.Tasks.__tablename__, c.name)
+

alembic/versions/4a5bef3702b_create_build_data_table.py

@@ -19,7 +19,7 @@ from drydock_provisioner.statemgmt.db import tables
 
 def upgrade():
     op.create_table(tables.BuildData.__tablename__,
-                    *tables.BuildData.__schema__)
+                    *tables.BuildData.__baseschema__)
 
 
 def downgrade():

alembic/versions/9593a123e7c5_create_base_database_tables.py

@@ -18,15 +18,15 @@ from drydock_provisioner.statemgmt.db import tables
 
 
 def upgrade():
-    op.create_table(tables.Tasks.__tablename__, *tables.Tasks.__schema__)
+    op.create_table(tables.Tasks.__tablename__, *tables.Tasks.__baseschema__)
     op.create_table(tables.ResultMessage.__tablename__,
-                    *tables.ResultMessage.__schema__)
+                    *tables.ResultMessage.__baseschema__)
     op.create_table(tables.ActiveInstance.__tablename__,
-                    *tables.ActiveInstance.__schema__)
+                    *tables.ActiveInstance.__baseschema__)
     op.create_table(tables.BootAction.__tablename__,
-                    *tables.BootAction.__schema__)
+                    *tables.BootAction.__baseschema__)
     op.create_table(tables.BootActionStatus.__tablename__,
-                    *tables.BootActionStatus.__schema__)
+                    *tables.BootActionStatus.__baseschema__)
 
 
 def downgrade():

drydock_provisioner/control/util.py

@@ -0,0 +1,30 @@
+# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Reusable utility functions for API access."""
+from drydock_provisioner.error import ApiError
+from drydock_provisioner.drydock_client.session import KeystoneClient
+from drydock_provisioner.util import KeystoneUtils
+
+def get_internal_api_href(ver):
+    """Get the internal API href for Drydock API version ``ver``."""
+
+    # TODO(sh8121att) Support versioned service registration
+    supported_versions = ['v1.0']
+    if ver in supported_versions:
+        ks_sess = KeystoneUtils.get_session()
+        url = KeystoneClient.get_endpoint(
+            "physicalprovisioner", ks_sess=ks_sess, interface='internal')
+        return url
+    else:
+        raise ApiError("API version %s unknown." % ver)

drydock_provisioner/drydock_client/session.py

@@ -180,12 +180,13 @@ class DrydockSession(object):
 
 class KeystoneClient(object):
     @staticmethod
-    def get_endpoint(endpoint, ks_sess=None, auth_info=None):
+    def get_endpoint(endpoint, ks_sess=None, auth_info=None, interface='internal'):
         """
         Wraps calls to keystone for lookup of an endpoint by service type
         :param endpoint: The endpoint to look up
         :param ks_sess: A keystone session to use for accessing endpoint catalogue
         :param auth_info: Authentication info to use for building a token if a ``ks_sess`` is not specified
+        :param str interface: Which registered endpoint to return
         :returns: The url string of the endpoint
         :rtype: str
         """
@@ -193,7 +194,7 @@ class KeystoneClient(object):
             ks_sess = KeystoneClient.get_ks_session(**auth_info)
 
         return ks_sess.get_endpoint(
-            interface='internal', service_type=endpoint)
+            interface=interface, service_type=endpoint)
 
     @staticmethod
     def get_token(ks_sess=None, auth_info=None):

drydock_provisioner/objects/task.py

@@ -371,6 +371,8 @@ class Task(object):
             self.result.successes,
             'result_failures':
             self.result.failures,
+            'result_links':
+            self.result.links,
             'status':
             self.status,
             'created':
@@ -486,6 +488,7 @@ class Task(object):
         i.result.status = d.get('result_status')
         i.result.successes = d.get('result_successes', [])
         i.result.failures = d.get('result_failures', [])
+        i.result.links = d.get('result_links', [])
 
         # Deserialize the request context for this task
         if i.request_context is not None:
@@ -506,6 +509,8 @@ class TaskStatus(object):
         self.reason = None
         self.status = hd_fields.ActionResult.Incomplete
 
+        self.links = dict()
+
         # For tasks operating on multiple contexts (nodes, networks, etc...)
         # track which contexts ended successfully and which failed
         self.successes = []
@@ -515,6 +520,31 @@ class TaskStatus(object):
     def obj_name(cls):
         return cls.__name__
 
+    def add_link(self, relation, uri):
+        """Add a external reference link to this status.
+
+        :param str relation: The relation of the link
+        :param str uri: A valid URI that references the external content
+        """
+        self.links.setdefault(relation, [])
+        self.links[relation].append(uri)
+
+    def get_links(self, relation=None):
+        """Get one or more links of this status.
+
+        If ``relation`` is None, then return all links.
+
+        :param str relation: Return only links that exhibit this relation
+        :returns: a list of str URIs or empty list
+        """
+        if relation:
+            return self.links.get(relation, [])
+        else:
+            all_links = list()
+            for v in self.links.values():
+                all_links.extend(v)
+            return all_links
+
     def set_message(self, msg):
         self.message = msg
 
@@ -560,6 +590,11 @@ class TaskStatus(object):
         return new_msg
 
     def to_dict(self):
+        links = list()
+        if self.links:
+            for k, v in self.links.items():
+                for r in v:
+                    links.append(dict(rel=k, href=r))
         return {
             'kind': 'Status',
             'apiVersion': 'v1.0',
@@ -569,6 +604,7 @@ class TaskStatus(object):
             'status': self.status,
             'successes': self.successes,
             'failures': self.failures,
+            'links': links,
             'details': {
                 'errorCount': self.error_count,
                 'messageList': [x.to_dict() for x in self.message_list],

drydock_provisioner/statemgmt/db/tables.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Definitions for Drydock database tables."""
+import copy
 
 from sqlalchemy.schema import Table, Column
 from sqlalchemy.types import Boolean, DateTime, String, Integer, Text
@@ -30,7 +31,7 @@ class Tasks(ExtendTable):
 
     __tablename__ = 'tasks'
 
-    __schema__ = [
+    __baseschema__ = [
         Column('task_id', pg.BYTEA(16), primary_key=True),
         Column('parent_task_id', pg.BYTEA(16)),
         Column('subtask_id_list', pg.ARRAY(pg.BYTEA(16))),
@@ -54,13 +55,19 @@ class Tasks(ExtendTable):
         Column('terminate', Boolean, default=False)
     ]
 
+    __add_result_links__ = [
+        Column('result_links', pg.JSON),
+    ]
+
+    __schema__ = copy.copy(__baseschema__)
+    __schema__.extend(__add_result_links__)
 
 class ResultMessage(ExtendTable):
     """Table for tracking result/status messages."""
 
     __tablename__ = 'result_message'
 
-    __schema__ = [
+    __baseschema__ = [
         Column('sequence', Integer, primary_key=True),
         Column('task_id', pg.BYTEA(16)),
         Column('message', String(1024)),
@@ -71,37 +78,43 @@ class ResultMessage(ExtendTable):
         Column('extra', pg.JSON)
     ]
 
+    __schema__ = copy.copy(__baseschema__)
+
 
 class ActiveInstance(ExtendTable):
     """Table to organize multiple orchestrator instances."""
 
     __tablename__ = 'active_instance'
 
-    __schema__ = [
+    __baseschema__ = [
         Column('dummy_key', Integer, primary_key=True),
         Column('identity', pg.BYTEA(16)),
         Column('last_ping', DateTime),
     ]
 
+    __schema__ = copy.copy(__baseschema__)
+
 
 class BootAction(ExtendTable):
     """Table persisting node build data."""
 
     __tablename__ = 'boot_action'
 
-    __schema__ = [
+    __baseschema__ = [
         Column('node_name', String(280), primary_key=True),
         Column('task_id', pg.BYTEA(16)),
         Column('identity_key', pg.BYTEA(32)),
     ]
 
+    __schema__ = copy.copy(__baseschema__)
+
 
 class BootActionStatus(ExtendTable):
     """Table tracking status of node boot actions."""
 
     __tablename__ = 'boot_action_status'
 
-    __schema__ = [
+    __baseschema__ = [
         Column('node_name', String(280), index=True),
         Column('action_id', pg.BYTEA(16), primary_key=True),
         Column('action_name', String(64)),
@@ -110,13 +123,15 @@ class BootActionStatus(ExtendTable):
         Column('action_status', String(32)),
     ]
 
+    __schema__ = copy.copy(__baseschema__)
+
 
 class BuildData(ExtendTable):
     """Table for persisting node build data."""
 
     __tablename__ = 'build_data'
 
-    __schema__ = [
+    __baseschema__ = [
         Column('node_name', String(32), index=True),
         Column('task_id', pg.BYTEA(16), index=True),
         Column('collected_date', DateTime),
@@ -124,3 +139,5 @@ class BuildData(ExtendTable):
         Column('data_format', String(32)),
         Column('data_element', Text),
     ]
+
+    __schema__ = copy.copy(__baseschema__)

etc/drydock/drydock.conf.sample

@@ -26,6 +26,9 @@
 # The URI database connect string. (string value)
 #database_connect_string = <None>
 
+# The SQLalchemy database connection pool size. (integer value)
+#pool_size = 15
+
 
 [keystone_authtoken]
 

tests/unit/test_task_link.py

@@ -0,0 +1,114 @@
+# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+'''Tests the functions for adding and retrieving task status links.'''
+from drydock_provisioner.objects import TaskStatus
+
+
+class TestTaskStatusLinks():
+    def test_links_add(self):
+        '''Add a link to a task status.'''
+        ts = TaskStatus()
+
+        relation = 'test'
+        uri = 'http://foo.com/test'
+
+        ts.add_link(relation, uri)
+
+        assert relation in ts.links
+        assert uri in ts.links.get(relation, [])
+
+    def test_links_get_empty(self):
+        '''Get links with an empty list.'''
+        ts = TaskStatus()
+
+        links = ts.get_links()
+
+        assert len(links) == 0
+
+        relation = 'test'
+        uri = 'http://foo.com/test'
+
+        ts.add_link(relation, uri)
+        links = ts.get_links(relation='none')
+
+        assert len(links) == 0
+
+    def test_links_get_all(self):
+        '''Get all links in a task status.'''
+        ts = TaskStatus()
+
+        relation = 'test'
+        uri = 'http://foo.com/test'
+
+        ts.add_link(relation, uri)
+        links = ts.get_links()
+
+        assert len(links) == 1
+        assert uri in links
+
+    def test_links_get_all_duplicate_relation(self):
+        '''Get all links where a relation has multiple uris.'''
+        ts = TaskStatus()
+
+        relation = 'test'
+        uri = 'http://foo.com/test'
+        uri2 = 'http://baz.com/test'
+
+        ts.add_link(relation, uri)
+        ts.add_link(relation, uri2)
+
+        links = ts.get_links()
+
+        assert len(links) == 2
+        assert uri in links
+        assert uri2 in links
+
+    def test_links_get_filter(self):
+        '''Get links with a filter.'''
+        ts = TaskStatus()
+
+        relation = 'test'
+        uri = 'http://foo.com/test'
+
+        relation2 = 'test2'
+        uri2 = 'http://baz.com/test'
+
+        ts.add_link(relation, uri)
+        ts.add_link(relation2, uri2)
+
+        links = ts.get_links(relation=relation)
+
+        assert len(links) == 1
+        assert uri in links
+
+        links = ts.get_links(relation=relation2)
+
+        assert len(links) == 1
+        assert uri2 in links
+
+    def test_links_serialization(self):
+        '''Check that task status serilization contains links correctly.'''
+        ts = TaskStatus()
+
+        relation = 'test'
+        uri = 'http://bar.com'
+
+        ts.set_message('foo')
+        ts.set_reason('bar')
+        ts.add_link(relation, uri)
+
+        ts_dict = ts.to_dict()
+
+        assert isinstance(ts_dict.get('links'), list)
+        assert {'rel': relation, 'href': uri} in ts_dict.get('links', [])