Merge pull request #6 from sh8121att/initial_api_service

Initial api service

README.md

@@ -1,6 +1,13 @@
 # helm_drydock
 A python REST orchestrator to translate a YAML host topology to a provisioned set of hosts and provide a set of cloud-init post-provisioning instructions.
 
+To run:
+
+    $ virtualenv -p python3 /var/tmp/drydock
+    $ . /var/tmp/drydock/bin/activate
+    $ python setup.py install
+    $ uwsgi --http :9000 -w helm_drydock.drydock --callable drydock --enable-threads -L
+
 ## Modular service
 
 ### Design Consumer ###

helm_drydock/config.py

@@ -21,13 +21,24 @@
 
 class DrydockConfig(object):
 
+    global_config = {
+        'log_level':    'DEBUG',
+    }
+
     node_driver = {
         'maasdriver':   {
             'api_key':  'KTMHgA42cNSMnfmJ82:cdg4yQUhp542aHsCTV:7Dc2KB9hQpWq3LfQAAAKAj6wdg22yWxZ',
-            'api_url':  'http://localhost:5240/MAAS/api/2.0/'
+            'api_url':  'http://localhost:5240/MAAS/api/2.0/',
         },
     }
 
     ingester_config = {
-        'plugins': ['helm_drydock.ingester.plugins.yaml']
+        'plugins': ['helm_drydock.ingester.plugins.yaml.YamlIngester'],
+    }
+
+    orchestrator_config = {
+        'drivers': {
+            'oob': 'helm_drydock.drivers.oob.pyghmi_driver.PyghmiDriver',
+            'node': 'helm_drydock.drivers.node.maasdriver.driver.MaasNodeDriver',
+        }
     }

helm_drydock/control/__init__.py

@@ -10,4 +10,4 @@
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
-# limitations under the License.
+# limitations under the License.

helm_drydock/control/api.py

@@ -0,0 +1,47 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import falcon
+
+from .designs import *
+from .tasks import *
+
+from .base import DrydockRequest
+from .middleware import AuthMiddleware, ContextMiddleware, LoggingMiddleware
+
+def start_api(state_manager=None, ingester=None, orchestrator=None):
+    """
+    Start the Drydock API service
+
+    :param state_manager:   Instance of helm_drydock.statemgmt.manager.DesignState for accessing
+                            state persistence
+    :param ingester:        Instance of helm_drydock.ingester.ingester.Ingester for handling design
+                            part input
+    """
+    control_api = falcon.API(request_type=DrydockRequest,
+                             middleware=[AuthMiddleware(), ContextMiddleware(), LoggingMiddleware()])
+
+    # API for managing orchestrator tasks
+    control_api.add_route('/tasks', TasksResource(state_manager=state_manager, orchestrator=orchestrator))
+    control_api.add_route('/tasks/{task_id}', TaskResource(state_manager=state_manager))
+
+    # API for managing site design data
+    control_api.add_route('/designs', DesignsResource(state_manager=state_manager))
+    control_api.add_route('/designs/{design_id}', DesignResource(state_manager=state_manager, orchestrator=orchestrator))
+    control_api.add_route('/designs/{design_id}/parts', DesignsPartsResource(state_manager=state_manager, ingester=ingester))
+    control_api.add_route('/designs/{design_id}/parts/{kind}', DesignsPartsKindsResource(state_manager=state_manager))
+
+    control_api.add_route('/designs/{design_id}/parts/{kind}/{name}',
+                          DesignsPartResource(state_manager=state_manager, orchestrator=orchestrator))
+
+    return control_api

helm_drydock/control/base.py

@@ -0,0 +1,144 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import falcon.request as request
+import uuid
+import json
+import logging
+
+import helm_drydock.error as errors
+
+class BaseResource(object):
+
+    def __init__(self):
+        self.logger = logging.getLogger('control')
+        self.authorized_roles = []
+
+    def on_options(self, req, resp):
+        self_attrs = dir(self)
+        methods = ['GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'PATCH']
+        allowed_methods = []
+
+        for m in methods:
+            if 'on_' + m.lower() in self_attrs:
+                allowed_methods.append(m)
+
+        resp.headers['Allow'] = ','.join(allowed_methods)
+        resp.status = falcon.HTTP_200
+
+    # For authorizing access at the Resource level. A Resource requiring
+    # finer grained authorization at the method or instance level must
+    # implement that in the request handlers
+    def authorize_roles(self, role_list):
+        authorized = set(self.authorized_roles)
+        applied = set(role_list)
+
+        if authorized.isdisjoint(applied):
+            return False
+        else:
+            return True
+
+    def req_json(self, req):
+        if req.content_length is None or req.content_length == 0:
+            return None
+
+        if req.content_type is not None and req.content_type.lower() == 'application/json':
+            raw_body = req.stream.read(req.content_length or 0)
+
+            if raw_body is None:
+                return None
+
+            try:
+                json_body = json.loads(raw_body.decode('utf-8'))
+                return json_body
+            except json.JSONDecodeError as jex:
+                raise errors.InvalidFormat("%s: Invalid JSON in body: %s" % (req.path, jex))
+        else:
+            raise errors.InvalidFormat("Requires application/json payload")
+
+    def return_error(self, resp, status_code, message="", retry=False):
+        resp.body = json.dumps({'type': 'error', 'message': message, 'retry': retry})
+        resp.status = status_code
+
+    def log_error(self, ctx, level, msg):
+        extra = {
+            'user': 'N/A',
+            'req_id': 'N/A',
+            'external_ctx': 'N/A'
+        }
+
+        if ctx is not None:
+            extra = {
+                'user': ctx.user,
+                'req_id': ctx.request_id,
+                'external_ctx': ctx.external_marker,
+            }
+
+        self.logger.log(level, msg, extra=extra)
+
+    def debug(self, ctx, msg):
+        self.log_error(ctx, logging.DEBUG, msg)
+
+    def info(self, ctx, msg):
+        self.log_error(ctx, logging.INFO, msg)
+
+    def warn(self, ctx, msg):
+        self.log_error(ctx, logging.WARN, msg)
+
+    def error(self, ctx, msg):
+        self.log_error(ctx, logging.ERROR, msg)
+
+
+class StatefulResource(BaseResource):
+
+    def __init__(self, state_manager=None):
+        super(StatefulResource, self).__init__()
+
+        if state_manager is None:
+            self.error(None, "StatefulResource:init - StatefulResources require a state manager be set")
+            raise ValueError("StatefulResources require a state manager be set")
+
+        self.state_manager = state_manager
+
+
+class DrydockRequestContext(object):
+
+    def __init__(self):
+        self.log_level = 'ERROR'
+        self.user = None
+        self.roles = ['anyone']
+        self.request_id = str(uuid.uuid4())
+        self.external_marker = None
+
+    def set_log_level(self, level):
+        if level in ['error', 'info', 'debug']:
+            self.log_level = level
+
+    def set_user(self, user):
+        self.user = user
+
+    def add_role(self, role):
+        self.roles.append(role)
+
+    def add_roles(self, roles):
+        self.roles.extend(roles)
+
+    def remove_role(self, role):
+        self.roles = [x for x in self.roles
+                      if x != role]
+
+    def set_external_marker(self, marker):
+        self.external_marker = str(marker)[:32]
+
+class DrydockRequest(request.Request):
+    context_type = DrydockRequestContext

helm_drydock/control/designs.py

@@ -0,0 +1,164 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import falcon
+import json
+import uuid
+import logging
+
+import helm_drydock.objects as hd_objects
+import helm_drydock.error as errors
+
+from .base import StatefulResource
+
+class DesignsResource(StatefulResource):
+
+    def __init__(self, **kwargs):
+        super(DesignsResource, self).__init__(**kwargs)
+        self.authorized_roles = ['user']
+
+    def on_get(self, req, resp):
+        state = self.state_manager
+
+        designs = list(state.designs.keys())
+
+        resp.body = json.dumps(designs)
+        resp.status = falcon.HTTP_200
+
+    def on_post(self, req, resp):
+        try:
+            json_data = self.req_json(req)
+            design = None
+            if json_data is not None:
+                base_design = json_data.get('base_design_id', None)
+
+                if base_design is not None:
+                    base_design = uuid.UUID(base_design)
+                    design = hd_objects.SiteDesign(base_design_id=base_design_uuid)
+            else:
+                design = hd_objects.SiteDesign()
+            design.assign_id()
+            design.create(req.context, self.state_manager)
+
+            resp.body = json.dumps(design.obj_to_simple())
+            resp.status = falcon.HTTP_201
+        except errors.StateError as stex:
+            self.error(req.context, "Error updating persistence")
+            self.return_error(resp, falcon.HTTP_500, message="Error updating persistence", retry=True)
+        except errors.InvalidFormat as fex:
+            self.error(req.context, str(fex))
+            self.return_error(resp, falcon.HTTP_400, message=str(fex), retry=False)
+
+
+class DesignResource(StatefulResource):
+
+    def __init__(self, orchestrator=None, **kwargs):
+        super(DesignResource, self).__init__(**kwargs)
+        self.authorized_roles = ['user']
+        self.orchestrator = orchestrator
+
+    def on_get(self, req, resp, design_id):
+        source = req.params.get('source', 'designed')
+
+        try:
+            design = None
+            if source == 'compiled':
+                design = self.orchestrator.get_effective_site(design_id)
+            elif source == 'designed':
+                design = self.orchestrator.get_described_site(design_id)
+
+            resp.body = json.dumps(design.obj_to_simple())
+        except errors.DesignError:
+            self.error(req.context, "Design %s not found" % design_id)
+            self.return_error(resp, falcon.HTTP_404, message="Design %s not found" % design_id, retry=False)
+
+class DesignsPartsResource(StatefulResource):
+
+    def __init__(self, ingester=None, **kwargs):
+        super(DesignsPartsResource, self).__init__(**kwargs)
+        self.ingester = ingester
+        self.authorized_roles = ['user']
+
+        if ingester is None:
+            self.error(None, "DesignsPartsResource requires a configured Ingester instance")
+            raise ValueError("DesignsPartsResource requires a configured Ingester instance")
+
+    def on_post(self, req, resp, design_id):
+        ingester_name = req.params.get('ingester', None)
+
+        if ingester_name is None:
+            self.error(None, "DesignsPartsResource POST requires parameter 'ingester'")
+            self.return_error(resp, falcon.HTTP_400, message="POST requires parameter 'ingester'", retry=False)
+        else:
+            try:
+                raw_body = req.stream.read(req.content_length or 0)
+                if raw_body is not None and len(raw_body) > 0:
+                    parsed_items = self.ingester.ingest_data(plugin_name=ingester_name, design_state=self.state_manager,
+                                                             content=raw_body, design_id=design_id, context=req.context)
+                    resp.status = falcon.HTTP_201
+                    resp.body = json.dumps([x.obj_to_simple() for x in parsed_items])
+                else:
+                    self.return_error(resp, falcon.HTTP_400, message="Empty body not supported", retry=False)    
+            except ValueError:
+                self.return_error(resp, falcon.HTTP_500, message="Error processing input", retry=False)
+            except LookupError:
+                self.return_error(resp, falcon.HTTP_400, message="Ingester %s not registered" % ingester_name, retry=False)
+
+
+class DesignsPartsKindsResource(StatefulResource):
+    def __init__(self, **kwargs):
+        super(DesignsPartsKindsResource, self).__init__(**kwargs)
+        self.authorized_roles = ['user']
+
+    def on_get(self, req, resp, design_id, kind):
+        pass
+
+class DesignsPartResource(StatefulResource):
+
+    def __init__(self, orchestrator=None, **kwargs):
+        super(DesignsPartResource, self).__init__(**kwargs)
+        self.authorized_roles = ['user']
+        self.orchestrator = orchestrator
+
+    def on_get(self, req , resp, design_id, kind, name):
+        source = req.params.get('source', 'designed')
+
+        try:
+            design = None
+            if source == 'compiled':
+                design = self.orchestrator.get_effective_site(design_id)
+            elif source == 'designed':
+                design = self.orchestrator.get_described_site(design_id)
+
+            part = None
+            if kind == 'Site':
+                part = design.get_site()
+            elif kind == 'Network':
+                part = design.get_network(name)
+            elif kind == 'NetworkLink':
+                part = design.get_network_link(name)
+            elif kind == 'HardwareProfile':
+                part = design.get_hardware_profile(name)
+            elif kind == 'HostProfile':
+                part = design.get_host_profile(name)
+            elif kind == 'BaremetalNode':
+                part = design.get_baremetal_node(name)
+            else:
+                self.error(req.context, "Kind %s unknown" % kind)
+                self.return_error(resp, falcon.HTTP_404, message="Kind %s unknown" % kind, retry=False)
+                return
+
+            resp.body = json.dumps(part.obj_to_simple())
+        except errors.DesignError as dex:
+            self.error(req.context, str(dex))
+            self.return_error(resp, falcon.HTTP_404, message=str(dex), retry=False)

helm_drydock/control/middleware.py

@@ -0,0 +1,94 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import falcon
+import logging
+import uuid
+
+import helm_drydock.config as config
+
+class AuthMiddleware(object):
+
+    # Authentication
+    def process_request(self, req, resp):
+        ctx = req.context
+        token = req.get_header('X-Auth-Token')
+
+        user = self.validate_token(token)
+
+        if user is not None:
+            ctx.set_user(user)
+            user_roles = self.role_list(user)
+            ctx.add_roles(user_roles)
+        else:
+            ctx.add_role('anyone')
+
+    # Authorization
+    def process_resource(self, req, resp, resource, params):
+        ctx = req.context
+
+        if not resource.authorize_roles(ctx.roles):
+            raise falcon.HTTPUnauthorized('Authentication required',
+                                          ('This resource requires an authorized role.'))
+
+    # Return the username associated with an authenticated token or None
+    def validate_token(self, token):
+        if token == '42':
+            return 'scott'
+        elif token == 'bigboss':
+            return 'admin'
+        else:
+            return None
+
+    # Return the list of roles assigned to the username
+    # Roles need to be an enum
+    def role_list(self, username):
+        if username == 'scott':
+            return ['user']
+        elif username == 'admin':
+            return ['user', 'admin']
+
+class ContextMiddleware(object):
+
+    def process_request(self, req, resp):
+        ctx = req.context
+
+        requested_logging = req.get_header('X-Log-Level')
+
+        if (config.DrydockConfig.global_config.get('log_level', '') == 'DEBUG' or
+           (requested_logging == 'DEBUG' and 'admin' in ctx.roles)):
+            ctx.set_log_level('DEBUG')
+        elif requested_logging == 'INFO':
+            ctx.set_log_level('INFO')
+
+        ctx.req_id = str(uuid.uuid4())
+
+        ext_marker = req.get_header('X-Context-Marker')
+
+        ctx.external_ctx = ext_marker if ext_marker is not None else ''
+
+class LoggingMiddleware(object):
+
+    def __init__(self):
+        self.logger = logging.getLogger('drydock.control')
+
+    def process_response(self, req, resp, resource, req_succeeded):
+        ctx = req.context
+        extra = {
+            'user': ctx.user,
+            'req_id': ctx.req_id,
+            'external_ctx': ctx.external_ctx,
+        }
+        resp.append_header('X-Drydock-Req', ctx.req_id)
+        self.logger.info("%s - %s" % (req.uri, resp.status), extra=extra)

helm_drydock/control/readme.md

@@ -3,8 +3,6 @@
 This is the external facing API service to control the rest
 of Drydock and query Drydock-managed data.
 
-Anticipate basing this service on the falcon Python library
-
 ## Endpoints ##
 
 ### /tasks ###
@@ -12,3 +10,21 @@ Anticipate basing this service on the falcon Python library
 POST - Create a new orchestration task and submit it for execution
 GET - Get status of a task
 DELETE - Cancel execution of a task if permitted
+
+### /designs ###
+
+POST - Create a new site design so design parts can be added
+
+### /designs/{id}
+
+GET - Get a current design if available. Param 'source=compiled' to calculate the inheritance chain and compile the effective design.
+
+### /designs/{id}/parts
+
+POST - Submit a new design part to be ingested and added to this design
+GET - View a currently defined design part
+PUT - Replace an existing design part *Not Implemented*
+
+### /designs/{id}/parts/{kind}/{name}
+
+GET - View a single design part. param 'source=compiled' to calculate the inheritance chain and compile the effective configuration for the design part.

helm_drydock/control/tasks.py

@@ -0,0 +1,79 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import falcon
+import json
+import threading
+import traceback
+
+import helm_drydock.objects.task as obj_task
+from .base import StatefulResource
+
+class TasksResource(StatefulResource):
+
+    def __init__(self, orchestrator=None, **kwargs):
+        super(TasksResource, self).__init__(**kwargs)
+        self.authorized_roles = ['user']
+        self.orchestrator = orchestrator
+
+    def on_get(self, req, resp):
+        task_id_list = [str(x.get_id()) for x in self.state_manager.tasks]
+        resp.body = json.dumps(task_id_list)
+
+    def on_post(self, req, resp):
+        try:
+            json_data = self.req_json(req)
+            
+            sitename = json_data.get('sitename', None)
+            design_id = json_data.get('design_id', None)
+            action = json_data.get('action', None)
+
+            if sitename is None or design_id is None or action is None:
+                self.info(req.context, "Task creation requires fields sitename, design_id, action")
+                self.return_error(resp, falcon.HTTP_400, message="Task creation requires fields sitename, design_id, action", retry=False)
+                return
+
+            task = self.orchestrator.create_task(obj_task.OrchestratorTask, site=sitename,
+                                                 design_id=design_id, action=action)
+
+            task_thread = threading.Thread(target=self.orchestrator.execute_task, args=[task.get_id()])
+            task_thread.start()
+
+            resp.body = json.dumps(task.to_dict())
+            resp.status = falcon.HTTP_201
+        except Exception as ex:
+            self.error(req.context, "Unknown error: %s\n%s" % (str(ex), traceback.format_exc()))
+            self.return_error(resp, falcon.HTTP_500, message="Unknown error", retry=False)
+
+
+class TaskResource(StatefulResource):
+
+    def __init__(self, orchestrator=None, **kwargs):
+        super(TaskResource, self).__init__(**kwargs)
+        self.authorized_roles = ['user']
+        self.orchestrator = orchestrator
+
+    def on_get(self, req, resp, task_id):
+        try:
+            task = self.state_manager.get_task(task_id)
+
+            if task is None:
+                self.info(req.context, "Task %s does not exist" % task_id )
+                self.return_error(resp, falcon.HTTP_404, message="Task %s does not exist" % task_id, retry=False)
+                return
+
+            resp.body = json.dumps(task.to_dict())
+            resp.status = falcon.HTTP_200
+        except Exception as ex:
+            self.error(req.context, "Unknown error: %s" % (str(ex)))
+            self.return_error(resp, falcon.HTTP_500, message="Unknown error", retry=False)

helm_drydock/drivers/node/maasdriver/driver.py

@@ -101,7 +101,7 @@ class MaasNodeDriver(NodeDriver):
         self.orchestrator.task_field_update(task.get_id(),
                             status=hd_fields.TaskStatus.Running)
 
-        site_design = self.orchestrator.get_effective_site(design_id, task.site_name)
+        site_design = self.orchestrator.get_effective_site(design_id)
 
         if task.action == hd_fields.OrchestratorAction.CreateNetworkTemplate:
             subtask = self.orchestrator.create_task(task_model.DriverTask,
@@ -149,8 +149,7 @@ class MaasTaskRunner(drivers.DriverTaskRunner):
         self.maas_client = MaasRequestFactory(self.driver_config['api_url'],
                                               self.driver_config['api_key'])
 
-        site_design = self.orchestrator.get_effective_site(self.task.design_id,
+        site_design = self.orchestrator.get_effective_site(self.task.design_id)
-                                                           self.task.site_name)
 
         if task_action == hd_fields.OrchestratorAction.CreateNetworkTemplate:
             # Try to true up MaaS definitions of fabrics/vlans/subnets
@@ -165,110 +164,120 @@ class MaasTaskRunner(drivers.DriverTaskRunner):
             }
 
             for n in design_networks:
-                exists = subnets.query({'cidr': n.cidr})
+                try:
+                    subnet = subnets.singleton({'cidr': n.cidr})
 
-                subnet = None
+                    if subnet is not None:
+                        subnet.name = n.name
+                        subnet.dns_servers = n.dns_servers
 
-                if len(exists) > 0:
+                        vlan_list = maas_vlan.Vlans(self.maas_client, fabric_id=subnet.fabric)
-                    subnet = exists[0]
-
-                    subnet.name = n.name
-                    subnet.dns_servers = n.dns_servers
-
-                    vlan_list = maas_vlan.Vlans(self.maas_client, fabric_id=subnet.fabric)
-                    vlan_list.refresh()
-
-                    vlan = vlan_list.select(subnet.vlan)
-
-                    if vlan is not None:
-                        if ((n.vlan_id is None and vlan.vid != 0) or 
-                           (n.vlan_id is not None and vlan.vid != n.vlan_id)):
-
-                            # if the VLAN name matches, assume this is the correct resource
-                            # and it needs to be updated
-                            if vlan.name == n.name:
-                                vlan.set_vid(n.vlan_id)
-                                vlan.mtu = n.mtu
-                                vlan.update()
-                            else:
-                                vlan_id = n.vlan_id if n.vlan_id is not None else 0
-                                target_vlan = vlan_list.query({'vid': vlan_id})
-                                if len(target_vlan) > 0:
-                                    subnet.vlan = target_vlan[0].resource_id
-                                else:
-                                    # This is a flag that after creating a fabric and
-                                    # VLAN below, update the subnet
-                                    subnet.vlan = None
-                    else:
-                        subnet.vlan = None
-            
-                    # Check if the routes have a default route
-                    subnet.gateway_ip = n.get_default_gateway()
-
-
-                    result_detail['detail'].append("Subnet %s found for network %s, updated attributes"
-                                                % (exists[0].resource_id, n.name))
-
-                # Need to create a Fabric/Vlan for this network
-                if (subnet is None or (subnet is not None and subnet.vlan is None)):
-                    fabric_list = maas_fabric.Fabrics(self.maas_client)
-                    fabric_list.refresh()
-                    matching_fabrics = fabric_list.query({'name': n.name})
-                    
-                    fabric = None
-                    vlan = None
-                    
-                    if len(matching_fabrics) > 0:
-                        # Fabric exists, update VLAN
-                        fabric = matching_fabrics[0]
-
-                        vlan_list = maas_vlan.Vlans(self.maas_client, fabric_id=fabric.resource_id)
                         vlan_list.refresh()
-                        vlan_id = n.vlan_id if n.vlan_id is not None else 0
-                        matching_vlans = vlan_list.query({'vid': vlan_id})
 
-                        if len(matching_vlans) > 0:
+                        vlan = vlan_list.select(subnet.vlan)
-                            vlan = matching_vlans[0]
+
+                        if vlan is not None:
+                            if ((n.vlan_id is None and vlan.vid != 0) or 
+                                (n.vlan_id is not None and vlan.vid != n.vlan_id)):
+
+                                # if the VLAN name matches, assume this is the correct resource
+                                # and it needs to be updated
+                                if vlan.name == n.name:
+                                    vlan.set_vid(n.vlan_id)
+                                    vlan.mtu = n.mtu
+                                    vlan.update()
+                                    result_detail['detail'].append("VLAN %s found for network %s, updated attributes"
+                                                                    % (vlan.resource_id, n.name))
+                                else:
+                                    # Found a VLAN with the correct VLAN tag, update subnet to use it
+                                    target_vlan = vlan_list.singleton({'vid': n.vlan_id if n.vlan_id is not None else 0})
+                                    if target_vlan is not None:
+                                        subnet.vlan = target_vlan.resource_id
+                                    else:
+                                        # This is a flag that after creating a fabric and
+                                        # VLAN below, update the subnet
+                                        subnet.vlan = None
+                        else:
+                            subnet.vlan = None
+            
+                        # Check if the routes have a default route
+                        subnet.gateway_ip = n.get_default_gateway()
+
+
+                        result_detail['detail'].append("Subnet %s found for network %s, updated attributes"
+                                                % (subnet.resource_id, n.name))
+
+                    # Need to find or create a Fabric/Vlan for this subnet
+                    if (subnet is None or (subnet is not None and subnet.vlan is None)):
+                        fabric_list = maas_fabric.Fabrics(self.maas_client)
+                        fabric_list.refresh()
+                        fabric = fabric_list.singleton({'name': n.name})
+
+                        vlan = None
+                    
+                        if fabric is not None:
+                            vlan_list = maas_vlan.Vlans(self.maas_client, fabric_id=fabric.resource_id)
+                            vlan_list.refresh()
+                        
+                            vlan = vlan_list.singleton({'vid': n.vlan_id if n.vlan_id is not None else 0})
+
+                            if vlan is not None:
+                                vlan = matching_vlans[0]
+
+                                vlan.name = n.name
+                                if getattr(n, 'mtu', None) is not None:
+                                    vlan.mtu = n.mtu
+
+                                if subnet is not None:
+                                    subnet.vlan = vlan.resource_id
+                                    subnet.update()
+
+                                vlan.update()
+                                result_detail['detail'].append("VLAN %s found for network %s, updated attributes"
+                                                % (vlan.resource_id, n.name))
+                            else:
+                                # Create a new VLAN in this fabric and assign subnet to it
+                                vlan = maas_vlan.Vlan(self.maas_client, name=n.name, vid=vlan_id,
+                                                    mtu=getattr(n, 'mtu', None),fabric_id=fabric.resource_id)
+                                vlan = vlan_list.add(vlan)
+
+                                result_detail['detail'].append("VLAN %s created for network %s"
+                                                                % (vlan.resource_id, n.name))
+                                if subnet is not None:
+                                    subnet.vlan = vlan.resource_id
+                                    subnet.update()
+
+                        else:
+                            # Create new fabric and VLAN
+                            fabric = maas_fabric.Fabric(self.maas_client, name=n.name)
+                            fabric = fabric_list.add(fabric)
+                            fabric_list.refresh()
+
+                            result_detail['detail'].append("Fabric %s created for network %s"
+                                                                % (fabric.resource_id, n.name))
+
+                            vlan_list = maas_vlan.Vlans(self.maas_client, fabric_id=new_fabric.resource_id)
+                            vlan_list.refresh()
+
+                            # A new fabric comes with a single default VLAN. Retrieve it and update attributes
+                            vlan = vlan_list.single()
 
                             vlan.name = n.name
+                            vlan.vid = n.vlan_id if n.vlan_id is not None else 0
                             if getattr(n, 'mtu', None) is not None:
                                 vlan.mtu = n.mtu
 
-                            if subnet is not None:
-                                subnet.vlan = vlan.resource_id
-                                subnet.update()
                             vlan.update()
-                        else:
+                            result_detail['detail'].append("VLAN %s updated for network %s"
-                            vlan = maas_vlan.Vlan(self.maas_client, name=n.name, vid=vlan_id,
+                                                            % (vlan.resource_id, n.name))
-                                                mtu=getattr(n, 'mtu', None),fabric_id=fabric.resource_id)
-                            vlan = vlan_list.add(vlan)
-
-                            if subnet is not None:
-                                subnet.vlan = vlan.resource_id
-                                subnet.update()
-
-                    else:
-                        new_fabric = maas_fabric.Fabric(self.maas_client, name=n.name)
-                        new_fabric = fabric_list.add(new_fabric)
-                        new_fabric.refresh()
-                        fabric = new_fabric
-
-                        vlan_list = maas_vlan.Vlans(self.maas_client, fabric_id=new_fabric.resource_id)
-                        vlan_list.refresh()
-                        vlan = vlan_list.single()
-
-                        vlan.name = n.name
-                        vlan.vid = n.vlan_id if n.vlan_id is not None else 0
-                        if getattr(n, 'mtu', None) is not None:
-                            vlan.mtu = n.mtu
-
-                        vlan.update()
-
                         if subnet is not None:
+                            # If subnet was found above, but needed attached to a new fabric/vlan then
+                            # attach it
                             subnet.vlan = vlan.resource_id
                             subnet.update()
 
                     if subnet is None:
+                        # If subnet did not exist, create it here and attach it to the fabric/VLAN
                         subnet = maas_subnet.Subnet(self.maas_client, name=n.name, cidr=n.cidr, fabric=fabric.resource_id,
                                                 vlan=vlan.resource_id, gateway_ip=n.get_default_gateway())
 

helm_drydock/drivers/node/maasdriver/models/base.py

@@ -250,8 +250,23 @@ class ResourceCollectionBase(object):
 
         return result
 
+    def singleton(self, query):
+        """
+        A query that requires a single item response
+
+        :param query: A dict of k:v pairs defining the query parameters
+        """
+        result = self.query(query)
+
+        if len(result) > 1:
+            raise ValueError("Multiple results found")
+        elif len(result) == 1:
+            return result[0]
+
+        return None
+        
     """
-    If the collection has a single item, return it
+    If the collection contains a single item, return it
     """
     def single(self):
         if self.len() == 1:

helm_drydock/drivers/oob/__init__.py

@@ -22,7 +22,7 @@ class OobDriver(ProviderDriver):
     def __init__(self, **kwargs):
         super(OobDriver, self).__init__(**kwargs)
 
-        self.supported_actions = [hd_fields.OrchestrationAction.ValidateOobServices,
+        self.supported_actions = [hd_fields.OrchestratorAction.ValidateOobServices,
                                   hd_fields.OrchestratorAction.ConfigNodePxe,
                                   hd_fields.OrchestratorAction.SetNodeBoot,
                                   hd_fields.OrchestratorAction.PowerOffNode,

helm_drydock/drivers/oob/pyghmi_driver/__init__.py

@@ -34,7 +34,7 @@ class PyghmiDriver(oob.OobDriver):
         self.driver_key = "pyghmi_driver"
         self.driver_desc = "Pyghmi OOB Driver"
 
-        self.config = config.DrydockConfig.node_driver[self.driver_key]
+        self.config = config.DrydockConfig.node_driver.get(self.driver_key, {})
 
     def execute_task(self, task_id):
         task = self.state_manager.get_task(task_id)

helm_drydock/drydock.py

@@ -0,0 +1,55 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+
+import helm_drydock.config as config
+import helm_drydock.objects as objects
+import helm_drydock.ingester as ingester
+import helm_drydock.statemgmt as statemgmt
+import helm_drydock.orchestrator as orch
+import helm_drydock.control.api as api
+
+def start_drydock():
+    objects.register_all()
+    
+    # Setup root logger
+    logger = logging.getLogger('drydock')
+
+    logger.setLevel(config.DrydockConfig.global_config.get('log_level'))
+    ch = logging.StreamHandler()
+    formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
+    ch.setFormatter(formatter)
+    logger.addHandler(ch)
+
+    # Specalized format for API logging
+    logger = logging.getLogger('drydock.control')
+    logger.propagate = False
+    formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(user)s - %(req_id)s - %(external_ctx)s - %(message)s')
+
+    ch = logging.StreamHandler()
+    ch.setFormatter(formatter)
+    logger.addHandler(ch)
+
+    state = statemgmt.DesignState()
+
+    orchestrator = orch.Orchestrator(config.DrydockConfig.orchestrator_config.get('drivers', {}),
+                             state_manager=state)
+    input_ingester = ingester.Ingester()
+    input_ingester.enable_plugins(config.DrydockConfig.ingester_config.get('plugins', []))
+
+    return api.start_api(state_manager=state, ingester=input_ingester,
+                         orchestrator=orchestrator)
+
+drydock = start_drydock()
+

helm_drydock/error.py

@@ -34,4 +34,10 @@ class TransientDriverError(DriverError):
     pass
 
 class PersistentDriverError(DriverError):
+    pass
+
+class ApiError(Exception):
+    pass
+
+class InvalidFormat(ApiError):
     pass

helm_drydock/ingester/__init__.py

@@ -18,6 +18,7 @@
 import logging
 import yaml
 import  uuid
+import importlib
 
 import helm_drydock.objects as objects
 import helm_drydock.objects.site as site
@@ -31,55 +32,62 @@ from helm_drydock.statemgmt import DesignState
 class Ingester(object):
 
     def __init__(self):
-        logging.basicConfig(format="%(asctime)-15s [%(levelname)] %(module)s %(process)d %(message)s")
+        self.logger = logging.getLogger("drydock.ingester")
-        self.log = logging.Logger("ingester")
         self.registered_plugins = {}
 
     def enable_plugins(self, plugins=[]):
+        """
+        enable_plugins
+
+        :params plugins: - A list of strings naming class objects denoting the ingester plugins to be enabled
+
+        Enable plugins that can be used for ingest_data calls. Each plugin should use
+        helm_drydock.ingester.plugins.IngesterPlugin as its base class. As long as one
+        enabled plugin successfully initializes, the call is considered successful. Otherwise
+        it will throw an exception
+        """
         if len(plugins) == 0:
             self.log.error("Cannot have an empty plugin list.")
 
         for plugin in plugins:
             try:
-                new_plugin = plugin()
+                (module, x, classname) = plugin.rpartition('.')
+
+                if module == '':
+                    raise Exception()
+                mod = importlib.import_module(module)
+                klass = getattr(mod, classname)
+                new_plugin = klass()
                 plugin_name = new_plugin.get_name()
                 self.registered_plugins[plugin_name] = new_plugin
-            except:
+            except Exception as ex:
-                self.log.error("Could not enable plugin %s" % (plugin.__name__))
+                self.logger.error("Could not enable plugin %s - %s" % (plugin, str(ex)))
 
         if len(self.registered_plugins) == 0:
-            self.log.error("Could not enable at least one plugin")
+            self.logger.error("Could not enable at least one plugin")
             raise Exception("Could not enable at least one plugin")
-    """
-      enable_plugins
 
-      params: plugins - A list of class objects denoting the ingester plugins to be enabled
-
-      Enable plugins that can be used for ingest_data calls. Each plugin should use
-      helm_drydock.ingester.plugins.IngesterPlugin as its base class. As long as one
-      enabled plugin successfully initializes, the call is considered successful. Otherwise
-      it will throw an exception
-    """
     
-    def ingest_data(self, plugin_name='', design_state=None, **kwargs):
+    def ingest_data(self, plugin_name='', design_state=None, design_id=None, context=None, **kwargs):
         if design_state is None:
-            self.log.error("ingest_data called without valid DesignState handler")
+            self.logger.error("Ingester:ingest_data called without valid DesignState handler")
-            raise Exception("Invalid design_state handler")
+            raise ValueError("Invalid design_state handler")
-
-        design_data = None
 
         # If no design_id is specified, instantiate a new one
-        if 'design_id' not in kwargs.keys():
+        if 'design_id' is None:
-            design_id = str(uuid.uuid4())
+            self.logger.error("Ingester:ingest_data required kwarg 'design_id' missing")
-            design_data = objects.SiteDesign(id=design_id)
+            raise ValueError("Ingester:ingest_data required kwarg 'design_id' missing")
-            design_state.post_design(design_data)
+        
-        else:
+        design_data = design_state.get_design(design_id)
-            design_id = kwargs.get('design_id')
+
-            design_data = design_state.get_design(design_id)
+        self.logger.debug("Ingester:ingest_data ingesting design parts for design %s" % design_id)
 
         if plugin_name in self.registered_plugins:
             design_items = self.registered_plugins[plugin_name].ingest_data(**kwargs)
+            self.logger.debug("Ingester:ingest_data parsed %s design parts" % str(len(design_items)))
             for m in design_items:
+                if context is not None:
+                    m.set_create_fields(context)
                 if type(m) is site.Site:
                     design_data.set_site(m)
                 elif type(m) is network.Network:
@@ -93,8 +101,9 @@ class Ingester(object):
                 elif type(m) is node.BaremetalNode:
                     design_data.add_baremetal_node(m)
             design_state.put_design(design_data)
+            return design_items
         else:
-            self.log.error("Could not find plugin %s to ingest data." % (plugin_name))
+            self.logger.error("Could not find plugin %s to ingest data." % (plugin_name))
             raise LookupError("Could not find plugin %s" % plugin_name)
     """
       ingest_data

helm_drydock/ingester/plugins/yaml.py

@@ -28,6 +28,7 @@ class YamlIngester(IngesterPlugin):
 
     def __init__(self):
         super(YamlIngester, self).__init__()
+        self.logger = logging.getLogger('drydock.ingester.yaml')
 
     def get_name(self):
         return "yaml"
@@ -52,12 +53,10 @@ class YamlIngester(IngesterPlugin):
                     file.close()
                     models.extend(self.parse_docs(contents))
                 except OSError as err:
-                    self.log.error(
+                    self.logger.error(
                         "Error opening input file %s for ingestion: %s" 
                         % (filename, err))
                     continue
-        
-                
         elif 'content' in kwargs:
             models.extend(self.parse_docs(kwargs.get('content')))
         else:

helm_drydock/objects/base.py

@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+import datetime
 
 from oslo_versionedobjects import base
 from oslo_versionedobjects import fields as obj_fields
@@ -38,6 +39,32 @@ class DrydockObject(base.VersionedObject):
         else:
             raise ValueError("Unknown field %s" % (attrname))
 
+    def obj_to_simple(self):
+        """
+        Create a simple primitive representation of this object excluding
+        all the versioning stuff. Used to serialize an object for public
+        consumption, not intended to be deserialized by OVO
+        """
+
+        primitive = dict()
+
+        primitive['model_type'] = self.__class__.__name__
+        primitive['model_version'] = self.VERSION
+
+        for name, field in self.fields.items():
+            if self.obj_attr_is_set(name):
+                value = getattr(self, name)
+                if (hasattr(value, 'obj_to_simple') and
+                    callable(value.obj_to_simple)):
+                    primitive[name] = value.obj_to_simple()
+                else:
+                    value = field.to_primitive(self, name, value)
+                    if value is not None:
+                        primitive[name] = value
+
+        return primitive
+
+
 class DrydockPersistentObject(base.VersionedObject):
 
     fields = {
@@ -47,6 +74,15 @@ class DrydockPersistentObject(base.VersionedObject):
         'updated_by': obj_fields.StringField(nullable=True),
     }
 
+    def set_create_fields(self, context):
+        self.created_at = datetime.datetime.now()
+        self.created_by = context.user
+
+    def set_update_fields(self, context):
+        self.updated_at = datetime.datetime.now()
+        self.updated_by = context.user
+
+
 class DrydockObjectListBase(base.ObjectListBase):
     
     def __init__(self, **kwargs):
@@ -73,3 +109,11 @@ class DrydockObjectListBase(base.ObjectListBase):
             model_list.append(o)
 
         return model_list
+
+    def obj_to_simple(self):
+        primitive_list = list()
+
+        for o in self.objects:
+            primitive_list.append(o.obj_to_simple())
+
+        return primitive_list

helm_drydock/objects/site.py

@@ -16,6 +16,7 @@
 #
 from copy import deepcopy
 import uuid
+import datetime
 
 import oslo_versionedobjects.fields as ovo_fields
 
@@ -126,8 +127,6 @@ class SiteDesign(base.DrydockPersistentObject, base.DrydockObject):
     def __init__(self, **kwargs):
         super(SiteDesign, self).__init__(**kwargs)
 
-
-
     # Assign UUID id
     def assign_id(self):
         self.id = uuid.uuid4()
@@ -228,6 +227,18 @@ class SiteDesign(base.DrydockPersistentObject, base.DrydockObject):
         raise DesignError("BaremetalNode %s not found in design state"
                           % node_key)
 
+    def create(self, ctx, state_manager):
+        self.created_at = datetime.datetime.now()
+        self.created_by = ctx.user
+
+        state_manager.post_design(self)
+
+    def save(self, ctx, state_manager):
+        self.updated_at = datetime.datetime.now()
+        self.updated_by = ctx.user
+
+        state_manager.put_design(self)
+
     """
     Support filtering on rack name, node name or node tag
     for now. Each filter can be a comma-delimited list of

helm_drydock/objects/task.py

@@ -66,18 +66,29 @@ class Task(object):
     def get_subtasks(self):
         return self.subtasks
 
+    def to_dict(self):
+        return {
+            'task_id':  str(self.task_id),
+            'action':   self.action,
+            'parent_task': str(self.parent_task_id),
+            'status':   self.status,
+            'result':   self.result,
+            'result_detail': self.result_detail,
+            'subtasks': [str(x) for x in self.subtasks],
+        }
+
 class OrchestratorTask(Task):
 
-    def __init__(self, **kwargs):
+    def __init__(self, site=None, design_id=None, **kwargs):
         super(OrchestratorTask, self).__init__(**kwargs)
 
         # Validate parameters based on action
-        self.site = kwargs.get('site', '')
+        self.site = site
 
-        if self.site == '':
+        if self.site is None:
             raise ValueError("Orchestration Task requires 'site' parameter")
 
-        self.design_id = kwargs.get('design_id', 0)
+        self.design_id = design_id
 
         if self.action in [hd_fields.OrchestratorAction.VerifyNode,
                       hd_fields.OrchestratorAction.PrepareNode,
@@ -85,6 +96,14 @@ class OrchestratorTask(Task):
                       hd_fields.OrchestratorAction.DestroyNode]:
             self.node_filter = kwargs.get('node_filter', None)
 
+    def to_dict(self):
+        _dict = super(OrchestratorTask, self).to_dict()
+
+        _dict['site'] = self.site
+        _dict['design_id'] = self.design_id
+        _dict['node_filter'] = getattr(self, 'node_filter', None)
+
+        return _dict
 
 class DriverTask(Task):
     def __init__(self, task_scope={}, **kwargs):
@@ -94,4 +113,13 @@ class DriverTask(Task):
 
         self.site_name = task_scope.get('site', None)
 
-        self.node_list = task_scope.get('node_names', [])
+        self.node_list = task_scope.get('node_names', [])
+
+    def to_dict(self):
+        _dict = super(DriverTask, self).to_dict()
+
+        _dict['site_name'] = self.site_name
+        _dict['design_id'] = self.design_id
+        _dict['node_list'] = self.node_list
+
+        return _dict

helm_drydock/orchestrator/__init__.py

@@ -1,3 +1,4 @@
+
 # Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -106,8 +107,7 @@ class Orchestrator(object):
             self.task_field_update(task_id,
                                    status=hd_fields.TaskStatus.Running)
             try:
-                site_design = self.get_effective_site(task_site,
+                site_design = self.get_effective_site(design_id)
-                                                  change_id=design_id)
                 self.task_field_update(task_id,
                                        result=hd_fields.ActionResult.Success)
             except:
@@ -175,7 +175,7 @@ class Orchestrator(object):
                         result=hd_fields.ActionResult.Failure)
                 return
 
-            site_design = self.get_effective_site(design_id, task_site)
+            site_design = self.get_effective_site(design_id)
 
             node_filter = task.node_filter
 
@@ -212,8 +212,7 @@ class Orchestrator(object):
                         result=hd_fields.ActionResult.Failure)
                 return
 
-            site_design = self.get_effective_site(task_site,
+            site_design = self.get_effective_site(design_id)
-                                                  change_id=design_id)
 
             node_filter = task.node_filter
 
@@ -331,7 +330,7 @@ class Orchestrator(object):
         # the baremetal nodes which recursively resolves it for host profiles
         # assigned to those nodes
 
-        for n in site_design.baremetal_nodes:
+        for n in getattr(site_design, 'baremetal_nodes', []):
             n.compile_applied_model(site_design)
         
         return
@@ -342,18 +341,13 @@ class Orchestrator(object):
     return a Site model reflecting the effective design for the site
     """
 
-    def get_described_site(self, design_id, site_name):
+    def get_described_site(self, design_id):
-        site_design = None
-
-        if site_name is None:
-            raise errors.OrchestratorError("Cannot source design for site None")
-
         site_design = self.state_manager.get_design(design_id)
         
         return site_design
 
-    def get_effective_site(self, design_id, site_name):
+    def get_effective_site(self, design_id):
-        site_design = self.get_described_site(design_id, site_name)
+        site_design = self.get_described_site(design_id)
 
         self.compute_model_inheritance(site_design)
 

helm_drydock/statemgmt/__init__.py

@@ -41,6 +41,7 @@ class DesignState(object):
     # has started
     def get_design(self, design_id):
         if design_id not in self.designs.keys():
+
             raise DesignError("Design ID %s not found" % (design_id))
 
         return objects.SiteDesign.obj_from_primitive(self.designs[design_id])
@@ -133,7 +134,7 @@ class DesignState(object):
 
     def get_task(self, task_id):
         for t in self.tasks:
-            if t.get_id() == task_id:
+            if t.get_id() == task_id or str(t.get_id()) == task_id:
                 return deepcopy(t)
         return None
 

setup.py

@@ -51,7 +51,8 @@ setup(name='helm_drydock',
                 'helm_drydock.drivers.oob.pyghmi_driver',
                 'helm_drydock.drivers.node',
                 'helm_drydock.drivers.node.maasdriver',
-                'helm_drydock.drivers.node.maasdriver.models'],
+                'helm_drydock.drivers.node.maasdriver.models',
+                'helm_drydock.control'],
       install_requires=[
         'PyYAML',
         'pyghmi>=1.0.18',
@@ -60,6 +61,7 @@ setup(name='helm_drydock',
         'oslo.versionedobjects>=1.23.0',
         'requests',
         'oauthlib',
+        'uwsgi>1.4',
       ]
      )