diff --git a/README.md b/README.md index dc619477..00c008f2 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,13 @@ # helm_drydock A python REST orchestrator to translate a YAML host topology to a provisioned set of hosts and provide a set of cloud-init post-provisioning instructions. +To run: + + $ virtualenv -p python3 /var/tmp/drydock + $ . /var/tmp/drydock/bin/activate + $ python setup.py install + $ uwsgi --http :9000 -w helm_drydock.drydock --callable drydock --enable-threads -L + ## Modular service ### Design Consumer ### diff --git a/helm_drydock/config.py b/helm_drydock/config.py index 244e85c0..b07a9b9e 100644 --- a/helm_drydock/config.py +++ b/helm_drydock/config.py @@ -21,13 +21,24 @@ class DrydockConfig(object): + global_config = { + 'log_level': 'DEBUG', + } + node_driver = { 'maasdriver': { 'api_key': 'KTMHgA42cNSMnfmJ82:cdg4yQUhp542aHsCTV:7Dc2KB9hQpWq3LfQAAAKAj6wdg22yWxZ', - 'api_url': 'http://localhost:5240/MAAS/api/2.0/' + 'api_url': 'http://localhost:5240/MAAS/api/2.0/', }, } ingester_config = { - 'plugins': ['helm_drydock.ingester.plugins.yaml'] + 'plugins': ['helm_drydock.ingester.plugins.yaml.YamlIngester'], + } + + orchestrator_config = { + 'drivers': { + 'oob': 'helm_drydock.drivers.oob.pyghmi_driver.PyghmiDriver', + 'node': 'helm_drydock.drivers.node.maasdriver.driver.MaasNodeDriver', + } } \ No newline at end of file diff --git a/helm_drydock/control/__init__.py b/helm_drydock/control/__init__.py index 2a385a45..f10bbbf6 100644 --- a/helm_drydock/control/__init__.py +++ b/helm_drydock/control/__init__.py @@ -10,4 +10,4 @@ # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and -# limitations under the License. \ No newline at end of file +# limitations under the License. diff --git a/helm_drydock/control/api.py b/helm_drydock/control/api.py index e584cdc0..7f32110c 100644 --- a/helm_drydock/control/api.py +++ b/helm_drydock/control/api.py @@ -13,24 +13,35 @@ # limitations under the License. import falcon -from .designs import DesignsResource, DesignPartsResource -from .tasks import TasksResource +from .designs import * +from .tasks import * + from .base import DrydockRequest from .middleware import AuthMiddleware, ContextMiddleware, LoggingMiddleware -def start_api(state_manager): +def start_api(state_manager=None, ingester=None, orchestrator=None): + """ + Start the Drydock API service + + :param state_manager: Instance of helm_drydock.statemgmt.manager.DesignState for accessing + state persistence + :param ingester: Instance of helm_drydock.ingester.ingester.Ingester for handling design + part input + """ control_api = falcon.API(request_type=DrydockRequest, middleware=[AuthMiddleware(), ContextMiddleware(), LoggingMiddleware()]) # API for managing orchestrator tasks - control_api.add_route('/tasks', TasksResource(state_manager=state_manager)) + control_api.add_route('/tasks', TasksResource(state_manager=state_manager, orchestrator=orchestrator)) control_api.add_route('/tasks/{task_id}', TaskResource(state_manager=state_manager)) # API for managing site design data control_api.add_route('/designs', DesignsResource(state_manager=state_manager)) - control_api.add_route('/designs/{design_id}', DesignResource(state_manager=state_manager)) - control_api.add_route('/designs/{design_id}/parts', DesignsPartsResource(state_manager=state_manager)) + control_api.add_route('/designs/{design_id}', DesignResource(state_manager=state_manager, orchestrator=orchestrator)) + control_api.add_route('/designs/{design_id}/parts', DesignsPartsResource(state_manager=state_manager, ingester=ingester)) control_api.add_route('/designs/{design_id}/parts/{kind}', DesignsPartsKindsResource(state_manager=state_manager)) - control_api.add_route('/designs/{design_id}/parts/{kind}/{name}', DesignsPartResource(state_manager=state_manager)) + + control_api.add_route('/designs/{design_id}/parts/{kind}/{name}', + DesignsPartResource(state_manager=state_manager, orchestrator=orchestrator)) return control_api diff --git a/helm_drydock/control/base.py b/helm_drydock/control/base.py index 04a1f8b1..289b6161 100644 --- a/helm_drydock/control/base.py +++ b/helm_drydock/control/base.py @@ -13,9 +13,17 @@ # limitations under the License. import falcon.request as request import uuid +import json +import logging + +import helm_drydock.error as errors class BaseResource(object): + def __init__(self): + self.logger = logging.getLogger('control') + self.authorized_roles = [] + def on_options(self, req, resp): self_attrs = dir(self) methods = ['GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'PATCH'] @@ -28,9 +36,68 @@ class BaseResource(object): resp.headers['Allow'] = ','.join(allowed_methods) resp.status = falcon.HTTP_200 - # By default, no one is authorized to use a resource + # For authorizing access at the Resource level. A Resource requiring + # finer grained authorization at the method or instance level must + # implement that in the request handlers def authorize_roles(self, role_list): - return False + authorized = set(self.authorized_roles) + applied = set(role_list) + + if authorized.isdisjoint(applied): + return False + else: + return True + + def req_json(self, req): + if req.content_length is None or req.content_length == 0: + return None + + if req.content_type is not None and req.content_type.lower() == 'application/json': + raw_body = req.stream.read(req.content_length or 0) + + if raw_body is None: + return None + + try: + json_body = json.loads(raw_body.decode('utf-8')) + return json_body + except json.JSONDecodeError as jex: + raise errors.InvalidFormat("%s: Invalid JSON in body: %s" % (req.path, jex)) + else: + raise errors.InvalidFormat("Requires application/json payload") + + def return_error(self, resp, status_code, message="", retry=False): + resp.body = json.dumps({'type': 'error', 'message': message, 'retry': retry}) + resp.status = status_code + + def log_error(self, ctx, level, msg): + extra = { + 'user': 'N/A', + 'req_id': 'N/A', + 'external_ctx': 'N/A' + } + + if ctx is not None: + extra = { + 'user': ctx.user, + 'req_id': ctx.request_id, + 'external_ctx': ctx.external_marker, + } + + self.logger.log(level, msg, extra=extra) + + def debug(self, ctx, msg): + self.log_error(ctx, logging.DEBUG, msg) + + def info(self, ctx, msg): + self.log_error(ctx, logging.INFO, msg) + + def warn(self, ctx, msg): + self.log_error(ctx, logging.WARN, msg) + + def error(self, ctx, msg): + self.log_error(ctx, logging.ERROR, msg) + class StatefulResource(BaseResource): @@ -38,6 +105,7 @@ class StatefulResource(BaseResource): super(StatefulResource, self).__init__() if state_manager is None: + self.error(None, "StatefulResource:init - StatefulResources require a state manager be set") raise ValueError("StatefulResources require a state manager be set") self.state_manager = state_manager @@ -46,7 +114,7 @@ class StatefulResource(BaseResource): class DrydockRequestContext(object): def __init__(self): - self.log_level = 'error' + self.log_level = 'ERROR' self.user = None self.roles = ['anyone'] self.request_id = str(uuid.uuid4()) @@ -72,5 +140,5 @@ class DrydockRequestContext(object): def set_external_marker(self, marker): self.external_marker = str(marker)[:32] -class DrydockRequest(request.Request) +class DrydockRequest(request.Request): context_type = DrydockRequestContext \ No newline at end of file diff --git a/helm_drydock/control/designs.py b/helm_drydock/control/designs.py index eead1d19..402b23ad 100644 --- a/helm_drydock/control/designs.py +++ b/helm_drydock/control/designs.py @@ -13,6 +13,11 @@ # limitations under the License. import falcon import json +import uuid +import logging + +import helm_drydock.objects as hd_objects +import helm_drydock.error as errors from .base import StatefulResource @@ -20,26 +25,140 @@ class DesignsResource(StatefulResource): def __init__(self, **kwargs): super(DesignsResource, self).__init__(**kwargs) + self.authorized_roles = ['user'] def on_get(self, req, resp): state = self.state_manager - designs = state.designs.keys() + designs = list(state.designs.keys()) resp.body = json.dumps(designs) resp.status = falcon.HTTP_200 - def authorize_roles(self, role_list): - if 'user' in role_list: - return True - - return False + def on_post(self, req, resp): + try: + json_data = self.req_json(req) + design = None + if json_data is not None: + base_design = json_data.get('base_design_id', None) + + if base_design is not None: + base_design = uuid.UUID(base_design) + design = hd_objects.SiteDesign(base_design_id=base_design_uuid) + else: + design = hd_objects.SiteDesign() + design.assign_id() + design.create(req.context, self.state_manager) + + resp.body = json.dumps(design.obj_to_simple()) + resp.status = falcon.HTTP_201 + except errors.StateError as stex: + self.error(req.context, "Error updating persistence") + self.return_error(resp, falcon.HTTP_500, message="Error updating persistence", retry=True) + except errors.InvalidFormat as fex: + self.error(req.context, str(fex)) + self.return_error(resp, falcon.HTTP_400, message=str(fex), retry=False) + class DesignResource(StatefulResource): + def __init__(self, orchestrator=None, **kwargs): + super(DesignResource, self).__init__(**kwargs) + self.authorized_roles = ['user'] + self.orchestrator = orchestrator + + def on_get(self, req, resp, design_id): + source = req.params.get('source', 'designed') + + try: + design = None + if source == 'compiled': + design = self.orchestrator.get_effective_site(design_id) + elif source == 'designed': + design = self.orchestrator.get_described_site(design_id) + + resp.body = json.dumps(design.obj_to_simple()) + except errors.DesignError: + self.error(req.context, "Design %s not found" % design_id) + self.return_error(resp, falcon.HTTP_404, message="Design %s not found" % design_id, retry=False) + class DesignsPartsResource(StatefulResource): + def __init__(self, ingester=None, **kwargs): + super(DesignsPartsResource, self).__init__(**kwargs) + self.ingester = ingester + self.authorized_roles = ['user'] + + if ingester is None: + self.error(None, "DesignsPartsResource requires a configured Ingester instance") + raise ValueError("DesignsPartsResource requires a configured Ingester instance") + + def on_post(self, req, resp, design_id): + ingester_name = req.params.get('ingester', None) + + if ingester_name is None: + self.error(None, "DesignsPartsResource POST requires parameter 'ingester'") + self.return_error(resp, falcon.HTTP_400, message="POST requires parameter 'ingester'", retry=False) + else: + try: + raw_body = req.stream.read(req.content_length or 0) + if raw_body is not None and len(raw_body) > 0: + parsed_items = self.ingester.ingest_data(plugin_name=ingester_name, design_state=self.state_manager, + content=raw_body, design_id=design_id, context=req.context) + resp.status = falcon.HTTP_201 + resp.body = json.dumps([x.obj_to_simple() for x in parsed_items]) + else: + self.return_error(resp, falcon.HTTP_400, message="Empty body not supported", retry=False) + except ValueError: + self.return_error(resp, falcon.HTTP_500, message="Error processing input", retry=False) + except LookupError: + self.return_error(resp, falcon.HTTP_400, message="Ingester %s not registered" % ingester_name, retry=False) + + class DesignsPartsKindsResource(StatefulResource): + def __init__(self, **kwargs): + super(DesignsPartsKindsResource, self).__init__(**kwargs) + self.authorized_roles = ['user'] + + def on_get(self, req, resp, design_id, kind): + pass class DesignsPartResource(StatefulResource): + def __init__(self, orchestrator=None, **kwargs): + super(DesignsPartResource, self).__init__(**kwargs) + self.authorized_roles = ['user'] + self.orchestrator = orchestrator + + def on_get(self, req , resp, design_id, kind, name): + source = req.params.get('source', 'designed') + + try: + design = None + if source == 'compiled': + design = self.orchestrator.get_effective_site(design_id) + elif source == 'designed': + design = self.orchestrator.get_described_site(design_id) + + part = None + if kind == 'Site': + part = design.get_site() + elif kind == 'Network': + part = design.get_network(name) + elif kind == 'NetworkLink': + part = design.get_network_link(name) + elif kind == 'HardwareProfile': + part = design.get_hardware_profile(name) + elif kind == 'HostProfile': + part = design.get_host_profile(name) + elif kind == 'BaremetalNode': + part = design.get_baremetal_node(name) + else: + self.error(req.context, "Kind %s unknown" % kind) + self.return_error(resp, falcon.HTTP_404, message="Kind %s unknown" % kind, retry=False) + return + + resp.body = json.dumps(part.obj_to_simple()) + except errors.DesignError as dex: + self.error(req.context, str(dex)) + self.return_error(resp, falcon.HTTP_404, message=str(dex), retry=False) \ No newline at end of file diff --git a/helm_drydock/control/middleware.py b/helm_drydock/control/middleware.py index 2ab5ad16..da877b36 100644 --- a/helm_drydock/control/middleware.py +++ b/helm_drydock/control/middleware.py @@ -13,6 +13,10 @@ # limitations under the License. import falcon +import logging +import uuid + +import helm_drydock.config as config class AuthMiddleware(object): @@ -31,7 +35,7 @@ class AuthMiddleware(object): ctx.add_role('anyone') # Authorization - def process_resource(self, req, resp, resource): + def process_resource(self, req, resp, resource, params): ctx = req.context if not resource.authorize_roles(ctx.roles): @@ -62,12 +66,29 @@ class ContextMiddleware(object): requested_logging = req.get_header('X-Log-Level') - if requested_logging == 'DEBUG' and 'admin' in ctx.roles: - ctx.set_log_level('debug') + if (config.DrydockConfig.global_config.get('log_level', '') == 'DEBUG' or + (requested_logging == 'DEBUG' and 'admin' in ctx.roles)): + ctx.set_log_level('DEBUG') elif requested_logging == 'INFO': - ctx.set_log_level('info') + ctx.set_log_level('INFO') + + ctx.req_id = str(uuid.uuid4()) + + ext_marker = req.get_header('X-Context-Marker') + + ctx.external_ctx = ext_marker if ext_marker is not None else '' class LoggingMiddleware(object): + def __init__(self): + self.logger = logging.getLogger('drydock.control') + def process_response(self, req, resp, resource, req_succeeded): - ctx = req.context \ No newline at end of file + ctx = req.context + extra = { + 'user': ctx.user, + 'req_id': ctx.req_id, + 'external_ctx': ctx.external_ctx, + } + resp.append_header('X-Drydock-Req', ctx.req_id) + self.logger.info("%s - %s" % (req.uri, resp.status), extra=extra) diff --git a/helm_drydock/control/readme.md b/helm_drydock/control/readme.md index a4bf0d13..a825e8ac 100644 --- a/helm_drydock/control/readme.md +++ b/helm_drydock/control/readme.md @@ -16,10 +16,17 @@ DELETE - Cancel execution of a task if permitted ### /designs ### POST - Create a new site design so design parts can be added -GET - Get a current design if available + +### /designs/{id} + +GET - Get a current design if available. Param 'source=compiled' to calculate the inheritance chain and compile the effective design. ### /designs/{id}/parts POST - Submit a new design part to be ingested and added to this design GET - View a currently defined design part -PUT - Replace an existing design part \ No newline at end of file +PUT - Replace an existing design part *Not Implemented* + +### /designs/{id}/parts/{kind}/{name} + +GET - View a single design part. param 'source=compiled' to calculate the inheritance chain and compile the effective configuration for the design part. \ No newline at end of file diff --git a/helm_drydock/control/tasks.py b/helm_drydock/control/tasks.py index 9c65e925..bf35861f 100644 --- a/helm_drydock/control/tasks.py +++ b/helm_drydock/control/tasks.py @@ -12,9 +12,68 @@ # See the License for the specific language governing permissions and # limitations under the License. import falcon +import json +import threading +import traceback +import helm_drydock.objects.task as obj_task from .base import StatefulResource class TasksResource(StatefulResource): + def __init__(self, orchestrator=None, **kwargs): + super(TasksResource, self).__init__(**kwargs) + self.authorized_roles = ['user'] + self.orchestrator = orchestrator + + def on_get(self, req, resp): + task_id_list = [str(x.get_id()) for x in self.state_manager.tasks] + resp.body = json.dumps(task_id_list) + + def on_post(self, req, resp): + try: + json_data = self.req_json(req) + + sitename = json_data.get('sitename', None) + design_id = json_data.get('design_id', None) + action = json_data.get('action', None) + + if sitename is None or design_id is None or action is None: + self.info(req.context, "Task creation requires fields sitename, design_id, action") + self.return_error(resp, falcon.HTTP_400, message="Task creation requires fields sitename, design_id, action", retry=False) + return + + task = self.orchestrator.create_task(obj_task.OrchestratorTask, site=sitename, + design_id=design_id, action=action) + + task_thread = threading.Thread(target=self.orchestrator.execute_task, args=[task.get_id()]) + task_thread.start() + + resp.body = json.dumps(task.to_dict()) + resp.status = falcon.HTTP_201 + except Exception as ex: + self.error(req.context, "Unknown error: %s\n%s" % (str(ex), traceback.format_exc())) + self.return_error(resp, falcon.HTTP_500, message="Unknown error", retry=False) + + class TaskResource(StatefulResource): + + def __init__(self, orchestrator=None, **kwargs): + super(TaskResource, self).__init__(**kwargs) + self.authorized_roles = ['user'] + self.orchestrator = orchestrator + + def on_get(self, req, resp, task_id): + try: + task = self.state_manager.get_task(task_id) + + if task is None: + self.info(req.context, "Task %s does not exist" % task_id ) + self.return_error(resp, falcon.HTTP_404, message="Task %s does not exist" % task_id, retry=False) + return + + resp.body = json.dumps(task.to_dict()) + resp.status = falcon.HTTP_200 + except Exception as ex: + self.error(req.context, "Unknown error: %s" % (str(ex))) + self.return_error(resp, falcon.HTTP_500, message="Unknown error", retry=False) diff --git a/helm_drydock/drivers/node/maasdriver/driver.py b/helm_drydock/drivers/node/maasdriver/driver.py index 83406a18..ef26b456 100644 --- a/helm_drydock/drivers/node/maasdriver/driver.py +++ b/helm_drydock/drivers/node/maasdriver/driver.py @@ -101,7 +101,7 @@ class MaasNodeDriver(NodeDriver): self.orchestrator.task_field_update(task.get_id(), status=hd_fields.TaskStatus.Running) - site_design = self.orchestrator.get_effective_site(design_id, task.site_name) + site_design = self.orchestrator.get_effective_site(design_id) if task.action == hd_fields.OrchestratorAction.CreateNetworkTemplate: subtask = self.orchestrator.create_task(task_model.DriverTask, @@ -149,8 +149,7 @@ class MaasTaskRunner(drivers.DriverTaskRunner): self.maas_client = MaasRequestFactory(self.driver_config['api_url'], self.driver_config['api_key']) - site_design = self.orchestrator.get_effective_site(self.task.design_id, - self.task.site_name) + site_design = self.orchestrator.get_effective_site(self.task.design_id) if task_action == hd_fields.OrchestratorAction.CreateNetworkTemplate: # Try to true up MaaS definitions of fabrics/vlans/subnets @@ -165,110 +164,120 @@ class MaasTaskRunner(drivers.DriverTaskRunner): } for n in design_networks: - exists = subnets.query({'cidr': n.cidr}) + try: + subnet = subnets.singleton({'cidr': n.cidr}) - subnet = None + if subnet is not None: + subnet.name = n.name + subnet.dns_servers = n.dns_servers - if len(exists) > 0: - subnet = exists[0] - - subnet.name = n.name - subnet.dns_servers = n.dns_servers - - vlan_list = maas_vlan.Vlans(self.maas_client, fabric_id=subnet.fabric) - vlan_list.refresh() - - vlan = vlan_list.select(subnet.vlan) - - if vlan is not None: - if ((n.vlan_id is None and vlan.vid != 0) or - (n.vlan_id is not None and vlan.vid != n.vlan_id)): - - # if the VLAN name matches, assume this is the correct resource - # and it needs to be updated - if vlan.name == n.name: - vlan.set_vid(n.vlan_id) - vlan.mtu = n.mtu - vlan.update() - else: - vlan_id = n.vlan_id if n.vlan_id is not None else 0 - target_vlan = vlan_list.query({'vid': vlan_id}) - if len(target_vlan) > 0: - subnet.vlan = target_vlan[0].resource_id - else: - # This is a flag that after creating a fabric and - # VLAN below, update the subnet - subnet.vlan = None - else: - subnet.vlan = None - - # Check if the routes have a default route - subnet.gateway_ip = n.get_default_gateway() - - - result_detail['detail'].append("Subnet %s found for network %s, updated attributes" - % (exists[0].resource_id, n.name)) - - # Need to create a Fabric/Vlan for this network - if (subnet is None or (subnet is not None and subnet.vlan is None)): - fabric_list = maas_fabric.Fabrics(self.maas_client) - fabric_list.refresh() - matching_fabrics = fabric_list.query({'name': n.name}) - - fabric = None - vlan = None - - if len(matching_fabrics) > 0: - # Fabric exists, update VLAN - fabric = matching_fabrics[0] - - vlan_list = maas_vlan.Vlans(self.maas_client, fabric_id=fabric.resource_id) + vlan_list = maas_vlan.Vlans(self.maas_client, fabric_id=subnet.fabric) vlan_list.refresh() - vlan_id = n.vlan_id if n.vlan_id is not None else 0 - matching_vlans = vlan_list.query({'vid': vlan_id}) - if len(matching_vlans) > 0: - vlan = matching_vlans[0] + vlan = vlan_list.select(subnet.vlan) + + if vlan is not None: + if ((n.vlan_id is None and vlan.vid != 0) or + (n.vlan_id is not None and vlan.vid != n.vlan_id)): + + # if the VLAN name matches, assume this is the correct resource + # and it needs to be updated + if vlan.name == n.name: + vlan.set_vid(n.vlan_id) + vlan.mtu = n.mtu + vlan.update() + result_detail['detail'].append("VLAN %s found for network %s, updated attributes" + % (vlan.resource_id, n.name)) + else: + # Found a VLAN with the correct VLAN tag, update subnet to use it + target_vlan = vlan_list.singleton({'vid': n.vlan_id if n.vlan_id is not None else 0}) + if target_vlan is not None: + subnet.vlan = target_vlan.resource_id + else: + # This is a flag that after creating a fabric and + # VLAN below, update the subnet + subnet.vlan = None + else: + subnet.vlan = None + + # Check if the routes have a default route + subnet.gateway_ip = n.get_default_gateway() + + + result_detail['detail'].append("Subnet %s found for network %s, updated attributes" + % (subnet.resource_id, n.name)) + + # Need to find or create a Fabric/Vlan for this subnet + if (subnet is None or (subnet is not None and subnet.vlan is None)): + fabric_list = maas_fabric.Fabrics(self.maas_client) + fabric_list.refresh() + fabric = fabric_list.singleton({'name': n.name}) + + vlan = None + + if fabric is not None: + vlan_list = maas_vlan.Vlans(self.maas_client, fabric_id=fabric.resource_id) + vlan_list.refresh() + + vlan = vlan_list.singleton({'vid': n.vlan_id if n.vlan_id is not None else 0}) + + if vlan is not None: + vlan = matching_vlans[0] + + vlan.name = n.name + if getattr(n, 'mtu', None) is not None: + vlan.mtu = n.mtu + + if subnet is not None: + subnet.vlan = vlan.resource_id + subnet.update() + + vlan.update() + result_detail['detail'].append("VLAN %s found for network %s, updated attributes" + % (vlan.resource_id, n.name)) + else: + # Create a new VLAN in this fabric and assign subnet to it + vlan = maas_vlan.Vlan(self.maas_client, name=n.name, vid=vlan_id, + mtu=getattr(n, 'mtu', None),fabric_id=fabric.resource_id) + vlan = vlan_list.add(vlan) + + result_detail['detail'].append("VLAN %s created for network %s" + % (vlan.resource_id, n.name)) + if subnet is not None: + subnet.vlan = vlan.resource_id + subnet.update() + + else: + # Create new fabric and VLAN + fabric = maas_fabric.Fabric(self.maas_client, name=n.name) + fabric = fabric_list.add(fabric) + fabric_list.refresh() + + result_detail['detail'].append("Fabric %s created for network %s" + % (fabric.resource_id, n.name)) + + vlan_list = maas_vlan.Vlans(self.maas_client, fabric_id=new_fabric.resource_id) + vlan_list.refresh() + + # A new fabric comes with a single default VLAN. Retrieve it and update attributes + vlan = vlan_list.single() vlan.name = n.name + vlan.vid = n.vlan_id if n.vlan_id is not None else 0 if getattr(n, 'mtu', None) is not None: vlan.mtu = n.mtu - if subnet is not None: - subnet.vlan = vlan.resource_id - subnet.update() vlan.update() - else: - vlan = maas_vlan.Vlan(self.maas_client, name=n.name, vid=vlan_id, - mtu=getattr(n, 'mtu', None),fabric_id=fabric.resource_id) - vlan = vlan_list.add(vlan) - - if subnet is not None: - subnet.vlan = vlan.resource_id - subnet.update() - - else: - new_fabric = maas_fabric.Fabric(self.maas_client, name=n.name) - new_fabric = fabric_list.add(new_fabric) - new_fabric.refresh() - fabric = new_fabric - - vlan_list = maas_vlan.Vlans(self.maas_client, fabric_id=new_fabric.resource_id) - vlan_list.refresh() - vlan = vlan_list.single() - - vlan.name = n.name - vlan.vid = n.vlan_id if n.vlan_id is not None else 0 - if getattr(n, 'mtu', None) is not None: - vlan.mtu = n.mtu - - vlan.update() - + result_detail['detail'].append("VLAN %s updated for network %s" + % (vlan.resource_id, n.name)) if subnet is not None: + # If subnet was found above, but needed attached to a new fabric/vlan then + # attach it subnet.vlan = vlan.resource_id subnet.update() if subnet is None: + # If subnet did not exist, create it here and attach it to the fabric/VLAN subnet = maas_subnet.Subnet(self.maas_client, name=n.name, cidr=n.cidr, fabric=fabric.resource_id, vlan=vlan.resource_id, gateway_ip=n.get_default_gateway()) diff --git a/helm_drydock/drivers/node/maasdriver/models/base.py b/helm_drydock/drivers/node/maasdriver/models/base.py index 9f3aa336..c64257f3 100644 --- a/helm_drydock/drivers/node/maasdriver/models/base.py +++ b/helm_drydock/drivers/node/maasdriver/models/base.py @@ -250,8 +250,23 @@ class ResourceCollectionBase(object): return result + def singleton(self, query): + """ + A query that requires a single item response + + :param query: A dict of k:v pairs defining the query parameters + """ + result = self.query(query) + + if len(result) > 1: + raise ValueError("Multiple results found") + elif len(result) == 1: + return result[0] + + return None + """ - If the collection has a single item, return it + If the collection contains a single item, return it """ def single(self): if self.len() == 1: diff --git a/helm_drydock/drivers/oob/__init__.py b/helm_drydock/drivers/oob/__init__.py index ada30fb8..18fc0c75 100644 --- a/helm_drydock/drivers/oob/__init__.py +++ b/helm_drydock/drivers/oob/__init__.py @@ -22,7 +22,7 @@ class OobDriver(ProviderDriver): def __init__(self, **kwargs): super(OobDriver, self).__init__(**kwargs) - self.supported_actions = [hd_fields.OrchestrationAction.ValidateOobServices, + self.supported_actions = [hd_fields.OrchestratorAction.ValidateOobServices, hd_fields.OrchestratorAction.ConfigNodePxe, hd_fields.OrchestratorAction.SetNodeBoot, hd_fields.OrchestratorAction.PowerOffNode, diff --git a/helm_drydock/drivers/oob/pyghmi_driver/__init__.py b/helm_drydock/drivers/oob/pyghmi_driver/__init__.py index 9a57efe9..0da3af70 100644 --- a/helm_drydock/drivers/oob/pyghmi_driver/__init__.py +++ b/helm_drydock/drivers/oob/pyghmi_driver/__init__.py @@ -34,7 +34,7 @@ class PyghmiDriver(oob.OobDriver): self.driver_key = "pyghmi_driver" self.driver_desc = "Pyghmi OOB Driver" - self.config = config.DrydockConfig.node_driver[self.driver_key] + self.config = config.DrydockConfig.node_driver.get(self.driver_key, {}) def execute_task(self, task_id): task = self.state_manager.get_task(task_id) diff --git a/helm_drydock/drydock.py b/helm_drydock/drydock.py new file mode 100644 index 00000000..4be2340e --- /dev/null +++ b/helm_drydock/drydock.py @@ -0,0 +1,55 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +import helm_drydock.config as config +import helm_drydock.objects as objects +import helm_drydock.ingester as ingester +import helm_drydock.statemgmt as statemgmt +import helm_drydock.orchestrator as orch +import helm_drydock.control.api as api + +def start_drydock(): + objects.register_all() + + # Setup root logger + logger = logging.getLogger('drydock') + + logger.setLevel(config.DrydockConfig.global_config.get('log_level')) + ch = logging.StreamHandler() + formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') + ch.setFormatter(formatter) + logger.addHandler(ch) + + # Specalized format for API logging + logger = logging.getLogger('drydock.control') + logger.propagate = False + formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(user)s - %(req_id)s - %(external_ctx)s - %(message)s') + + ch = logging.StreamHandler() + ch.setFormatter(formatter) + logger.addHandler(ch) + + state = statemgmt.DesignState() + + orchestrator = orch.Orchestrator(config.DrydockConfig.orchestrator_config.get('drivers', {}), + state_manager=state) + input_ingester = ingester.Ingester() + input_ingester.enable_plugins(config.DrydockConfig.ingester_config.get('plugins', [])) + + return api.start_api(state_manager=state, ingester=input_ingester, + orchestrator=orchestrator) + +drydock = start_drydock() + diff --git a/helm_drydock/error.py b/helm_drydock/error.py index a8988f97..04a38196 100644 --- a/helm_drydock/error.py +++ b/helm_drydock/error.py @@ -34,4 +34,10 @@ class TransientDriverError(DriverError): pass class PersistentDriverError(DriverError): + pass + +class ApiError(Exception): + pass + +class InvalidFormat(ApiError): pass \ No newline at end of file diff --git a/helm_drydock/ingester/__init__.py b/helm_drydock/ingester/__init__.py index b2b69072..07862605 100644 --- a/helm_drydock/ingester/__init__.py +++ b/helm_drydock/ingester/__init__.py @@ -18,6 +18,7 @@ import logging import yaml import uuid +import importlib import helm_drydock.objects as objects import helm_drydock.objects.site as site @@ -31,55 +32,62 @@ from helm_drydock.statemgmt import DesignState class Ingester(object): def __init__(self): - logging.basicConfig(format="%(asctime)-15s [%(levelname)] %(module)s %(process)d %(message)s") - self.log = logging.Logger("ingester") + self.logger = logging.getLogger("drydock.ingester") self.registered_plugins = {} def enable_plugins(self, plugins=[]): + """ + enable_plugins + + :params plugins: - A list of strings naming class objects denoting the ingester plugins to be enabled + + Enable plugins that can be used for ingest_data calls. Each plugin should use + helm_drydock.ingester.plugins.IngesterPlugin as its base class. As long as one + enabled plugin successfully initializes, the call is considered successful. Otherwise + it will throw an exception + """ if len(plugins) == 0: self.log.error("Cannot have an empty plugin list.") for plugin in plugins: try: - new_plugin = plugin() + (module, x, classname) = plugin.rpartition('.') + + if module == '': + raise Exception() + mod = importlib.import_module(module) + klass = getattr(mod, classname) + new_plugin = klass() plugin_name = new_plugin.get_name() self.registered_plugins[plugin_name] = new_plugin - except: - self.log.error("Could not enable plugin %s" % (plugin.__name__)) + except Exception as ex: + self.logger.error("Could not enable plugin %s - %s" % (plugin, str(ex))) if len(self.registered_plugins) == 0: - self.log.error("Could not enable at least one plugin") + self.logger.error("Could not enable at least one plugin") raise Exception("Could not enable at least one plugin") - """ - enable_plugins - params: plugins - A list of class objects denoting the ingester plugins to be enabled - - Enable plugins that can be used for ingest_data calls. Each plugin should use - helm_drydock.ingester.plugins.IngesterPlugin as its base class. As long as one - enabled plugin successfully initializes, the call is considered successful. Otherwise - it will throw an exception - """ - def ingest_data(self, plugin_name='', design_state=None, **kwargs): + def ingest_data(self, plugin_name='', design_state=None, design_id=None, context=None, **kwargs): if design_state is None: - self.log.error("ingest_data called without valid DesignState handler") - raise Exception("Invalid design_state handler") - - design_data = None + self.logger.error("Ingester:ingest_data called without valid DesignState handler") + raise ValueError("Invalid design_state handler") # If no design_id is specified, instantiate a new one - if 'design_id' not in kwargs.keys(): - design_id = str(uuid.uuid4()) - design_data = objects.SiteDesign(id=design_id) - design_state.post_design(design_data) - else: - design_id = kwargs.get('design_id') - design_data = design_state.get_design(design_id) + if 'design_id' is None: + self.logger.error("Ingester:ingest_data required kwarg 'design_id' missing") + raise ValueError("Ingester:ingest_data required kwarg 'design_id' missing") + + design_data = design_state.get_design(design_id) + + self.logger.debug("Ingester:ingest_data ingesting design parts for design %s" % design_id) if plugin_name in self.registered_plugins: design_items = self.registered_plugins[plugin_name].ingest_data(**kwargs) + self.logger.debug("Ingester:ingest_data parsed %s design parts" % str(len(design_items))) for m in design_items: + if context is not None: + m.set_create_fields(context) if type(m) is site.Site: design_data.set_site(m) elif type(m) is network.Network: @@ -93,8 +101,9 @@ class Ingester(object): elif type(m) is node.BaremetalNode: design_data.add_baremetal_node(m) design_state.put_design(design_data) + return design_items else: - self.log.error("Could not find plugin %s to ingest data." % (plugin_name)) + self.logger.error("Could not find plugin %s to ingest data." % (plugin_name)) raise LookupError("Could not find plugin %s" % plugin_name) """ ingest_data diff --git a/helm_drydock/ingester/plugins/yaml.py b/helm_drydock/ingester/plugins/yaml.py index ce531f8c..26160481 100644 --- a/helm_drydock/ingester/plugins/yaml.py +++ b/helm_drydock/ingester/plugins/yaml.py @@ -28,6 +28,7 @@ class YamlIngester(IngesterPlugin): def __init__(self): super(YamlIngester, self).__init__() + self.logger = logging.getLogger('drydock.ingester.yaml') def get_name(self): return "yaml" @@ -52,12 +53,10 @@ class YamlIngester(IngesterPlugin): file.close() models.extend(self.parse_docs(contents)) except OSError as err: - self.log.error( + self.logger.error( "Error opening input file %s for ingestion: %s" % (filename, err)) continue - - elif 'content' in kwargs: models.extend(self.parse_docs(kwargs.get('content'))) else: diff --git a/helm_drydock/objects/base.py b/helm_drydock/objects/base.py index d22b2183..5164a8ca 100644 --- a/helm_drydock/objects/base.py +++ b/helm_drydock/objects/base.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import datetime from oslo_versionedobjects import base from oslo_versionedobjects import fields as obj_fields @@ -38,6 +39,32 @@ class DrydockObject(base.VersionedObject): else: raise ValueError("Unknown field %s" % (attrname)) + def obj_to_simple(self): + """ + Create a simple primitive representation of this object excluding + all the versioning stuff. Used to serialize an object for public + consumption, not intended to be deserialized by OVO + """ + + primitive = dict() + + primitive['model_type'] = self.__class__.__name__ + primitive['model_version'] = self.VERSION + + for name, field in self.fields.items(): + if self.obj_attr_is_set(name): + value = getattr(self, name) + if (hasattr(value, 'obj_to_simple') and + callable(value.obj_to_simple)): + primitive[name] = value.obj_to_simple() + else: + value = field.to_primitive(self, name, value) + if value is not None: + primitive[name] = value + + return primitive + + class DrydockPersistentObject(base.VersionedObject): fields = { @@ -47,6 +74,15 @@ class DrydockPersistentObject(base.VersionedObject): 'updated_by': obj_fields.StringField(nullable=True), } + def set_create_fields(self, context): + self.created_at = datetime.datetime.now() + self.created_by = context.user + + def set_update_fields(self, context): + self.updated_at = datetime.datetime.now() + self.updated_by = context.user + + class DrydockObjectListBase(base.ObjectListBase): def __init__(self, **kwargs): @@ -73,3 +109,11 @@ class DrydockObjectListBase(base.ObjectListBase): model_list.append(o) return model_list + + def obj_to_simple(self): + primitive_list = list() + + for o in self.objects: + primitive_list.append(o.obj_to_simple()) + + return primitive_list diff --git a/helm_drydock/objects/site.py b/helm_drydock/objects/site.py index f786d911..afa853b7 100644 --- a/helm_drydock/objects/site.py +++ b/helm_drydock/objects/site.py @@ -16,6 +16,7 @@ # from copy import deepcopy import uuid +import datetime import oslo_versionedobjects.fields as ovo_fields @@ -126,8 +127,6 @@ class SiteDesign(base.DrydockPersistentObject, base.DrydockObject): def __init__(self, **kwargs): super(SiteDesign, self).__init__(**kwargs) - - # Assign UUID id def assign_id(self): self.id = uuid.uuid4() @@ -228,6 +227,18 @@ class SiteDesign(base.DrydockPersistentObject, base.DrydockObject): raise DesignError("BaremetalNode %s not found in design state" % node_key) + def create(self, ctx, state_manager): + self.created_at = datetime.datetime.now() + self.created_by = ctx.user + + state_manager.post_design(self) + + def save(self, ctx, state_manager): + self.updated_at = datetime.datetime.now() + self.updated_by = ctx.user + + state_manager.put_design(self) + """ Support filtering on rack name, node name or node tag for now. Each filter can be a comma-delimited list of diff --git a/helm_drydock/objects/task.py b/helm_drydock/objects/task.py index 9985b285..0405cd32 100644 --- a/helm_drydock/objects/task.py +++ b/helm_drydock/objects/task.py @@ -66,18 +66,29 @@ class Task(object): def get_subtasks(self): return self.subtasks + def to_dict(self): + return { + 'task_id': str(self.task_id), + 'action': self.action, + 'parent_task': str(self.parent_task_id), + 'status': self.status, + 'result': self.result, + 'result_detail': self.result_detail, + 'subtasks': [str(x) for x in self.subtasks], + } + class OrchestratorTask(Task): - def __init__(self, **kwargs): + def __init__(self, site=None, design_id=None, **kwargs): super(OrchestratorTask, self).__init__(**kwargs) # Validate parameters based on action - self.site = kwargs.get('site', '') + self.site = site - if self.site == '': + if self.site is None: raise ValueError("Orchestration Task requires 'site' parameter") - self.design_id = kwargs.get('design_id', 0) + self.design_id = design_id if self.action in [hd_fields.OrchestratorAction.VerifyNode, hd_fields.OrchestratorAction.PrepareNode, @@ -85,6 +96,14 @@ class OrchestratorTask(Task): hd_fields.OrchestratorAction.DestroyNode]: self.node_filter = kwargs.get('node_filter', None) + def to_dict(self): + _dict = super(OrchestratorTask, self).to_dict() + + _dict['site'] = self.site + _dict['design_id'] = self.design_id + _dict['node_filter'] = getattr(self, 'node_filter', None) + + return _dict class DriverTask(Task): def __init__(self, task_scope={}, **kwargs): @@ -94,4 +113,13 @@ class DriverTask(Task): self.site_name = task_scope.get('site', None) - self.node_list = task_scope.get('node_names', []) \ No newline at end of file + self.node_list = task_scope.get('node_names', []) + + def to_dict(self): + _dict = super(DriverTask, self).to_dict() + + _dict['site_name'] = self.site_name + _dict['design_id'] = self.design_id + _dict['node_list'] = self.node_list + + return _dict \ No newline at end of file diff --git a/helm_drydock/orchestrator/__init__.py b/helm_drydock/orchestrator/__init__.py index 2b589156..fba338ba 100644 --- a/helm_drydock/orchestrator/__init__.py +++ b/helm_drydock/orchestrator/__init__.py @@ -1,3 +1,4 @@ + # Copyright 2017 AT&T Intellectual Property. All other rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -106,8 +107,7 @@ class Orchestrator(object): self.task_field_update(task_id, status=hd_fields.TaskStatus.Running) try: - site_design = self.get_effective_site(task_site, - change_id=design_id) + site_design = self.get_effective_site(design_id) self.task_field_update(task_id, result=hd_fields.ActionResult.Success) except: @@ -175,7 +175,7 @@ class Orchestrator(object): result=hd_fields.ActionResult.Failure) return - site_design = self.get_effective_site(design_id, task_site) + site_design = self.get_effective_site(design_id) node_filter = task.node_filter @@ -212,8 +212,7 @@ class Orchestrator(object): result=hd_fields.ActionResult.Failure) return - site_design = self.get_effective_site(task_site, - change_id=design_id) + site_design = self.get_effective_site(design_id) node_filter = task.node_filter @@ -331,7 +330,7 @@ class Orchestrator(object): # the baremetal nodes which recursively resolves it for host profiles # assigned to those nodes - for n in site_design.baremetal_nodes: + for n in getattr(site_design, 'baremetal_nodes', []): n.compile_applied_model(site_design) return @@ -342,18 +341,13 @@ class Orchestrator(object): return a Site model reflecting the effective design for the site """ - def get_described_site(self, design_id, site_name): - site_design = None - - if site_name is None: - raise errors.OrchestratorError("Cannot source design for site None") - + def get_described_site(self, design_id): site_design = self.state_manager.get_design(design_id) return site_design - def get_effective_site(self, design_id, site_name): - site_design = self.get_described_site(design_id, site_name) + def get_effective_site(self, design_id): + site_design = self.get_described_site(design_id) self.compute_model_inheritance(site_design) diff --git a/helm_drydock/statemgmt/__init__.py b/helm_drydock/statemgmt/__init__.py index 22112c29..8936649e 100644 --- a/helm_drydock/statemgmt/__init__.py +++ b/helm_drydock/statemgmt/__init__.py @@ -41,6 +41,7 @@ class DesignState(object): # has started def get_design(self, design_id): if design_id not in self.designs.keys(): + raise DesignError("Design ID %s not found" % (design_id)) return objects.SiteDesign.obj_from_primitive(self.designs[design_id]) @@ -133,7 +134,7 @@ class DesignState(object): def get_task(self, task_id): for t in self.tasks: - if t.get_id() == task_id: + if t.get_id() == task_id or str(t.get_id()) == task_id: return deepcopy(t) return None diff --git a/setup.py b/setup.py index 01bbff5f..4436a8f1 100644 --- a/setup.py +++ b/setup.py @@ -51,7 +51,8 @@ setup(name='helm_drydock', 'helm_drydock.drivers.oob.pyghmi_driver', 'helm_drydock.drivers.node', 'helm_drydock.drivers.node.maasdriver', - 'helm_drydock.drivers.node.maasdriver.models'], + 'helm_drydock.drivers.node.maasdriver.models', + 'helm_drydock.control'], install_requires=[ 'PyYAML', 'pyghmi>=1.0.18', @@ -60,6 +61,7 @@ setup(name='helm_drydock', 'oslo.versionedobjects>=1.23.0', 'requests', 'oauthlib', + 'uwsgi>1.4', ] )