From 80b3a1e99d8cc72c509f46fa8822658135754a26 Mon Sep 17 00:00:00 2001 From: Scott Hussey Date: Thu, 15 Jun 2017 22:42:53 -0500 Subject: [PATCH] Orchestration of MaaS enlistment (#42) * WIP - Initial API implementation with Falcon * API service for /designs and /tasks endpoints REST API using falcon library Middleware for authentication (stubbed until Keystone is avail) Middleware for context and logging Request logging and initial error logging README updates * Cleanup readme formatting * Rename helm_drydock to drydock_provisioner * Fixed missing except statement * Fixed missing except statement * Reorganize API paths in a list Fix duplication of context init Add API version prefix to URLs * DRYD-2 MVP - phase 1 - node enlistment Add node driver task for IdentifyNode Implement MaaS API interface for Machines and Interfaces --- README.md | 9 +- .../__init__.py | 0 .../config.py | 17 +- .../control}/__init__.py | 0 drydock_provisioner/control/api.py | 51 ++ drydock_provisioner/control/base.py | 144 ++++++ drydock_provisioner/control/designs.py | 164 +++++++ drydock_provisioner/control/middleware.py | 92 ++++ drydock_provisioner/control/readme.md | 30 ++ drydock_provisioner/control/tasks.py | 79 ++++ .../drivers/__init__.py | 8 +- .../drivers/node/__init__.py | 7 +- .../drivers/node/maasdriver}/__init__.py | 2 +- .../drivers/node/maasdriver/api_client.py | 0 .../drivers/node/maasdriver/driver.py | 447 ++++++++++++++++++ .../node/maasdriver/models}/__init__.py | 0 .../drivers/node/maasdriver/models/base.py | 43 +- .../drivers/node/maasdriver/models/fabric.py | 4 +- .../node/maasdriver/models/interface.py | 34 ++ .../drivers/node/maasdriver/models/machine.py | 185 ++++++++ .../drivers/node/maasdriver/models/subnet.py | 2 +- .../drivers/node/maasdriver/models/vlan.py | 4 +- .../drivers/node/maasdriver/readme.md | 0 .../drivers/oob/__init__.py | 8 +- .../drivers/oob/pyghmi_driver/__init__.py | 36 +- .../drivers/readme.md | 1 + drydock_provisioner/drydock.py | 55 +++ .../error.py | 6 + drydock_provisioner/ingester/__init__.py | 120 +++++ .../ingester/plugins/__init__.py | 0 .../ingester/plugins/yaml.py | 15 +- .../ingester/readme.md | 2 +- .../introspection/readme.md | 0 .../objects/__init__.py | 12 +- .../objects/base.py | 50 +- .../objects/fields.py | 1 + .../objects/hostprofile.py | 6 +- .../objects/hwprofile.py | 6 +- .../objects/network.py | 8 +- .../objects/node.py | 12 +- .../objects/readme.md | 0 .../objects/site.py | 23 +- .../objects/task.py | 42 +- .../orchestrator/__init__.py | 157 ++++-- .../orchestrator/readme.md | 0 .../statemgmt/__init__.py | 9 +- .../statemgmt/readme.md | 0 examples/designparts_v1.0.yaml | 4 +- helm_drydock/control/readme.md | 14 - .../drivers/node/maasdriver/driver.py | 306 ------------ helm_drydock/ingester/__init__.py | 107 ----- setup.py | 33 +- tests/integration/test_maasdriver_client.py | 4 +- tests/integration/test_maasdriver_network.py | 8 +- tests/integration/test_orch_node_networks.py | 26 +- tests/unit/test_design_inheritance.py | 12 +- tests/unit/test_ingester.py | 12 +- tests/unit/test_ingester_yaml.py | 2 +- tests/unit/test_models.py | 12 +- tests/unit/test_orch_generic.py | 10 +- tests/unit/test_orch_oob.py | 20 +- tests/unit/test_statemgmt.py | 4 +- 62 files changed, 1822 insertions(+), 643 deletions(-) rename {helm_drydock => drydock_provisioner}/__init__.py (100%) rename {helm_drydock => drydock_provisioner}/config.py (59%) rename {helm_drydock/drivers/node/maasdriver => drydock_provisioner/control}/__init__.py (100%) create mode 100644 drydock_provisioner/control/api.py create mode 100644 drydock_provisioner/control/base.py create mode 100644 drydock_provisioner/control/designs.py create mode 100644 drydock_provisioner/control/middleware.py create mode 100644 drydock_provisioner/control/readme.md create mode 100644 drydock_provisioner/control/tasks.py rename {helm_drydock => drydock_provisioner}/drivers/__init__.py (94%) rename {helm_drydock => drydock_provisioner}/drivers/node/__init__.py (90%) rename {helm_drydock/drivers/node/maasdriver/models => drydock_provisioner/drivers/node/maasdriver}/__init__.py (94%) rename {helm_drydock => drydock_provisioner}/drivers/node/maasdriver/api_client.py (100%) create mode 100644 drydock_provisioner/drivers/node/maasdriver/driver.py rename {helm_drydock/control => drydock_provisioner/drivers/node/maasdriver/models}/__init__.py (100%) rename {helm_drydock => drydock_provisioner}/drivers/node/maasdriver/models/base.py (87%) rename {helm_drydock => drydock_provisioner}/drivers/node/maasdriver/models/fabric.py (88%) create mode 100644 drydock_provisioner/drivers/node/maasdriver/models/interface.py create mode 100644 drydock_provisioner/drivers/node/maasdriver/models/machine.py rename {helm_drydock => drydock_provisioner}/drivers/node/maasdriver/models/subnet.py (96%) rename {helm_drydock => drydock_provisioner}/drivers/node/maasdriver/models/vlan.py (95%) rename {helm_drydock => drydock_provisioner}/drivers/node/maasdriver/readme.md (100%) rename {helm_drydock => drydock_provisioner}/drivers/oob/__init__.py (87%) rename {helm_drydock => drydock_provisioner}/drivers/oob/pyghmi_driver/__init__.py (92%) rename {helm_drydock => drydock_provisioner}/drivers/readme.md (93%) create mode 100644 drydock_provisioner/drydock.py rename {helm_drydock => drydock_provisioner}/error.py (92%) create mode 100644 drydock_provisioner/ingester/__init__.py rename {helm_drydock => drydock_provisioner}/ingester/plugins/__init__.py (100%) rename {helm_drydock => drydock_provisioner}/ingester/plugins/yaml.py (97%) rename {helm_drydock => drydock_provisioner}/ingester/readme.md (84%) rename {helm_drydock => drydock_provisioner}/introspection/readme.md (100%) rename {helm_drydock => drydock_provisioner}/objects/__init__.py (93%) rename {helm_drydock => drydock_provisioner}/objects/base.py (59%) rename {helm_drydock => drydock_provisioner}/objects/fields.py (99%) rename {helm_drydock => drydock_provisioner}/objects/hostprofile.py (98%) rename {helm_drydock => drydock_provisioner}/objects/hwprofile.py (96%) rename {helm_drydock => drydock_provisioner}/objects/network.py (94%) rename {helm_drydock => drydock_provisioner}/objects/node.py (92%) rename {helm_drydock => drydock_provisioner}/objects/readme.md (100%) rename {helm_drydock => drydock_provisioner}/objects/site.py (93%) rename {helm_drydock => drydock_provisioner}/objects/task.py (70%) rename {helm_drydock => drydock_provisioner}/orchestrator/__init__.py (71%) rename {helm_drydock => drydock_provisioner}/orchestrator/readme.md (100%) rename {helm_drydock => drydock_provisioner}/statemgmt/__init__.py (97%) rename {helm_drydock => drydock_provisioner}/statemgmt/readme.md (100%) delete mode 100644 helm_drydock/control/readme.md delete mode 100644 helm_drydock/drivers/node/maasdriver/driver.py delete mode 100644 helm_drydock/ingester/__init__.py diff --git a/README.md b/README.md index dc619477..8e2f0347 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,13 @@ -# helm_drydock +# drydock_provisioner A python REST orchestrator to translate a YAML host topology to a provisioned set of hosts and provide a set of cloud-init post-provisioning instructions. +To run: + + $ virtualenv -p python3 /var/tmp/drydock + $ . /var/tmp/drydock/bin/activate + $ python setup.py install + $ uwsgi --http :9000 -w drydock_provisioner.drydock --callable drydock --enable-threads -L + ## Modular service ### Design Consumer ### diff --git a/helm_drydock/__init__.py b/drydock_provisioner/__init__.py similarity index 100% rename from helm_drydock/__init__.py rename to drydock_provisioner/__init__.py diff --git a/helm_drydock/config.py b/drydock_provisioner/config.py similarity index 59% rename from helm_drydock/config.py rename to drydock_provisioner/config.py index 244e85c0..f64e91e9 100644 --- a/helm_drydock/config.py +++ b/drydock_provisioner/config.py @@ -21,13 +21,24 @@ class DrydockConfig(object): + global_config = { + 'log_level': 'DEBUG', + } + node_driver = { 'maasdriver': { - 'api_key': 'KTMHgA42cNSMnfmJ82:cdg4yQUhp542aHsCTV:7Dc2KB9hQpWq3LfQAAAKAj6wdg22yWxZ', - 'api_url': 'http://localhost:5240/MAAS/api/2.0/' + 'api_key': 'UTBfxGL69XWjaffQek:NuKZSYGuBs6ZpYC6B9:byvXBgY8CsW5VQKxGdQjvJXtjXwr5G4U', + 'api_url': 'http://10.23.19.16:30773/MAAS/api/2.0/', }, } ingester_config = { - 'plugins': ['helm_drydock.ingester.plugins.yaml'] + 'plugins': ['drydock_provisioner.ingester.plugins.yaml.YamlIngester'], + } + + orchestrator_config = { + 'drivers': { + 'oob': 'drydock_provisioner.drivers.oob.pyghmi_driver.PyghmiDriver', + 'node': 'drydock_provisioner.drivers.node.maasdriver.driver.MaasNodeDriver', + } } \ No newline at end of file diff --git a/helm_drydock/drivers/node/maasdriver/__init__.py b/drydock_provisioner/control/__init__.py similarity index 100% rename from helm_drydock/drivers/node/maasdriver/__init__.py rename to drydock_provisioner/control/__init__.py diff --git a/drydock_provisioner/control/api.py b/drydock_provisioner/control/api.py new file mode 100644 index 00000000..a6d832f9 --- /dev/null +++ b/drydock_provisioner/control/api.py @@ -0,0 +1,51 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import falcon + +from .designs import * +from .tasks import * + +from .base import DrydockRequest +from .middleware import AuthMiddleware, ContextMiddleware, LoggingMiddleware + +def start_api(state_manager=None, ingester=None, orchestrator=None): + """ + Start the Drydock API service + + :param state_manager: Instance of drydock_provisioner.statemgmt.manager.DesignState for accessing + state persistence + :param ingester: Instance of drydock_provisioner.ingester.ingester.Ingester for handling design + part input + """ + control_api = falcon.API(request_type=DrydockRequest, + middleware=[AuthMiddleware(), ContextMiddleware(), LoggingMiddleware()]) + + # v1.0 of Drydock API + v1_0_routes = [ + # API for managing orchestrator tasks + ('/tasks', TasksResource(state_manager=state_manager, orchestrator=orchestrator)), + ('/tasks/{task_id}', TaskResource(state_manager=state_manager)), + + # API for managing site design data + ('/designs', DesignsResource(state_manager=state_manager)), + ('/designs/{design_id}', DesignResource(state_manager=state_manager, orchestrator=orchestrator)), + ('/designs/{design_id}/parts', DesignsPartsResource(state_manager=state_manager, ingester=ingester)), + ('/designs/{design_id}/parts/{kind}', DesignsPartsKindsResource(state_manager=state_manager)), + ('/designs/{design_id}/parts/{kind}/{name}', DesignsPartResource(state_manager=state_manager, orchestrator=orchestrator)) + ] + + for path, res in v1_0_routes: + control_api.add_route('/api/v1.0' + path, res) + + return control_api diff --git a/drydock_provisioner/control/base.py b/drydock_provisioner/control/base.py new file mode 100644 index 00000000..3f0f8d88 --- /dev/null +++ b/drydock_provisioner/control/base.py @@ -0,0 +1,144 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import falcon.request as request +import uuid +import json +import logging + +import drydock_provisioner.error as errors + +class BaseResource(object): + + def __init__(self): + self.logger = logging.getLogger('control') + self.authorized_roles = [] + + def on_options(self, req, resp): + self_attrs = dir(self) + methods = ['GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'PATCH'] + allowed_methods = [] + + for m in methods: + if 'on_' + m.lower() in self_attrs: + allowed_methods.append(m) + + resp.headers['Allow'] = ','.join(allowed_methods) + resp.status = falcon.HTTP_200 + + # For authorizing access at the Resource level. A Resource requiring + # finer grained authorization at the method or instance level must + # implement that in the request handlers + def authorize_roles(self, role_list): + authorized = set(self.authorized_roles) + applied = set(role_list) + + if authorized.isdisjoint(applied): + return False + else: + return True + + def req_json(self, req): + if req.content_length is None or req.content_length == 0: + return None + + if req.content_type is not None and req.content_type.lower() == 'application/json': + raw_body = req.stream.read(req.content_length or 0) + + if raw_body is None: + return None + + try: + json_body = json.loads(raw_body.decode('utf-8')) + return json_body + except json.JSONDecodeError as jex: + raise errors.InvalidFormat("%s: Invalid JSON in body: %s" % (req.path, jex)) + else: + raise errors.InvalidFormat("Requires application/json payload") + + def return_error(self, resp, status_code, message="", retry=False): + resp.body = json.dumps({'type': 'error', 'message': message, 'retry': retry}) + resp.status = status_code + + def log_error(self, ctx, level, msg): + extra = { + 'user': 'N/A', + 'req_id': 'N/A', + 'external_ctx': 'N/A' + } + + if ctx is not None: + extra = { + 'user': ctx.user, + 'req_id': ctx.request_id, + 'external_ctx': ctx.external_marker, + } + + self.logger.log(level, msg, extra=extra) + + def debug(self, ctx, msg): + self.log_error(ctx, logging.DEBUG, msg) + + def info(self, ctx, msg): + self.log_error(ctx, logging.INFO, msg) + + def warn(self, ctx, msg): + self.log_error(ctx, logging.WARN, msg) + + def error(self, ctx, msg): + self.log_error(ctx, logging.ERROR, msg) + + +class StatefulResource(BaseResource): + + def __init__(self, state_manager=None): + super(StatefulResource, self).__init__() + + if state_manager is None: + self.error(None, "StatefulResource:init - StatefulResources require a state manager be set") + raise ValueError("StatefulResources require a state manager be set") + + self.state_manager = state_manager + + +class DrydockRequestContext(object): + + def __init__(self): + self.log_level = 'ERROR' + self.user = None + self.roles = ['anyone'] + self.req_id = str(uuid.uuid4()) + self.external_marker = None + + def set_log_level(self, level): + if level in ['error', 'info', 'debug']: + self.log_level = level + + def set_user(self, user): + self.user = user + + def add_role(self, role): + self.roles.append(role) + + def add_roles(self, roles): + self.roles.extend(roles) + + def remove_role(self, role): + self.roles = [x for x in self.roles + if x != role] + + def set_external_marker(self, marker): + self.external_marker = str(marker)[:20] + +class DrydockRequest(request.Request): + context_type = DrydockRequestContext \ No newline at end of file diff --git a/drydock_provisioner/control/designs.py b/drydock_provisioner/control/designs.py new file mode 100644 index 00000000..23afc457 --- /dev/null +++ b/drydock_provisioner/control/designs.py @@ -0,0 +1,164 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import falcon +import json +import uuid +import logging + +import drydock_provisioner.objects as hd_objects +import drydock_provisioner.error as errors + +from .base import StatefulResource + +class DesignsResource(StatefulResource): + + def __init__(self, **kwargs): + super(DesignsResource, self).__init__(**kwargs) + self.authorized_roles = ['user'] + + def on_get(self, req, resp): + state = self.state_manager + + designs = list(state.designs.keys()) + + resp.body = json.dumps(designs) + resp.status = falcon.HTTP_200 + + def on_post(self, req, resp): + try: + json_data = self.req_json(req) + design = None + if json_data is not None: + base_design = json_data.get('base_design_id', None) + + if base_design is not None: + base_design = uuid.UUID(base_design) + design = hd_objects.SiteDesign(base_design_id=base_design_uuid) + else: + design = hd_objects.SiteDesign() + design.assign_id() + design.create(req.context, self.state_manager) + + resp.body = json.dumps(design.obj_to_simple()) + resp.status = falcon.HTTP_201 + except errors.StateError as stex: + self.error(req.context, "Error updating persistence") + self.return_error(resp, falcon.HTTP_500, message="Error updating persistence", retry=True) + except errors.InvalidFormat as fex: + self.error(req.context, str(fex)) + self.return_error(resp, falcon.HTTP_400, message=str(fex), retry=False) + + +class DesignResource(StatefulResource): + + def __init__(self, orchestrator=None, **kwargs): + super(DesignResource, self).__init__(**kwargs) + self.authorized_roles = ['user'] + self.orchestrator = orchestrator + + def on_get(self, req, resp, design_id): + source = req.params.get('source', 'designed') + + try: + design = None + if source == 'compiled': + design = self.orchestrator.get_effective_site(design_id) + elif source == 'designed': + design = self.orchestrator.get_described_site(design_id) + + resp.body = json.dumps(design.obj_to_simple()) + except errors.DesignError: + self.error(req.context, "Design %s not found" % design_id) + self.return_error(resp, falcon.HTTP_404, message="Design %s not found" % design_id, retry=False) + +class DesignsPartsResource(StatefulResource): + + def __init__(self, ingester=None, **kwargs): + super(DesignsPartsResource, self).__init__(**kwargs) + self.ingester = ingester + self.authorized_roles = ['user'] + + if ingester is None: + self.error(None, "DesignsPartsResource requires a configured Ingester instance") + raise ValueError("DesignsPartsResource requires a configured Ingester instance") + + def on_post(self, req, resp, design_id): + ingester_name = req.params.get('ingester', None) + + if ingester_name is None: + self.error(None, "DesignsPartsResource POST requires parameter 'ingester'") + self.return_error(resp, falcon.HTTP_400, message="POST requires parameter 'ingester'", retry=False) + else: + try: + raw_body = req.stream.read(req.content_length or 0) + if raw_body is not None and len(raw_body) > 0: + parsed_items = self.ingester.ingest_data(plugin_name=ingester_name, design_state=self.state_manager, + content=raw_body, design_id=design_id, context=req.context) + resp.status = falcon.HTTP_201 + resp.body = json.dumps([x.obj_to_simple() for x in parsed_items]) + else: + self.return_error(resp, falcon.HTTP_400, message="Empty body not supported", retry=False) + except ValueError: + self.return_error(resp, falcon.HTTP_500, message="Error processing input", retry=False) + except LookupError: + self.return_error(resp, falcon.HTTP_400, message="Ingester %s not registered" % ingester_name, retry=False) + + +class DesignsPartsKindsResource(StatefulResource): + def __init__(self, **kwargs): + super(DesignsPartsKindsResource, self).__init__(**kwargs) + self.authorized_roles = ['user'] + + def on_get(self, req, resp, design_id, kind): + pass + +class DesignsPartResource(StatefulResource): + + def __init__(self, orchestrator=None, **kwargs): + super(DesignsPartResource, self).__init__(**kwargs) + self.authorized_roles = ['user'] + self.orchestrator = orchestrator + + def on_get(self, req , resp, design_id, kind, name): + source = req.params.get('source', 'designed') + + try: + design = None + if source == 'compiled': + design = self.orchestrator.get_effective_site(design_id) + elif source == 'designed': + design = self.orchestrator.get_described_site(design_id) + + part = None + if kind == 'Site': + part = design.get_site() + elif kind == 'Network': + part = design.get_network(name) + elif kind == 'NetworkLink': + part = design.get_network_link(name) + elif kind == 'HardwareProfile': + part = design.get_hardware_profile(name) + elif kind == 'HostProfile': + part = design.get_host_profile(name) + elif kind == 'BaremetalNode': + part = design.get_baremetal_node(name) + else: + self.error(req.context, "Kind %s unknown" % kind) + self.return_error(resp, falcon.HTTP_404, message="Kind %s unknown" % kind, retry=False) + return + + resp.body = json.dumps(part.obj_to_simple()) + except errors.DesignError as dex: + self.error(req.context, str(dex)) + self.return_error(resp, falcon.HTTP_404, message=str(dex), retry=False) \ No newline at end of file diff --git a/drydock_provisioner/control/middleware.py b/drydock_provisioner/control/middleware.py new file mode 100644 index 00000000..a522af7b --- /dev/null +++ b/drydock_provisioner/control/middleware.py @@ -0,0 +1,92 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import falcon +import logging +import uuid + +import drydock_provisioner.config as config + +class AuthMiddleware(object): + + # Authentication + def process_request(self, req, resp): + ctx = req.context + token = req.get_header('X-Auth-Token') + + user = self.validate_token(token) + + if user is not None: + ctx.set_user(user) + user_roles = self.role_list(user) + ctx.add_roles(user_roles) + else: + ctx.add_role('anyone') + + # Authorization + def process_resource(self, req, resp, resource, params): + ctx = req.context + + if not resource.authorize_roles(ctx.roles): + raise falcon.HTTPUnauthorized('Authentication required', + ('This resource requires an authorized role.')) + + # Return the username associated with an authenticated token or None + def validate_token(self, token): + if token == '42': + return 'scott' + elif token == 'bigboss': + return 'admin' + else: + return None + + # Return the list of roles assigned to the username + # Roles need to be an enum + def role_list(self, username): + if username == 'scott': + return ['user'] + elif username == 'admin': + return ['user', 'admin'] + +class ContextMiddleware(object): + + def process_request(self, req, resp): + ctx = req.context + + requested_logging = req.get_header('X-Log-Level') + + if (config.DrydockConfig.global_config.get('log_level', '') == 'DEBUG' or + (requested_logging == 'DEBUG' and 'admin' in ctx.roles)): + ctx.set_log_level('DEBUG') + elif requested_logging == 'INFO': + ctx.set_log_level('INFO') + + ext_marker = req.get_header('X-Context-Marker') + + ctx.set_external_marker(ext_marker if ext_marker is not None else '') + +class LoggingMiddleware(object): + + def __init__(self): + self.logger = logging.getLogger('drydock.control') + + def process_response(self, req, resp, resource, req_succeeded): + ctx = req.context + extra = { + 'user': ctx.user, + 'req_id': ctx.req_id, + 'external_ctx': ctx.external_marker, + } + resp.append_header('X-Drydock-Req', ctx.req_id) + self.logger.info("%s - %s" % (req.uri, resp.status), extra=extra) diff --git a/drydock_provisioner/control/readme.md b/drydock_provisioner/control/readme.md new file mode 100644 index 00000000..d09f2d0e --- /dev/null +++ b/drydock_provisioner/control/readme.md @@ -0,0 +1,30 @@ +# Control # + +This is the external facing API service to control the rest +of Drydock and query Drydock-managed data. + +## v1.0 Endpoints ## + +### /api/v1.0/tasks ### + +POST - Create a new orchestration task and submit it for execution +GET - Get status of a task +DELETE - Cancel execution of a task if permitted + +### /api/v1.0/designs ### + +POST - Create a new site design so design parts can be added + +### /api/v1.0/designs/{id} + +GET - Get a current design if available. Param 'source=compiled' to calculate the inheritance chain and compile the effective design. + +### /api/v1.0/designs/{id}/parts + +POST - Submit a new design part to be ingested and added to this design +GET - View a currently defined design part +PUT - Replace an existing design part *Not Implemented* + +### /api/v1.0/designs/{id}/parts/{kind}/{name} + +GET - View a single design part. param 'source=compiled' to calculate the inheritance chain and compile the effective configuration for the design part. \ No newline at end of file diff --git a/drydock_provisioner/control/tasks.py b/drydock_provisioner/control/tasks.py new file mode 100644 index 00000000..a4a49da9 --- /dev/null +++ b/drydock_provisioner/control/tasks.py @@ -0,0 +1,79 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import falcon +import json +import threading +import traceback + +import drydock_provisioner.objects.task as obj_task +from .base import StatefulResource + +class TasksResource(StatefulResource): + + def __init__(self, orchestrator=None, **kwargs): + super(TasksResource, self).__init__(**kwargs) + self.authorized_roles = ['user'] + self.orchestrator = orchestrator + + def on_get(self, req, resp): + task_id_list = [str(x.get_id()) for x in self.state_manager.tasks] + resp.body = json.dumps(task_id_list) + + def on_post(self, req, resp): + try: + json_data = self.req_json(req) + + sitename = json_data.get('sitename', None) + design_id = json_data.get('design_id', None) + action = json_data.get('action', None) + + if sitename is None or design_id is None or action is None: + self.info(req.context, "Task creation requires fields sitename, design_id, action") + self.return_error(resp, falcon.HTTP_400, message="Task creation requires fields sitename, design_id, action", retry=False) + return + + task = self.orchestrator.create_task(obj_task.OrchestratorTask, site=sitename, + design_id=design_id, action=action) + + task_thread = threading.Thread(target=self.orchestrator.execute_task, args=[task.get_id()]) + task_thread.start() + + resp.body = json.dumps(task.to_dict()) + resp.status = falcon.HTTP_201 + except Exception as ex: + self.error(req.context, "Unknown error: %s\n%s" % (str(ex), traceback.format_exc())) + self.return_error(resp, falcon.HTTP_500, message="Unknown error", retry=False) + + +class TaskResource(StatefulResource): + + def __init__(self, orchestrator=None, **kwargs): + super(TaskResource, self).__init__(**kwargs) + self.authorized_roles = ['user'] + self.orchestrator = orchestrator + + def on_get(self, req, resp, task_id): + try: + task = self.state_manager.get_task(task_id) + + if task is None: + self.info(req.context, "Task %s does not exist" % task_id ) + self.return_error(resp, falcon.HTTP_404, message="Task %s does not exist" % task_id, retry=False) + return + + resp.body = json.dumps(task.to_dict()) + resp.status = falcon.HTTP_200 + except Exception as ex: + self.error(req.context, "Unknown error: %s" % (str(ex))) + self.return_error(resp, falcon.HTTP_500, message="Unknown error", retry=False) diff --git a/helm_drydock/drivers/__init__.py b/drydock_provisioner/drivers/__init__.py similarity index 94% rename from helm_drydock/drivers/__init__.py rename to drydock_provisioner/drivers/__init__.py index 2ced9f00..c3868af1 100644 --- a/helm_drydock/drivers/__init__.py +++ b/drydock_provisioner/drivers/__init__.py @@ -15,10 +15,10 @@ from threading import Thread, Lock import uuid import time -import helm_drydock.objects.fields as hd_fields -import helm_drydock.statemgmt as statemgmt -import helm_drydock.objects.task as tasks -import helm_drydock.error as errors +import drydock_provisioner.objects.fields as hd_fields +import drydock_provisioner.statemgmt as statemgmt +import drydock_provisioner.objects.task as tasks +import drydock_provisioner.error as errors # This is the interface for the orchestrator to access a driver # TODO Need to have each driver spin up a seperate thread to manage diff --git a/helm_drydock/drivers/node/__init__.py b/drydock_provisioner/drivers/node/__init__.py similarity index 90% rename from helm_drydock/drivers/node/__init__.py rename to drydock_provisioner/drivers/node/__init__.py index 87ea3046..a28c7cec 100644 --- a/helm_drydock/drivers/node/__init__.py +++ b/drydock_provisioner/drivers/node/__init__.py @@ -13,10 +13,10 @@ # limitations under the License. # -import helm_drydock.objects.fields as hd_fields -import helm_drydock.error as errors +import drydock_provisioner.objects.fields as hd_fields +import drydock_provisioner.error as errors -from helm_drydock.drivers import ProviderDriver +from drydock_provisioner.drivers import ProviderDriver class NodeDriver(ProviderDriver): @@ -28,6 +28,7 @@ class NodeDriver(ProviderDriver): hd_fields.OrchestratorAction.CreateStorageTemplate, hd_fields.OrchestratorAction.CreateBootMedia, hd_fields.OrchestratorAction.PrepareHardwareConfig, + hd_fields.OrchestratorAction.IdentifyNode, hd_fields.OrchestratorAction.ConfigureHardware, hd_fields.OrchestratorAction.InterrogateNode, hd_fields.OrchestratorAction.ApplyNodeNetworking, diff --git a/helm_drydock/drivers/node/maasdriver/models/__init__.py b/drydock_provisioner/drivers/node/maasdriver/__init__.py similarity index 94% rename from helm_drydock/drivers/node/maasdriver/models/__init__.py rename to drydock_provisioner/drivers/node/maasdriver/__init__.py index 2a385a45..f10bbbf6 100644 --- a/helm_drydock/drivers/node/maasdriver/models/__init__.py +++ b/drydock_provisioner/drivers/node/maasdriver/__init__.py @@ -10,4 +10,4 @@ # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and -# limitations under the License. \ No newline at end of file +# limitations under the License. diff --git a/helm_drydock/drivers/node/maasdriver/api_client.py b/drydock_provisioner/drivers/node/maasdriver/api_client.py similarity index 100% rename from helm_drydock/drivers/node/maasdriver/api_client.py rename to drydock_provisioner/drivers/node/maasdriver/api_client.py diff --git a/drydock_provisioner/drivers/node/maasdriver/driver.py b/drydock_provisioner/drivers/node/maasdriver/driver.py new file mode 100644 index 00000000..f7079dcc --- /dev/null +++ b/drydock_provisioner/drivers/node/maasdriver/driver.py @@ -0,0 +1,447 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import time +import logging + +import drydock_provisioner.error as errors +import drydock_provisioner.config as config +import drydock_provisioner.drivers as drivers +import drydock_provisioner.objects.fields as hd_fields +import drydock_provisioner.objects.task as task_model + +from drydock_provisioner.drivers.node import NodeDriver +from .api_client import MaasRequestFactory +import drydock_provisioner.drivers.node.maasdriver.models.fabric as maas_fabric +import drydock_provisioner.drivers.node.maasdriver.models.vlan as maas_vlan +import drydock_provisioner.drivers.node.maasdriver.models.subnet as maas_subnet +import drydock_provisioner.drivers.node.maasdriver.models.machine as maas_machine + +class MaasNodeDriver(NodeDriver): + + def __init__(self, **kwargs): + super(MaasNodeDriver, self).__init__(**kwargs) + + self.driver_name = "maasdriver" + self.driver_key = "maasdriver" + self.driver_desc = "MaaS Node Provisioning Driver" + + self.config = config.DrydockConfig.node_driver[self.driver_key] + + self.logger = logging.getLogger('drydock.nodedriver.maasdriver') + + def execute_task(self, task_id): + task = self.state_manager.get_task(task_id) + + if task is None: + raise errors.DriverError("Invalid task %s" % (task_id)) + + if task.action not in self.supported_actions: + raise errors.DriverError("Driver %s doesn't support task action %s" + % (self.driver_desc, task.action)) + + if task.action == hd_fields.OrchestratorAction.ValidateNodeServices: + self.orchestrator.task_field_update(task.get_id(), + status=hd_fields.TaskStatus.Running) + maas_client = MaasRequestFactory(self.config['api_url'], self.config['api_key']) + + try: + if maas_client.test_connectivity(): + if maas_client.test_authentication(): + self.orchestrator.task_field_update(task.get_id(), + status=hd_fields.TaskStatus.Complete, + result=hd_fields.ActionResult.Success) + return + except errors.TransientDriverError(ex): + result = { + 'retry': True, + 'detail': str(ex), + } + self.orchestrator.task_field_update(task.get_id(), + status=hd_fields.TaskStatus.Complete, + result=hd_fields.ActionResult.Failure, + result_details=result) + return + except errors.PersistentDriverError(ex): + result = { + 'retry': False, + 'detail': str(ex), + } + self.orchestrator.task_field_update(task.get_id(), + status=hd_fields.TaskStatus.Complete, + result=hd_fields.ActionResult.Failure, + result_details=result) + return + except Exception(ex): + result = { + 'retry': False, + 'detail': str(ex), + } + self.orchestrator.task_field_update(task.get_id(), + status=hd_fields.TaskStatus.Complete, + result=hd_fields.ActionResult.Failure, + result_details=result) + return + + design_id = getattr(task, 'design_id', None) + + if design_id is None: + raise errors.DriverError("No design ID specified in task %s" % + (task_id)) + + + if task.site_name is None: + raise errors.DriverError("No site specified for task %s." % + (task_id)) + + self.orchestrator.task_field_update(task.get_id(), + status=hd_fields.TaskStatus.Running) + + site_design = self.orchestrator.get_effective_site(design_id) + + if task.action == hd_fields.OrchestratorAction.CreateNetworkTemplate: + self.orchestrator.task_field_update(task.get_id(), status=hd_fields.TaskStatus.Running) + + subtask = self.orchestrator.create_task(task_model.DriverTask, + parent_task_id=task.get_id(), design_id=design_id, + action=task.action, site_name=task.site_name, + task_scope={'site': task.site_name}) + runner = MaasTaskRunner(state_manager=self.state_manager, + orchestrator=self.orchestrator, + task_id=subtask.get_id(),config=self.config) + + self.logger.info("Starting thread for task %s to create network templates" % (subtask.get_id())) + + runner.start() + + # TODO Figure out coherent system for putting all the timeouts in + # the config + runner.join(timeout=120) + + if runner.is_alive(): + result = { + 'retry': False, + 'detail': 'MaaS Network creation timed-out' + } + self.logger.warn("Thread for task %s timed out after 120s" % (subtask.get_id())) + self.orchestrator.task_field_update(task.get_id(), + status=hd_fields.TaskStatus.Complete, + result=hd_fields.ActionResult.Failure, + result_detail=result) + else: + subtask = self.state_manager.get_task(subtask.get_id()) + self.logger.info("Thread for task %s completed - result %s" % (subtask.get_id(), subtask.get_result())) + self.orchestrator.task_field_update(task.get_id(), + status=hd_fields.TaskStatus.Complete, + result=subtask.get_result()) + + return + elif task.action == hd_fields.OrchestratorAction.IdentifyNode: + self.orchestrator.task_field_update(task.get_id(), + status=hd_fields.TaskStatus.Running) + + subtasks = [] + + result_detail = { + 'detail': [] + } + + for n in task.node_list: + subtask = self.orchestrator.create_task(task_model.DriverTask, + parent_task_id=task.get_id(), design_id=design_id, + action=hd_fields.OrchestratorAction.IdentifyNode, + site_name=task.site_name, + task_scope={'site': task.site_name, 'node_names': [n]}) + runner = MaasTaskRunner(state_manager=self.state_manager, + orchestrator=self.orchestrator, + task_id=subtask.get_id(),config=self.config) + + self.logger.info("Starting thread for task %s to identify node %s" % (subtask.get_id(), n)) + + runner.start() + subtasks.append(subtask.get_id()) + + running_subtasks = len(subtasks) + attempts = 0 + worked = failed = False + + #TODO Add timeout to config + while running_subtasks > 0 and attempts < 3: + for t in subtasks: + subtask = self.state_manager.get_task(t) + + if subtask.status == hd_fields.TaskStatus.Complete: + self.logger.info("Task %s to identify node %s complete - status %s" % + (subtask.get_id(), n, subtask.get_result())) + + result_detail['detail'].extend(subtask.result_detail['detail']) + running_subtasks = running_subtasks - 1 + + if subtask.result in [hd_fields.ActionResult.Success, + hd_fields.ActionResult.PartialSuccess]: + worked = True + elif subtask.result in [hd_fields.ActionResult.Failure, + hd_fields.ActionResult.PartialSuccess]: + failed = True + + time.sleep(1 * 60) + attempts = attempts + 1 + + if running_subtasks > 0: + self.logger.warn("Time out for task %s before all subtask threads complete" % (task.get_id())) + result = hd_fields.ActionResult.DependentFailure + result_detail['detail'].append('Some subtasks did not complete before the timeout threshold') + if worked and failed: + result = hd_fields.ActionResult.PartialSuccess + elif worked: + result = hd_fields.ActionResult.Success + else: + result = hd_fields.ActionResult.Failure + + self.orchestrator.task_field_update(task.get_id(), + status=hd_fields.TaskStatus.Complete, + result=result, + result_detail=result_detail) + +class MaasTaskRunner(drivers.DriverTaskRunner): + + def __init__(self, config=None, **kwargs): + super(MaasTaskRunner, self).__init__(**kwargs) + + self.driver_config = config + self.logger = logging.getLogger('drydock.nodedriver.maasdriver') + + def execute_task(self): + task_action = self.task.action + + self.orchestrator.task_field_update(self.task.get_id(), + status=hd_fields.TaskStatus.Running, + result=hd_fields.ActionResult.Incomplete) + + self.maas_client = MaasRequestFactory(self.driver_config['api_url'], + self.driver_config['api_key']) + + site_design = self.orchestrator.get_effective_site(self.task.design_id) + + if task_action == hd_fields.OrchestratorAction.CreateNetworkTemplate: + # Try to true up MaaS definitions of fabrics/vlans/subnets + # with the networks defined in Drydock + design_networks = site_design.networks + + subnets = maas_subnet.Subnets(self.maas_client) + subnets.refresh() + + result_detail = { + 'detail': [] + } + + for n in design_networks: + try: + subnet = subnets.singleton({'cidr': n.cidr}) + + if subnet is not None: + subnet.name = n.name + subnet.dns_servers = n.dns_servers + + vlan_list = maas_vlan.Vlans(self.maas_client, fabric_id=subnet.fabric) + vlan_list.refresh() + + vlan = vlan_list.select(subnet.vlan) + + if vlan is not None: + if ((n.vlan_id is None and vlan.vid != 0) or + (n.vlan_id is not None and vlan.vid != n.vlan_id)): + + # if the VLAN name matches, assume this is the correct resource + # and it needs to be updated + if vlan.name == n.name: + vlan.set_vid(n.vlan_id) + vlan.mtu = n.mtu + vlan.update() + result_detail['detail'].append("VLAN %s found for network %s, updated attributes" + % (vlan.resource_id, n.name)) + else: + # Found a VLAN with the correct VLAN tag, update subnet to use it + target_vlan = vlan_list.singleton({'vid': n.vlan_id if n.vlan_id is not None else 0}) + if target_vlan is not None: + subnet.vlan = target_vlan.resource_id + else: + # This is a flag that after creating a fabric and + # VLAN below, update the subnet + subnet.vlan = None + else: + subnet.vlan = None + + # Check if the routes have a default route + subnet.gateway_ip = n.get_default_gateway() + + + result_detail['detail'].append("Subnet %s found for network %s, updated attributes" + % (subnet.resource_id, n.name)) + + # Need to find or create a Fabric/Vlan for this subnet + if (subnet is None or (subnet is not None and subnet.vlan is None)): + fabric_list = maas_fabric.Fabrics(self.maas_client) + fabric_list.refresh() + fabric = fabric_list.singleton({'name': n.name}) + + vlan = None + + if fabric is not None: + vlan_list = maas_vlan.Vlans(self.maas_client, fabric_id=fabric.resource_id) + vlan_list.refresh() + + vlan = vlan_list.singleton({'vid': n.vlan_id if n.vlan_id is not None else 0}) + + if vlan is not None: + vlan = matching_vlans[0] + + vlan.name = n.name + if getattr(n, 'mtu', None) is not None: + vlan.mtu = n.mtu + + if subnet is not None: + subnet.vlan = vlan.resource_id + subnet.update() + + vlan.update() + result_detail['detail'].append("VLAN %s found for network %s, updated attributes" + % (vlan.resource_id, n.name)) + else: + # Create a new VLAN in this fabric and assign subnet to it + vlan = maas_vlan.Vlan(self.maas_client, name=n.name, vid=vlan_id, + mtu=getattr(n, 'mtu', None),fabric_id=fabric.resource_id) + vlan = vlan_list.add(vlan) + + result_detail['detail'].append("VLAN %s created for network %s" + % (vlan.resource_id, n.name)) + if subnet is not None: + subnet.vlan = vlan.resource_id + subnet.update() + + else: + # Create new fabric and VLAN + fabric = maas_fabric.Fabric(self.maas_client, name=n.name) + fabric = fabric_list.add(fabric) + fabric_list.refresh() + + result_detail['detail'].append("Fabric %s created for network %s" + % (fabric.resource_id, n.name)) + + vlan_list = maas_vlan.Vlans(self.maas_client, fabric_id=new_fabric.resource_id) + vlan_list.refresh() + + # A new fabric comes with a single default VLAN. Retrieve it and update attributes + vlan = vlan_list.single() + + vlan.name = n.name + vlan.vid = n.vlan_id if n.vlan_id is not None else 0 + if getattr(n, 'mtu', None) is not None: + vlan.mtu = n.mtu + + vlan.update() + result_detail['detail'].append("VLAN %s updated for network %s" + % (vlan.resource_id, n.name)) + if subnet is not None: + # If subnet was found above, but needed attached to a new fabric/vlan then + # attach it + subnet.vlan = vlan.resource_id + subnet.update() + + if subnet is None: + # If subnet did not exist, create it here and attach it to the fabric/VLAN + subnet = maas_subnet.Subnet(self.maas_client, name=n.name, cidr=n.cidr, fabric=fabric.resource_id, + vlan=vlan.resource_id, gateway_ip=n.get_default_gateway()) + + subnet_list = maas_subnet.Subnets(self.maas_client) + subnet = subnet_list.add(subnet) + except ValueError as vex: + raise errors.DriverError("Inconsistent data from MaaS") + + subnet_list = maas_subnet.Subnets(self.maas_client) + subnet_list.refresh() + + action_result = hd_fields.ActionResult.Incomplete + + success_rate = 0 + + for n in design_networks: + exists = subnet_list.query({'cidr': n.cidr}) + if len(exists) > 0: + subnet = exists[0] + if subnet.name == n.name: + success_rate = success_rate + 1 + else: + success_rate = success_rate + 1 + else: + success_rate = success_rate + 1 + + if success_rate == len(design_networks): + action_result = hd_fields.ActionResult.Success + elif success_rate == - (len(design_networks)): + action_result = hd_fields.ActionResult.Failure + else: + action_result = hd_fields.ActionResult.PartialSuccess + + self.orchestrator.task_field_update(self.task.get_id(), + status=hd_fields.TaskStatus.Complete, + result=action_result, + result_detail=result_detail) + elif task_action == hd_fields.OrchestratorAction.IdentifyNode: + try: + machine_list = maas_machine.Machines(self.maas_client) + machine_list.refresh() + except: + self.orchestrator.task_field_update(self.task.get_id(), + status=hd_fields.TaskStatus.Complete, + result=hd_fields.ActionResult.Failure, + result_detail={'detail': 'Error accessing MaaS Machines API', 'retry': True}) + return + + nodes = self.task.node_list + + result_detail = {'detail': []} + + worked = failed = False + + for n in nodes: + try: + node = site_design.get_baremetal_node(n) + machine = machine_list.identify_baremetal_node(node) + if machine is not None: + worked = True + result_detail['detail'].append("Node %s identified in MaaS" % n) + else: + failed = True + result_detail['detail'].append("Node %s not found in MaaS" % n) + except Exception as ex: + failed = True + result_detail['detail'].append("Error identifying node %s: %s" % (n, str(ex))) + + result = None + if worked and failed: + result = hd_fields.ActionResult.PartialSuccess + elif worked: + result = hd_fields.ActionResult.Success + elif failed: + result = hd_fields.ActionResult.Failure + + self.orchestrator.task_field_update(self.task.get_id(), + status=hd_fields.TaskStatus.Complete, + result=result, + result_detail=result_detail) + + + + + diff --git a/helm_drydock/control/__init__.py b/drydock_provisioner/drivers/node/maasdriver/models/__init__.py similarity index 100% rename from helm_drydock/control/__init__.py rename to drydock_provisioner/drivers/node/maasdriver/models/__init__.py diff --git a/helm_drydock/drivers/node/maasdriver/models/base.py b/drydock_provisioner/drivers/node/maasdriver/models/base.py similarity index 87% rename from helm_drydock/drivers/node/maasdriver/models/base.py rename to drydock_provisioner/drivers/node/maasdriver/models/base.py index 9f3aa336..7d6cbd49 100644 --- a/helm_drydock/drivers/node/maasdriver/models/base.py +++ b/drydock_provisioner/drivers/node/maasdriver/models/base.py @@ -13,8 +13,9 @@ # limitations under the License. import json import re +import logging -import helm_drydock.error as errors +import drydock_provisioner.error as errors """ A representation of a MaaS REST resource. Should be subclassed for different resources and augmented with operations specific @@ -28,6 +29,7 @@ class ResourceBase(object): def __init__(self, api_client, **kwargs): self.api_client = api_client + self.logger = logging.getLogger('drydock.drivers.maasdriver') for f in self.fields: if f in kwargs.keys(): @@ -143,13 +145,16 @@ class ResourceBase(object): return i -""" -A collection of MaaS resources. -Rather than a simple list, we will key the collection on resource -ID for more efficient access. -""" class ResourceCollectionBase(object): + """ + A collection of MaaS resources. + + Rather than a simple list, we will key the collection on resource + ID for more efficient access. + + :param api_client: An instance of api_client.MaasRequestFactory + """ collection_url = '' collection_resource = ResourceBase @@ -157,12 +162,13 @@ class ResourceCollectionBase(object): def __init__(self, api_client): self.api_client = api_client self.resources = {} + self.logger = logging.getLogger('drydock.drivers.maasdriver') - """ - Parse URL for placeholders and replace them with current - instance values - """ def interpolate_url(self): + """ + Parse URL for placeholders and replace them with current + instance values + """ pattern = '\{([a-z_]+)\}' regex = re.compile(pattern) start = 0 @@ -250,8 +256,23 @@ class ResourceCollectionBase(object): return result + def singleton(self, query): + """ + A query that requires a single item response + + :param query: A dict of k:v pairs defining the query parameters + """ + result = self.query(query) + + if len(result) > 1: + raise ValueError("Multiple results found") + elif len(result) == 1: + return result[0] + + return None + """ - If the collection has a single item, return it + If the collection contains a single item, return it """ def single(self): if self.len() == 1: diff --git a/helm_drydock/drivers/node/maasdriver/models/fabric.py b/drydock_provisioner/drivers/node/maasdriver/models/fabric.py similarity index 88% rename from helm_drydock/drivers/node/maasdriver/models/fabric.py rename to drydock_provisioner/drivers/node/maasdriver/models/fabric.py index a105f354..90e954af 100644 --- a/helm_drydock/drivers/node/maasdriver/models/fabric.py +++ b/drydock_provisioner/drivers/node/maasdriver/models/fabric.py @@ -13,8 +13,8 @@ # limitations under the License. import json -import helm_drydock.drivers.node.maasdriver.models.base as model_base -import helm_drydock.drivers.node.maasdriver.models.vlan as model_vlan +import drydock_provisioner.drivers.node.maasdriver.models.base as model_base +import drydock_provisioner.drivers.node.maasdriver.models.vlan as model_vlan class Fabric(model_base.ResourceBase): diff --git a/drydock_provisioner/drivers/node/maasdriver/models/interface.py b/drydock_provisioner/drivers/node/maasdriver/models/interface.py new file mode 100644 index 00000000..5c257279 --- /dev/null +++ b/drydock_provisioner/drivers/node/maasdriver/models/interface.py @@ -0,0 +1,34 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import drydock_provisioner.drivers.node.maasdriver.models.base as model_base + +class Interface(model_base.ResourceBase): + + resource_url = 'nodes/{system_id}/interfaces/{resource_id}/' + fields = ['resource_id', 'system_id', 'name', 'type', 'mac_address', 'vlan', + 'links', 'effective_mtu'] + json_fields = ['name', 'type', 'mac_address', 'vlan', 'links', 'effective_mtu'] + + def __init__(self, api_client, **kwargs): + super(Interface, self).__init__(api_client, **kwargs) + +class Interfaces(model_base.ResourceCollectionBase): + + collection_url = 'nodes/{system_id}/interfaces/' + collection_resource = Interface + + def __init__(self, api_client, **kwargs): + super(Interfaces, self).__init__(api_client) + self.system_id = kwargs.get('system_id', None) \ No newline at end of file diff --git a/drydock_provisioner/drivers/node/maasdriver/models/machine.py b/drydock_provisioner/drivers/node/maasdriver/models/machine.py new file mode 100644 index 00000000..f4ec3609 --- /dev/null +++ b/drydock_provisioner/drivers/node/maasdriver/models/machine.py @@ -0,0 +1,185 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import drydock_provisioner.drivers.node.maasdriver.models.base as model_base +import drydock_provisioner.drivers.node.maasdriver.models.interface as maas_interface +import bson +import yaml + +class Machine(model_base.ResourceBase): + + resource_url = 'machines/{resource_id}/' + fields = ['resource_id', 'hostname', 'power_type', 'power_state', 'power_parameters', 'interfaces', + 'boot_interface', 'memory', 'cpu_count', 'tag_names'] + json_fields = ['hostname', 'power_type'] + + def __init__(self, api_client, **kwargs): + super(Machine, self).__init__(api_client, **kwargs) + + # Replace generic dicts with interface collection model + if getattr(self, 'resource_id', None) is not None: + self.interfaces = maas_interface.Interfaces(api_client, system_id=self.resource_id) + self.interfaces.refresh() + + def get_power_params(self): + url = self.interpolate_url() + + resp = self.api_client.get(url, op='power_parameters') + + if resp.status_code == 200: + self.power_parameters = resp.json() + + def commission(self, debug=False): + url = self.interpolate_url() + + # If we want to debug this node commissioning, enable SSH + # after commissioning and leave the node powered up + + options = {'enable_ssh': '1' if debug else '0'} + + resp = self.api_client.post(url, op='commission', files=options) + + # Need to sort out how to handle exceptions + if not resp.ok: + raise Exception() + + def get_details(self): + url = self.interpolate_url() + + resp = self.api_client.get(url, op='details') + + if resp.status_code == 200: + detail_config = bson.loads(resp.text) + return detail_config + + + def to_dict(self): + """ + Serialize this resource instance into a dict matching the + MAAS representation of the resource + """ + data_dict = {} + + for f in self.json_fields: + if getattr(self, f, None) is not None: + if f == 'resource_id': + data_dict['system_id'] = getattr(self, f) + else: + data_dict[f] = getattr(self, f) + + return data_dict + + @classmethod + def from_dict(cls, api_client, obj_dict): + """ + Create a instance of this resource class based on a dict + of MaaS type attributes + + Customized for Machine due to use of system_id instead of id + as resource key + + :param api_client: Instance of api_client.MaasRequestFactory for accessing MaaS API + :param obj_dict: Python dict as parsed from MaaS API JSON representing this resource type + """ + + refined_dict = {k: obj_dict.get(k, None) for k in cls.fields} + + if 'system_id' in obj_dict.keys(): + refined_dict['resource_id'] = obj_dict.get('system_id') + + i = cls(api_client, **refined_dict) + return i + +class Machines(model_base.ResourceCollectionBase): + + collection_url = 'machines/' + collection_resource = Machine + + def __init__(self, api_client, **kwargs): + super(Machines, self).__init__(api_client) + + # Add the OOB power parameters to each machine instance + def collect_power_params(self): + for k, v in self.resources.items(): + v.get_power_params() + + + def identify_baremetal_node(self, node_model, update_name=True): + """ + Search all the defined MaaS Machines and attempt to match + one against the provided Drydock BaremetalNode model. Update + the MaaS instance with the correct hostname + + :param node_model: Instance of objects.node.BaremetalNode to search MaaS for matching resource + :param update_name: Whether Drydock should update the MaaS resource name to match the Drydock design + """ + node_oob_network = node_model.oob_network + node_oob_ip = node_model.get_network_address(node_oob_network) + + if node_oob_ip is None: + self.logger.warn("Node model missing OOB IP address") + raise ValueError('Node model missing OOB IP address') + + try: + self.collect_power_params() + + maas_node = self.singleton({'power_params.power_address': node_oob_ip}) + + self.logger.debug("Found MaaS resource %s matching Node %s" % (maas_node.resource_id, node_model.get_id())) + + if maas_node.hostname != node_model.name and update_name: + maas_node.hostname = node_model.name + maas_node.update() + self.logger.debug("Updated MaaS resource %s hostname to %s" % (maas_node.resource_id, node_model.name)) + return maas_node + + except ValueError as ve: + self.logger.warn("Error locating matching MaaS resource for OOB IP %s" % (node_oob_ip)) + return None + + def query(self, query): + """ + Custom query method to deal with complex fields + """ + result = list(self.resources.values()) + for (k, v) in query.items(): + if k.startswith('power_params.'): + field = k[13:] + result = [i for i in result + if str(getattr(i,'power_parameters', {}).get(field, None)) == str(v)] + else: + result = [i for i in result + if str(getattr(i, k, None)) == str(v)] + + return result + + + def add(self, res): + """ + Create a new resource in this collection in MaaS + + Customize as Machine resources use 'system_id' instead of 'id' + """ + data_dict = res.to_dict() + url = self.interpolate_url() + + resp = self.api_client.post(url, files=data_dict) + + if resp.status_code == 200: + resp_json = resp.json() + res.set_resource_id(resp_json.get('system_id')) + return res + + raise errors.DriverError("Failed updating MAAS url %s - return code %s" + % (url, resp.status_code)) \ No newline at end of file diff --git a/helm_drydock/drivers/node/maasdriver/models/subnet.py b/drydock_provisioner/drivers/node/maasdriver/models/subnet.py similarity index 96% rename from helm_drydock/drivers/node/maasdriver/models/subnet.py rename to drydock_provisioner/drivers/node/maasdriver/models/subnet.py index ccf677c2..8aec521e 100644 --- a/helm_drydock/drivers/node/maasdriver/models/subnet.py +++ b/drydock_provisioner/drivers/node/maasdriver/models/subnet.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import helm_drydock.drivers.node.maasdriver.models.base as model_base +import drydock_provisioner.drivers.node.maasdriver.models.base as model_base class Subnet(model_base.ResourceBase): diff --git a/helm_drydock/drivers/node/maasdriver/models/vlan.py b/drydock_provisioner/drivers/node/maasdriver/models/vlan.py similarity index 95% rename from helm_drydock/drivers/node/maasdriver/models/vlan.py rename to drydock_provisioner/drivers/node/maasdriver/models/vlan.py index f4f506ef..36acceac 100644 --- a/helm_drydock/drivers/node/maasdriver/models/vlan.py +++ b/drydock_provisioner/drivers/node/maasdriver/models/vlan.py @@ -13,8 +13,8 @@ # limitations under the License. import json -import helm_drydock.error as errors -import helm_drydock.drivers.node.maasdriver.models.base as model_base +import drydock_provisioner.error as errors +import drydock_provisioner.drivers.node.maasdriver.models.base as model_base class Vlan(model_base.ResourceBase): diff --git a/helm_drydock/drivers/node/maasdriver/readme.md b/drydock_provisioner/drivers/node/maasdriver/readme.md similarity index 100% rename from helm_drydock/drivers/node/maasdriver/readme.md rename to drydock_provisioner/drivers/node/maasdriver/readme.md diff --git a/helm_drydock/drivers/oob/__init__.py b/drydock_provisioner/drivers/oob/__init__.py similarity index 87% rename from helm_drydock/drivers/oob/__init__.py rename to drydock_provisioner/drivers/oob/__init__.py index ada30fb8..8e7a9c26 100644 --- a/helm_drydock/drivers/oob/__init__.py +++ b/drydock_provisioner/drivers/oob/__init__.py @@ -12,17 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -import helm_drydock.objects.fields as hd_fields -import helm_drydock.error as errors +import drydock_provisioner.objects.fields as hd_fields +import drydock_provisioner.error as errors -from helm_drydock.drivers import ProviderDriver +from drydock_provisioner.drivers import ProviderDriver class OobDriver(ProviderDriver): def __init__(self, **kwargs): super(OobDriver, self).__init__(**kwargs) - self.supported_actions = [hd_fields.OrchestrationAction.ValidateOobServices, + self.supported_actions = [hd_fields.OrchestratorAction.ValidateOobServices, hd_fields.OrchestratorAction.ConfigNodePxe, hd_fields.OrchestratorAction.SetNodeBoot, hd_fields.OrchestratorAction.PowerOffNode, diff --git a/helm_drydock/drivers/oob/pyghmi_driver/__init__.py b/drydock_provisioner/drivers/oob/pyghmi_driver/__init__.py similarity index 92% rename from helm_drydock/drivers/oob/pyghmi_driver/__init__.py rename to drydock_provisioner/drivers/oob/pyghmi_driver/__init__.py index 9a57efe9..73c2bd5c 100644 --- a/helm_drydock/drivers/oob/pyghmi_driver/__init__.py +++ b/drydock_provisioner/drivers/oob/pyghmi_driver/__init__.py @@ -12,17 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. import time +import logging from pyghmi.ipmi.command import Command -import helm_drydock.error as errors -import helm_drydock.config as config +import drydock_provisioner.error as errors +import drydock_provisioner.config as config -import helm_drydock.objects.fields as hd_fields -import helm_drydock.objects.task as task_model +import drydock_provisioner.objects.fields as hd_fields +import drydock_provisioner.objects.task as task_model -import helm_drydock.drivers.oob as oob -import helm_drydock.drivers as drivers +import drydock_provisioner.drivers.oob as oob +import drydock_provisioner.drivers as drivers class PyghmiDriver(oob.OobDriver): @@ -34,15 +35,19 @@ class PyghmiDriver(oob.OobDriver): self.driver_key = "pyghmi_driver" self.driver_desc = "Pyghmi OOB Driver" - self.config = config.DrydockConfig.node_driver[self.driver_key] + self.logger = logging.getLogger('drydock.oobdriver.pyghmi') + self.config = config.DrydockConfig.node_driver.get(self.driver_key, {}) def execute_task(self, task_id): task = self.state_manager.get_task(task_id) if task is None: + self.logger.error("Invalid task %s" % (task_id)) raise errors.DriverError("Invalid task %s" % (task_id)) if task.action not in self.supported_actions: + self.logger.error("Driver %s doesn't support task action %s" + % (self.driver_desc, task.action)) raise errors.DriverError("Driver %s doesn't support task action %s" % (self.driver_desc, task.action)) @@ -66,7 +71,7 @@ class PyghmiDriver(oob.OobDriver): result=hd_fields.ActionResult.Success) return - site_design = self.orchestrator.get_effective_site(design_id, task.site_name) + site_design = self.orchestrator.get_effective_site(design_id) target_nodes = [] @@ -118,13 +123,6 @@ class PyghmiDriver(oob.OobDriver): if x.get_result() in [hd_fields.ActionResult.PartialSuccess, hd_fields.ActionResult.Failure]] - print("Task %s successful subtasks: %s" % - (task.get_id(), len(success_subtasks))) - print("Task %s unsuccessful subtasks: %s" % - (task.get_id(), len(nosuccess_subtasks))) - print("Task %s total subtasks: %s" % - (task.get_id(), len(task.get_subtasks()))) - task_result = None if len(success_subtasks) > 0 and len(nosuccess_subtasks) > 0: task_result = hd_fields.ActionResult.PartialSuccess @@ -145,9 +143,11 @@ class PyghmiTaskRunner(drivers.DriverTaskRunner): def __init__(self, node=None, **kwargs): super(PyghmiTaskRunner, self).__init__(**kwargs) + self.logger = logging.getLogger('drydock.oobdriver.pyghmi') # We cheat here by providing the Node model instead # of making the runner source it from statemgmt if node is None: + self.logger.error("Did not specify target node") raise errors.DriverError("Did not specify target node") self.node = node @@ -172,7 +172,7 @@ class PyghmiTaskRunner(drivers.DriverTaskRunner): "task node scope") - ipmi_network = self.node.applied.get('oob_network') + ipmi_network = self.node.oob_network ipmi_address = self.node.get_network_address(ipmi_network) if ipmi_address is None: @@ -184,8 +184,8 @@ class PyghmiTaskRunner(drivers.DriverTaskRunner): self.orchestrator.task_field_update(self.task.get_id(), status=hd_fields.TaskStatus.Running) - ipmi_account = self.node.applied.get('oob_account', '') - ipmi_credential = self.node.applied.get('oob_credential', '') + ipmi_account = self.node.oob_account + ipmi_credential = self.node.oob_credential ipmi_session = Command(bmc=ipmi_address, userid=ipmi_account, password=ipmi_credential) diff --git a/helm_drydock/drivers/readme.md b/drydock_provisioner/drivers/readme.md similarity index 93% rename from helm_drydock/drivers/readme.md rename to drydock_provisioner/drivers/readme.md index 0aab4c1c..63214dd7 100644 --- a/helm_drydock/drivers/readme.md +++ b/drydock_provisioner/drivers/readme.md @@ -32,6 +32,7 @@ and storage. * CreateStorageTemplate - Configure site-wide storage information in bootstrapper * CreateBootMedia - Ensure all needed boot media is available to the bootstrapper including external repositories * PrepareHardwareConfig - Prepare the bootstrapper to handle all hardware configuration actions (firmware updates, RAID configuration, driver installation) +* IdentifyNode - Correlate a node definition in the Drydock internal model with a node detected by the downstream node bootstrapper. * ConfigureHardware - Update and validate all hardware configurations on a node prior to deploying the OS on it * InterrogateNode - Interrogate the bootstrapper about node information. Depending on the current state of the node, this interrogation will produce different information. * ApplyNodeNetworking - Configure networking for a node diff --git a/drydock_provisioner/drydock.py b/drydock_provisioner/drydock.py new file mode 100644 index 00000000..18e53a13 --- /dev/null +++ b/drydock_provisioner/drydock.py @@ -0,0 +1,55 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +import drydock_provisioner.config as config +import drydock_provisioner.objects as objects +import drydock_provisioner.ingester as ingester +import drydock_provisioner.statemgmt as statemgmt +import drydock_provisioner.orchestrator as orch +import drydock_provisioner.control.api as api + +def start_drydock(): + objects.register_all() + + # Setup root logger + logger = logging.getLogger('drydock') + + logger.setLevel(config.DrydockConfig.global_config.get('log_level')) + ch = logging.StreamHandler() + formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(filename)s:%(funcName)s - %(message)s') + ch.setFormatter(formatter) + logger.addHandler(ch) + + # Specalized format for API logging + logger = logging.getLogger('drydock.control') + logger.propagate = False + formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(user)s - %(req_id)s - %(external_ctx)s - %(message)s') + + ch = logging.StreamHandler() + ch.setFormatter(formatter) + logger.addHandler(ch) + + state = statemgmt.DesignState() + + orchestrator = orch.Orchestrator(config.DrydockConfig.orchestrator_config.get('drivers', {}), + state_manager=state) + input_ingester = ingester.Ingester() + input_ingester.enable_plugins(config.DrydockConfig.ingester_config.get('plugins', [])) + + return api.start_api(state_manager=state, ingester=input_ingester, + orchestrator=orchestrator) + +drydock = start_drydock() + diff --git a/helm_drydock/error.py b/drydock_provisioner/error.py similarity index 92% rename from helm_drydock/error.py rename to drydock_provisioner/error.py index a8988f97..04a38196 100644 --- a/helm_drydock/error.py +++ b/drydock_provisioner/error.py @@ -34,4 +34,10 @@ class TransientDriverError(DriverError): pass class PersistentDriverError(DriverError): + pass + +class ApiError(Exception): + pass + +class InvalidFormat(ApiError): pass \ No newline at end of file diff --git a/drydock_provisioner/ingester/__init__.py b/drydock_provisioner/ingester/__init__.py new file mode 100644 index 00000000..daaff310 --- /dev/null +++ b/drydock_provisioner/ingester/__init__.py @@ -0,0 +1,120 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ingester - Ingest host topologies to define site design and +# persist design to helm-drydock's statemgmt service + +import logging +import yaml +import uuid +import importlib + +import drydock_provisioner.objects as objects +import drydock_provisioner.objects.site as site +import drydock_provisioner.objects.network as network +import drydock_provisioner.objects.hwprofile as hwprofile +import drydock_provisioner.objects.node as node +import drydock_provisioner.objects.hostprofile as hostprofile + +from drydock_provisioner.statemgmt import DesignState + +class Ingester(object): + + def __init__(self): + self.logger = logging.getLogger("drydock.ingester") + self.registered_plugins = {} + + def enable_plugins(self, plugins=[]): + """ + enable_plugins + + :params plugins: - A list of strings naming class objects denoting the ingester plugins to be enabled + + Enable plugins that can be used for ingest_data calls. Each plugin should use + drydock_provisioner.ingester.plugins.IngesterPlugin as its base class. As long as one + enabled plugin successfully initializes, the call is considered successful. Otherwise + it will throw an exception + """ + if len(plugins) == 0: + self.log.error("Cannot have an empty plugin list.") + + for plugin in plugins: + try: + (module, x, classname) = plugin.rpartition('.') + + if module == '': + raise Exception() + mod = importlib.import_module(module) + klass = getattr(mod, classname) + new_plugin = klass() + plugin_name = new_plugin.get_name() + self.registered_plugins[plugin_name] = new_plugin + except Exception as ex: + self.logger.error("Could not enable plugin %s - %s" % (plugin, str(ex))) + + if len(self.registered_plugins) == 0: + self.logger.error("Could not enable at least one plugin") + raise Exception("Could not enable at least one plugin") + + + def ingest_data(self, plugin_name='', design_state=None, design_id=None, context=None, **kwargs): + if design_state is None: + self.logger.error("Ingester:ingest_data called without valid DesignState handler") + raise ValueError("Invalid design_state handler") + + # If no design_id is specified, instantiate a new one + if 'design_id' is None: + self.logger.error("Ingester:ingest_data required kwarg 'design_id' missing") + raise ValueError("Ingester:ingest_data required kwarg 'design_id' missing") + + design_data = design_state.get_design(design_id) + + self.logger.debug("Ingester:ingest_data ingesting design parts for design %s" % design_id) + + if plugin_name in self.registered_plugins: + try: + design_items = self.registered_plugins[plugin_name].ingest_data(**kwargs) + except ValueError as vex: + self.logger.warn("Ingester:ingest_data - Error process data - %s" % (str(vex))) + return None + self.logger.debug("Ingester:ingest_data parsed %s design parts" % str(len(design_items))) + for m in design_items: + if context is not None: + m.set_create_fields(context) + if type(m) is site.Site: + design_data.set_site(m) + elif type(m) is network.Network: + design_data.add_network(m) + elif type(m) is network.NetworkLink: + design_data.add_network_link(m) + elif type(m) is hostprofile.HostProfile: + design_data.add_host_profile(m) + elif type(m) is hwprofile.HardwareProfile: + design_data.add_hardware_profile(m) + elif type(m) is node.BaremetalNode: + design_data.add_baremetal_node(m) + design_state.put_design(design_data) + return design_items + else: + self.logger.error("Could not find plugin %s to ingest data." % (plugin_name)) + raise LookupError("Could not find plugin %s" % plugin_name) + """ + ingest_data + + params: plugin_name - Which plugin should be used for ingestion + params: params - A map of parameters that will be passed to the plugin's ingest_data method + + Execute a data ingestion using the named plugin (assuming it is enabled) + """ + diff --git a/helm_drydock/ingester/plugins/__init__.py b/drydock_provisioner/ingester/plugins/__init__.py similarity index 100% rename from helm_drydock/ingester/plugins/__init__.py rename to drydock_provisioner/ingester/plugins/__init__.py diff --git a/helm_drydock/ingester/plugins/yaml.py b/drydock_provisioner/ingester/plugins/yaml.py similarity index 97% rename from helm_drydock/ingester/plugins/yaml.py rename to drydock_provisioner/ingester/plugins/yaml.py index ce531f8c..d3a9c4dd 100644 --- a/helm_drydock/ingester/plugins/yaml.py +++ b/drydock_provisioner/ingester/plugins/yaml.py @@ -19,15 +19,16 @@ import yaml import logging -import helm_drydock.objects.fields as hd_fields +import drydock_provisioner.objects.fields as hd_fields -from helm_drydock import objects -from helm_drydock.ingester.plugins import IngesterPlugin +from drydock_provisioner import objects +from drydock_provisioner.ingester.plugins import IngesterPlugin class YamlIngester(IngesterPlugin): def __init__(self): super(YamlIngester, self).__init__() + self.logger = logging.getLogger('drydock.ingester.yaml') def get_name(self): return "yaml" @@ -37,7 +38,7 @@ class YamlIngester(IngesterPlugin): filenames - Array of absolute path to the YAML files to ingest - returns an array of objects from helm_drydock.model + returns an array of objects from drydock_provisioner.model """ def ingest_data(self, **kwargs): @@ -52,12 +53,10 @@ class YamlIngester(IngesterPlugin): file.close() models.extend(self.parse_docs(contents)) except OSError as err: - self.log.error( + self.logger.error( "Error opening input file %s for ingestion: %s" % (filename, err)) continue - - elif 'content' in kwargs: models.extend(self.parse_docs(kwargs.get('content'))) else: @@ -71,6 +70,8 @@ class YamlIngester(IngesterPlugin): def parse_docs(self, yaml_string): models = [] + self.logger.debug("yamlingester:parse_docs - Parsing YAML string \n%s" % (yaml_string)) + try: parsed_data = yaml.load_all(yaml_string) except yaml.YAMLError as err: diff --git a/helm_drydock/ingester/readme.md b/drydock_provisioner/ingester/readme.md similarity index 84% rename from helm_drydock/ingester/readme.md rename to drydock_provisioner/ingester/readme.md index 1437c216..16352970 100644 --- a/helm_drydock/ingester/readme.md +++ b/drydock_provisioner/ingester/readme.md @@ -6,7 +6,7 @@ different sources. Each ingester plugin should be able source data based on user-provided parameters and parse that data -into the Drydock internal model (helm_drydock.model). +into the Drydock internal model (drydock_provisioner.model). Each plugin does not need to support every type of design data as a single site design may be federated from multiple diff --git a/helm_drydock/introspection/readme.md b/drydock_provisioner/introspection/readme.md similarity index 100% rename from helm_drydock/introspection/readme.md rename to drydock_provisioner/introspection/readme.md diff --git a/helm_drydock/objects/__init__.py b/drydock_provisioner/objects/__init__.py similarity index 93% rename from helm_drydock/objects/__init__.py rename to drydock_provisioner/objects/__init__.py index b88365d2..be295cf6 100644 --- a/helm_drydock/objects/__init__.py +++ b/drydock_provisioner/objects/__init__.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# Models for helm_drydock +# Models for drydock_provisioner # import logging @@ -23,11 +23,11 @@ def register_all(): # NOTE(sh8121att) - Import all versioned objects so # they are available via RPC. Any new object definitions # need to be added here. - __import__('helm_drydock.objects.network') - __import__('helm_drydock.objects.node') - __import__('helm_drydock.objects.hostprofile') - __import__('helm_drydock.objects.hwprofile') - __import__('helm_drydock.objects.site') + __import__('drydock_provisioner.objects.network') + __import__('drydock_provisioner.objects.node') + __import__('drydock_provisioner.objects.hostprofile') + __import__('drydock_provisioner.objects.hwprofile') + __import__('drydock_provisioner.objects.site') # Utility class for calculating inheritance diff --git a/helm_drydock/objects/base.py b/drydock_provisioner/objects/base.py similarity index 59% rename from helm_drydock/objects/base.py rename to drydock_provisioner/objects/base.py index d22b2183..3bf2c027 100644 --- a/helm_drydock/objects/base.py +++ b/drydock_provisioner/objects/base.py @@ -11,16 +11,17 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import datetime from oslo_versionedobjects import base from oslo_versionedobjects import fields as obj_fields -import helm_drydock.objects as objects +import drydock_provisioner.objects as objects class DrydockObjectRegistry(base.VersionedObjectRegistry): # Steal this from Cinder to bring all registered objects - # into the helm_drydock.objects namespace + # into the drydock_provisioner.objects namespace def registration_hook(self, cls, index): setattr(objects, cls.obj_name(), cls) @@ -29,7 +30,7 @@ class DrydockObject(base.VersionedObject): VERSION = '1.0' - OBJ_PROJECT_NAMESPACE = 'helm_drydock.objects' + OBJ_PROJECT_NAMESPACE = 'drydock_provisioner.objects' # Return None for undefined attributes def obj_load_attr(self, attrname): @@ -38,6 +39,32 @@ class DrydockObject(base.VersionedObject): else: raise ValueError("Unknown field %s" % (attrname)) + def obj_to_simple(self): + """ + Create a simple primitive representation of this object excluding + all the versioning stuff. Used to serialize an object for public + consumption, not intended to be deserialized by OVO + """ + + primitive = dict() + + primitive['model_type'] = self.__class__.__name__ + primitive['model_version'] = self.VERSION + + for name, field in self.fields.items(): + if self.obj_attr_is_set(name): + value = getattr(self, name) + if (hasattr(value, 'obj_to_simple') and + callable(value.obj_to_simple)): + primitive[name] = value.obj_to_simple() + else: + value = field.to_primitive(self, name, value) + if value is not None: + primitive[name] = value + + return primitive + + class DrydockPersistentObject(base.VersionedObject): fields = { @@ -47,6 +74,15 @@ class DrydockPersistentObject(base.VersionedObject): 'updated_by': obj_fields.StringField(nullable=True), } + def set_create_fields(self, context): + self.created_at = datetime.datetime.now() + self.created_by = context.user + + def set_update_fields(self, context): + self.updated_at = datetime.datetime.now() + self.updated_by = context.user + + class DrydockObjectListBase(base.ObjectListBase): def __init__(self, **kwargs): @@ -73,3 +109,11 @@ class DrydockObjectListBase(base.ObjectListBase): model_list.append(o) return model_list + + def obj_to_simple(self): + primitive_list = list() + + for o in self.objects: + primitive_list.append(o.obj_to_simple()) + + return primitive_list diff --git a/helm_drydock/objects/fields.py b/drydock_provisioner/objects/fields.py similarity index 99% rename from helm_drydock/objects/fields.py rename to drydock_provisioner/objects/fields.py index c6ac8ac3..2abb6098 100644 --- a/helm_drydock/objects/fields.py +++ b/drydock_provisioner/objects/fields.py @@ -44,6 +44,7 @@ class OrchestratorAction(BaseDrydockEnum): CreateStorageTemplate = 'create_storage_template' CreateBootMedia = 'create_boot_media' PrepareHardwareConfig = 'prepare_hardware_config' + IdentifyNode = 'identify_node' ConfigureHardware = 'configure_hardware' InterrogateNode = 'interrogate_node' ApplyNodeNetworking = 'apply_node_networking' diff --git a/helm_drydock/objects/hostprofile.py b/drydock_provisioner/objects/hostprofile.py similarity index 98% rename from helm_drydock/objects/hostprofile.py rename to drydock_provisioner/objects/hostprofile.py index 5a416dde..36aad003 100644 --- a/helm_drydock/objects/hostprofile.py +++ b/drydock_provisioner/objects/hostprofile.py @@ -16,9 +16,9 @@ from copy import deepcopy import oslo_versionedobjects.fields as obj_fields -import helm_drydock.objects as objects -import helm_drydock.objects.base as base -import helm_drydock.objects.fields as hd_fields +import drydock_provisioner.objects as objects +import drydock_provisioner.objects.base as base +import drydock_provisioner.objects.fields as hd_fields @base.DrydockObjectRegistry.register diff --git a/helm_drydock/objects/hwprofile.py b/drydock_provisioner/objects/hwprofile.py similarity index 96% rename from helm_drydock/objects/hwprofile.py rename to drydock_provisioner/objects/hwprofile.py index 3ff5afbe..bda20183 100644 --- a/helm_drydock/objects/hwprofile.py +++ b/drydock_provisioner/objects/hwprofile.py @@ -16,9 +16,9 @@ from copy import deepcopy from oslo_versionedobjects import fields as ovo_fields -import helm_drydock.objects as objects -import helm_drydock.objects.base as base -import helm_drydock.objects.fields as hd_fields +import drydock_provisioner.objects as objects +import drydock_provisioner.objects.base as base +import drydock_provisioner.objects.fields as hd_fields @base.DrydockObjectRegistry.register class HardwareProfile(base.DrydockPersistentObject, base.DrydockObject): diff --git a/helm_drydock/objects/network.py b/drydock_provisioner/objects/network.py similarity index 94% rename from helm_drydock/objects/network.py rename to drydock_provisioner/objects/network.py index e1ccc693..028161da 100644 --- a/helm_drydock/objects/network.py +++ b/drydock_provisioner/objects/network.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# Models for helm_drydock +# Models for drydock_provisioner # import logging @@ -20,9 +20,9 @@ from copy import deepcopy import oslo_versionedobjects.fields as ovo_fields -import helm_drydock.objects as objects -import helm_drydock.objects.base as base -import helm_drydock.objects.fields as hd_fields +import drydock_provisioner.objects as objects +import drydock_provisioner.objects.base as base +import drydock_provisioner.objects.fields as hd_fields @base.DrydockObjectRegistry.register class NetworkLink(base.DrydockPersistentObject, base.DrydockObject): diff --git a/helm_drydock/objects/node.py b/drydock_provisioner/objects/node.py similarity index 92% rename from helm_drydock/objects/node.py rename to drydock_provisioner/objects/node.py index 1d353576..7854b346 100644 --- a/helm_drydock/objects/node.py +++ b/drydock_provisioner/objects/node.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# Models for helm_drydock +# Models for drydock_provisioner # import logging @@ -20,13 +20,13 @@ from copy import deepcopy from oslo_versionedobjects import fields as ovo_fields -import helm_drydock.objects as objects -import helm_drydock.objects.hostprofile -import helm_drydock.objects.base as base -import helm_drydock.objects.fields as hd_fields +import drydock_provisioner.objects as objects +import drydock_provisioner.objects.hostprofile +import drydock_provisioner.objects.base as base +import drydock_provisioner.objects.fields as hd_fields @base.DrydockObjectRegistry.register -class BaremetalNode(helm_drydock.objects.hostprofile.HostProfile): +class BaremetalNode(drydock_provisioner.objects.hostprofile.HostProfile): VERSION = '1.0' diff --git a/helm_drydock/objects/readme.md b/drydock_provisioner/objects/readme.md similarity index 100% rename from helm_drydock/objects/readme.md rename to drydock_provisioner/objects/readme.md diff --git a/helm_drydock/objects/site.py b/drydock_provisioner/objects/site.py similarity index 93% rename from helm_drydock/objects/site.py rename to drydock_provisioner/objects/site.py index f786d911..c86a9184 100644 --- a/helm_drydock/objects/site.py +++ b/drydock_provisioner/objects/site.py @@ -12,16 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# Models for helm_drydock +# Models for drydock_provisioner # from copy import deepcopy import uuid +import datetime import oslo_versionedobjects.fields as ovo_fields -import helm_drydock.objects as objects -import helm_drydock.objects.base as base -import helm_drydock.objects.fields as hd_fields +import drydock_provisioner.objects as objects +import drydock_provisioner.objects.base as base +import drydock_provisioner.objects.fields as hd_fields @base.DrydockObjectRegistry.register @@ -126,8 +127,6 @@ class SiteDesign(base.DrydockPersistentObject, base.DrydockObject): def __init__(self, **kwargs): super(SiteDesign, self).__init__(**kwargs) - - # Assign UUID id def assign_id(self): self.id = uuid.uuid4() @@ -228,6 +227,18 @@ class SiteDesign(base.DrydockPersistentObject, base.DrydockObject): raise DesignError("BaremetalNode %s not found in design state" % node_key) + def create(self, ctx, state_manager): + self.created_at = datetime.datetime.now() + self.created_by = ctx.user + + state_manager.post_design(self) + + def save(self, ctx, state_manager): + self.updated_at = datetime.datetime.now() + self.updated_by = ctx.user + + state_manager.put_design(self) + """ Support filtering on rack name, node name or node tag for now. Each filter can be a comma-delimited list of diff --git a/helm_drydock/objects/task.py b/drydock_provisioner/objects/task.py similarity index 70% rename from helm_drydock/objects/task.py rename to drydock_provisioner/objects/task.py index 9985b285..6da718c8 100644 --- a/helm_drydock/objects/task.py +++ b/drydock_provisioner/objects/task.py @@ -15,9 +15,9 @@ import uuid from threading import Lock -import helm_drydock.error as errors +import drydock_provisioner.error as errors -import helm_drydock.objects.fields as hd_fields +import drydock_provisioner.objects.fields as hd_fields class Task(object): @@ -66,18 +66,29 @@ class Task(object): def get_subtasks(self): return self.subtasks + def to_dict(self): + return { + 'task_id': str(self.task_id), + 'action': self.action, + 'parent_task': str(self.parent_task_id), + 'status': self.status, + 'result': self.result, + 'result_detail': self.result_detail, + 'subtasks': [str(x) for x in self.subtasks], + } + class OrchestratorTask(Task): - def __init__(self, **kwargs): + def __init__(self, site=None, design_id=None, **kwargs): super(OrchestratorTask, self).__init__(**kwargs) # Validate parameters based on action - self.site = kwargs.get('site', '') + self.site = site - if self.site == '': + if self.site is None: raise ValueError("Orchestration Task requires 'site' parameter") - self.design_id = kwargs.get('design_id', 0) + self.design_id = design_id if self.action in [hd_fields.OrchestratorAction.VerifyNode, hd_fields.OrchestratorAction.PrepareNode, @@ -85,6 +96,14 @@ class OrchestratorTask(Task): hd_fields.OrchestratorAction.DestroyNode]: self.node_filter = kwargs.get('node_filter', None) + def to_dict(self): + _dict = super(OrchestratorTask, self).to_dict() + + _dict['site'] = self.site + _dict['design_id'] = self.design_id + _dict['node_filter'] = getattr(self, 'node_filter', None) + + return _dict class DriverTask(Task): def __init__(self, task_scope={}, **kwargs): @@ -94,4 +113,13 @@ class DriverTask(Task): self.site_name = task_scope.get('site', None) - self.node_list = task_scope.get('node_names', []) \ No newline at end of file + self.node_list = task_scope.get('node_names', []) + + def to_dict(self): + _dict = super(DriverTask, self).to_dict() + + _dict['site_name'] = self.site_name + _dict['design_id'] = self.design_id + _dict['node_list'] = self.node_list + + return _dict \ No newline at end of file diff --git a/helm_drydock/orchestrator/__init__.py b/drydock_provisioner/orchestrator/__init__.py similarity index 71% rename from helm_drydock/orchestrator/__init__.py rename to drydock_provisioner/orchestrator/__init__.py index 2b589156..69984336 100644 --- a/helm_drydock/orchestrator/__init__.py +++ b/drydock_provisioner/orchestrator/__init__.py @@ -1,3 +1,4 @@ + # Copyright 2017 AT&T Intellectual Property. All other rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,13 +16,14 @@ import uuid import time import threading import importlib +import logging from copy import deepcopy -import helm_drydock.drivers as drivers -import helm_drydock.objects.task as tasks -import helm_drydock.error as errors -import helm_drydock.objects.fields as hd_fields +import drydock_provisioner.drivers as drivers +import drydock_provisioner.objects.task as tasks +import drydock_provisioner.error as errors +import drydock_provisioner.objects.fields as hd_fields class Orchestrator(object): @@ -32,6 +34,8 @@ class Orchestrator(object): self.state_manager = state_manager + self.logger = logging.getLogger('drydock.orchestrator') + if enabled_drivers is not None: oob_driver_name = enabled_drivers.get('oob', None) if oob_driver_name is not None: @@ -106,8 +110,7 @@ class Orchestrator(object): self.task_field_update(task_id, status=hd_fields.TaskStatus.Running) try: - site_design = self.get_effective_site(task_site, - change_id=design_id) + site_design = self.get_effective_site(design_id) self.task_field_update(task_id, result=hd_fields.ActionResult.Success) except: @@ -155,10 +158,14 @@ class Orchestrator(object): task_scope=task_scope, action=hd_fields.OrchestratorAction.CreateNetworkTemplate) + self.logger.info("Starting node driver task %s to create network templates" % (driver_task.get_id())) + driver.execute_task(driver_task.get_id()) driver_task = self.state_manager.get_task(driver_task.get_id()) + self.logger.info("Node driver task %s complete" % (driver_task.get_id())) + self.task_field_update(task_id, status=hd_fields.TaskStatus.Complete, result=driver_task.get_result()) @@ -167,15 +174,16 @@ class Orchestrator(object): self.task_field_update(task_id, status=hd_fields.TaskStatus.Running) - driver = self.enabled_drivers['oob'] + oob_driver = self.enabled_drivers['oob'] - if driver is None: + if oob_driver is None: self.task_field_update(task_id, status=hd_fields.TaskStatus.Errored, - result=hd_fields.ActionResult.Failure) + result=hd_fields.ActionResult.Failure, + result_detail={'detail': 'Error: No oob driver configured', 'retry': False}) return - site_design = self.get_effective_site(design_id, task_site) + site_design = self.get_effective_site(design_id) node_filter = task.node_filter @@ -186,34 +194,45 @@ class Orchestrator(object): task_scope = {'site' : task_site, 'node_names' : target_names} - driver_task = self.create_task(tasks.DriverTask, + oob_driver_task = self.create_task(tasks.DriverTask, parent_task_id=task.get_id(), design_id=design_id, - action=hd_fields.OrchestratorAction.InterrogateNode, + action=hd_fields.OrchestratorAction.InterrogateOob, task_scope=task_scope) - driver.execute_task(driver_task.get_id()) + oob_driver.execute_task(oob_driver_task.get_id()) - driver_task = self.state_manager.get_task(driver_task.get_id()) + oob_driver_task = self.state_manager.get_task(oob_driver_task.get_id()) self.task_field_update(task_id, status=hd_fields.TaskStatus.Complete, - result=driver_task.get_result()) + result=oob_driver_task.get_result()) return elif task.action == hd_fields.OrchestratorAction.PrepareNode: + failed = worked = False + self.task_field_update(task_id, status=hd_fields.TaskStatus.Running) - driver = self.enabled_drivers['oob'] + oob_driver = self.enabled_drivers['oob'] - if driver is None: + if oob_driver is None: self.task_field_update(task_id, status=hd_fields.TaskStatus.Errored, - result=hd_fields.ActionResult.Failure) + result=hd_fields.ActionResult.Failure, + result_detail={'detail': 'Error: No oob driver configured', 'retry': False}) return - site_design = self.get_effective_site(task_site, - change_id=design_id) + node_driver = self.enabled_drivers['node'] + + if node_driver is None: + self.task_field_update(task_id, + status=hd_fields.TaskStatus.Errored, + result=hd_fields.ActionResult.Failure, + result_detail={'detail': 'Error: No node driver configured', 'retry': False}) + return + + site_design = self.get_effective_site(design_id) node_filter = task.node_filter @@ -230,33 +249,88 @@ class Orchestrator(object): action=hd_fields.OrchestratorAction.SetNodeBoot, task_scope=task_scope) - driver.execute_task(setboot_task.get_id()) + self.logger.info("Starting OOB driver task %s to set PXE boot" % (setboot_task.get_id())) + + oob_driver.execute_task(setboot_task.get_id()) + + self.logger.info("OOB driver task %s complete" % (setboot_task.get_id())) setboot_task = self.state_manager.get_task(setboot_task.get_id()) + if setboot_task.get_result() == hd_fields.ActionResult.Success: + worked = True + elif setboot_task.get_result() == hd_fields.ActionResult.PartialSuccess: + worked = failed = True + elif setboot_task.get_result() == hd_fields.ActionResult.Failure: + failed = True + cycle_task = self.create_task(tasks.DriverTask, parent_task_id=task.get_id(), design_id=design_id, action=hd_fields.OrchestratorAction.PowerCycleNode, task_scope=task_scope) - driver.execute_task(cycle_task.get_id()) + + self.logger.info("Starting OOB driver task %s to power cycle nodes" % (cycle_task.get_id())) + + oob_driver.execute_task(cycle_task.get_id()) + + self.logger.info("OOB driver task %s complete" % (cycle_task.get_id())) cycle_task = self.state_manager.get_task(cycle_task.get_id()) - if (setboot_task.get_result() == hd_fields.ActionResult.Success and - cycle_task.get_result() == hd_fields.ActionResult.Success): - self.task_field_update(task_id, - status=hd_fields.TaskStatus.Complete, - result=hd_fields.ActionResult.Success) - elif (setboot_task.get_result() == hd_fields.ActionResult.Success or - cycle_task.get_result() == hd_fields.ActionResult.Success): - self.task_field_update(task_id, - status=hd_fields.TaskStatus.Complete, - result=hd_fields.ActionResult.PartialSuccess) + if cycle_task.get_result() == hd_fields.ActionResult.Success: + worked = True + elif cycle_task.get_result() == hd_fields.ActionResult.PartialSuccess: + worked = failed = True + elif cycle_task.get_result() == hd_fields.ActionResult.Failure: + failed = True + + + # IdentifyNode success will take some time after PowerCycleNode finishes + # Retry the operation a few times if it fails before considering it a final failure + # Each attempt is a new task which might make the final task tree a bit confusing + + node_identify_attempts = 0 + + while True: + + node_identify_task = self.create_task(tasks.DriverTask, + parent_task_id=task.get_id(), + design_id=design_id, + action=hd_fields.OrchestratorAction.IdentifyNode, + task_scope=task_scope) + + self.logger.info("Starting node driver task %s to identify node - attempt %s" % + (node_identify_task.get_id(), node_identify_attempts+1)) + + node_driver.execute_task(node_identify_task.get_id()) + node_identify_attempts = node_identify_attempts + 1 + + node_identify_task = self.state_manager.get_task(node_identify_task.get_id()) + + if node_identify_task.get_result() == hd_fields.ActionResult.Success: + worked = True + break + elif node_identify_task.get_result() in [hd_fields.ActionResult.PartialSuccess, + hd_fields.ActionResult.Failure]: + # TODO This threshold should be a configurable default and tunable by task API + if node_identify_attempts > 2: + failed = True + break + + time.sleep(5 * 60) + + final_result = None + if worked and failed: + final_result = hd_fields.ActionResult.PartialSuccess + elif worked: + final_result = hd_fields.ActionResult.Success else: - self.task_field_update(task_id, - status=hd_fields.TaskStatus.Complete, - result=hd_fields.ActionResult.Failure) + final_result = hd_fields.ActionResult.Failure + + self.task_field_update(task_id, + status=hd_fields.TaskStatus.Complete, + result=final_result) return else: @@ -331,7 +405,7 @@ class Orchestrator(object): # the baremetal nodes which recursively resolves it for host profiles # assigned to those nodes - for n in site_design.baremetal_nodes: + for n in getattr(site_design, 'baremetal_nodes', []): n.compile_applied_model(site_design) return @@ -342,18 +416,13 @@ class Orchestrator(object): return a Site model reflecting the effective design for the site """ - def get_described_site(self, design_id, site_name): - site_design = None - - if site_name is None: - raise errors.OrchestratorError("Cannot source design for site None") - + def get_described_site(self, design_id): site_design = self.state_manager.get_design(design_id) return site_design - def get_effective_site(self, design_id, site_name): - site_design = self.get_described_site(design_id, site_name) + def get_effective_site(self, design_id): + site_design = self.get_described_site(design_id) self.compute_model_inheritance(site_design) diff --git a/helm_drydock/orchestrator/readme.md b/drydock_provisioner/orchestrator/readme.md similarity index 100% rename from helm_drydock/orchestrator/readme.md rename to drydock_provisioner/orchestrator/readme.md diff --git a/helm_drydock/statemgmt/__init__.py b/drydock_provisioner/statemgmt/__init__.py similarity index 97% rename from helm_drydock/statemgmt/__init__.py rename to drydock_provisioner/statemgmt/__init__.py index 22112c29..a6be0d0c 100644 --- a/helm_drydock/statemgmt/__init__.py +++ b/drydock_provisioner/statemgmt/__init__.py @@ -18,10 +18,10 @@ from threading import Lock import uuid -import helm_drydock.objects as objects -import helm_drydock.objects.task as tasks +import drydock_provisioner.objects as objects +import drydock_provisioner.objects.task as tasks -from helm_drydock.error import DesignError, StateError +from drydock_provisioner.error import DesignError, StateError class DesignState(object): @@ -41,6 +41,7 @@ class DesignState(object): # has started def get_design(self, design_id): if design_id not in self.designs.keys(): + raise DesignError("Design ID %s not found" % (design_id)) return objects.SiteDesign.obj_from_primitive(self.designs[design_id]) @@ -133,7 +134,7 @@ class DesignState(object): def get_task(self, task_id): for t in self.tasks: - if t.get_id() == task_id: + if t.get_id() == task_id or str(t.get_id()) == task_id: return deepcopy(t) return None diff --git a/helm_drydock/statemgmt/readme.md b/drydock_provisioner/statemgmt/readme.md similarity index 100% rename from helm_drydock/statemgmt/readme.md rename to drydock_provisioner/statemgmt/readme.md diff --git a/examples/designparts_v1.0.yaml b/examples/designparts_v1.0.yaml index 530efbc0..1f669fab 100644 --- a/examples/designparts_v1.0.yaml +++ b/examples/designparts_v1.0.yaml @@ -153,7 +153,7 @@ metadata: description: Describe server configuration attributes. Not a specific server, but profile adopted by a server definition spec: # The HostProfile this profile adopts initial state from. No default. - # See helm_drydock/objects/readme.md for information on how HostProfile and BaremetalNode inheritance works + # See drydock_provisioner/objects/readme.md for information on how HostProfile and BaremetalNode inheritance works host_profile: 'defaults' # The HardwareProfile describing the node hardware. No default. hardware_profile: 'DellR720v1' @@ -242,7 +242,7 @@ metadata: description: Specify a physical server. spec: # The HostProfile this server adopts initial state from. No default. - # See helm_drydock/objects/readme.md for information on how HostProfile and BaremetalNode inheritance works + # See drydock_provisioner/objects/readme.md for information on how HostProfile and BaremetalNode inheritance works host_profile: 'defaults' # The HardwareProfile describing the node hardware. No default. hardware_profile: 'DellR720v1' diff --git a/helm_drydock/control/readme.md b/helm_drydock/control/readme.md deleted file mode 100644 index 2dad5b24..00000000 --- a/helm_drydock/control/readme.md +++ /dev/null @@ -1,14 +0,0 @@ -# Control # - -This is the external facing API service to control the rest -of Drydock and query Drydock-managed data. - -Anticipate basing this service on the falcon Python library - -## Endpoints ## - -### /tasks ### - -POST - Create a new orchestration task and submit it for execution -GET - Get status of a task -DELETE - Cancel execution of a task if permitted diff --git a/helm_drydock/drivers/node/maasdriver/driver.py b/helm_drydock/drivers/node/maasdriver/driver.py deleted file mode 100644 index 83406a18..00000000 --- a/helm_drydock/drivers/node/maasdriver/driver.py +++ /dev/null @@ -1,306 +0,0 @@ -# Copyright 2017 AT&T Intellectual Property. All other rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import helm_drydock.error as errors -import helm_drydock.config as config -import helm_drydock.drivers as drivers -import helm_drydock.objects.fields as hd_fields -import helm_drydock.objects.task as task_model - -from helm_drydock.drivers.node import NodeDriver -from .api_client import MaasRequestFactory -import helm_drydock.drivers.node.maasdriver.models.fabric as maas_fabric -import helm_drydock.drivers.node.maasdriver.models.vlan as maas_vlan -import helm_drydock.drivers.node.maasdriver.models.subnet as maas_subnet - -class MaasNodeDriver(NodeDriver): - - def __init__(self, **kwargs): - super(MaasNodeDriver, self).__init__(**kwargs) - - self.driver_name = "maasdriver" - self.driver_key = "maasdriver" - self.driver_desc = "MaaS Node Provisioning Driver" - - self.config = config.DrydockConfig.node_driver[self.driver_key] - - def execute_task(self, task_id): - task = self.state_manager.get_task(task_id) - - if task is None: - raise errors.DriverError("Invalid task %s" % (task_id)) - - if task.action not in self.supported_actions: - raise errors.DriverError("Driver %s doesn't support task action %s" - % (self.driver_desc, task.action)) - - if task.action == hd_fields.OrchestratorAction.ValidateNodeServices: - self.orchestrator.task_field_update(task.get_id(), - status=hd_fields.TaskStatus.Running) - maas_client = MaasRequestFactory(self.config['api_url'], self.config['api_key']) - - try: - if maas_client.test_connectivity(): - if maas_client.test_authentication(): - self.orchestrator.task_field_update(task.get_id(), - status=hd_fields.TaskStatus.Complete, - result=hd_fields.ActionResult.Success) - return - except errors.TransientDriverError(ex): - result = { - 'retry': True, - 'detail': str(ex), - } - self.orchestrator.task_field_update(task.get_id(), - status=hd_fields.TaskStatus.Complete, - result=hd_fields.ActionResult.Failure, - result_details=result) - return - except errors.PersistentDriverError(ex): - result = { - 'retry': False, - 'detail': str(ex), - } - self.orchestrator.task_field_update(task.get_id(), - status=hd_fields.TaskStatus.Complete, - result=hd_fields.ActionResult.Failure, - result_details=result) - return - except Exception(ex): - result = { - 'retry': False, - 'detail': str(ex), - } - self.orchestrator.task_field_update(task.get_id(), - status=hd_fields.TaskStatus.Complete, - result=hd_fields.ActionResult.Failure, - result_details=result) - return - - design_id = getattr(task, 'design_id', None) - - if design_id is None: - raise errors.DriverError("No design ID specified in task %s" % - (task_id)) - - - if task.site_name is None: - raise errors.DriverError("No site specified for task %s." % - (task_id)) - - self.orchestrator.task_field_update(task.get_id(), - status=hd_fields.TaskStatus.Running) - - site_design = self.orchestrator.get_effective_site(design_id, task.site_name) - - if task.action == hd_fields.OrchestratorAction.CreateNetworkTemplate: - subtask = self.orchestrator.create_task(task_model.DriverTask, - parent_task_id=task.get_id(), design_id=design_id, - action=task.action, site_name=task.site_name, - task_scope={'site': task.site_name}) - runner = MaasTaskRunner(state_manager=self.state_manager, - orchestrator=self.orchestrator, - task_id=subtask.get_id(),config=self.config) - runner.start() - - runner.join(timeout=120) - - if runner.is_alive(): - result = { - 'retry': False, - 'detail': 'MaaS Network creation timed-out' - } - self.orchestrator.task_field_update(task.get_id(), - status=hd_fields.TaskStatus.Complete, - result=hd_fields.ActionResult.Failure, - result_detail=result) - else: - subtask = self.state_manager.get_task(subtask.get_id()) - self.orchestrator.task_field_update(task.get_id(), - status=hd_fields.TaskStatus.Complete, - result=subtask.get_result()) - - return - -class MaasTaskRunner(drivers.DriverTaskRunner): - - def __init__(self, config=None, **kwargs): - super(MaasTaskRunner, self).__init__(**kwargs) - - self.driver_config = config - - def execute_task(self): - task_action = self.task.action - - self.orchestrator.task_field_update(self.task.get_id(), - status=hd_fields.TaskStatus.Running, - result=hd_fields.ActionResult.Incomplete) - - self.maas_client = MaasRequestFactory(self.driver_config['api_url'], - self.driver_config['api_key']) - - site_design = self.orchestrator.get_effective_site(self.task.design_id, - self.task.site_name) - - if task_action == hd_fields.OrchestratorAction.CreateNetworkTemplate: - # Try to true up MaaS definitions of fabrics/vlans/subnets - # with the networks defined in Drydock - design_networks = site_design.networks - - subnets = maas_subnet.Subnets(self.maas_client) - subnets.refresh() - - result_detail = { - 'detail': [] - } - - for n in design_networks: - exists = subnets.query({'cidr': n.cidr}) - - subnet = None - - if len(exists) > 0: - subnet = exists[0] - - subnet.name = n.name - subnet.dns_servers = n.dns_servers - - vlan_list = maas_vlan.Vlans(self.maas_client, fabric_id=subnet.fabric) - vlan_list.refresh() - - vlan = vlan_list.select(subnet.vlan) - - if vlan is not None: - if ((n.vlan_id is None and vlan.vid != 0) or - (n.vlan_id is not None and vlan.vid != n.vlan_id)): - - # if the VLAN name matches, assume this is the correct resource - # and it needs to be updated - if vlan.name == n.name: - vlan.set_vid(n.vlan_id) - vlan.mtu = n.mtu - vlan.update() - else: - vlan_id = n.vlan_id if n.vlan_id is not None else 0 - target_vlan = vlan_list.query({'vid': vlan_id}) - if len(target_vlan) > 0: - subnet.vlan = target_vlan[0].resource_id - else: - # This is a flag that after creating a fabric and - # VLAN below, update the subnet - subnet.vlan = None - else: - subnet.vlan = None - - # Check if the routes have a default route - subnet.gateway_ip = n.get_default_gateway() - - - result_detail['detail'].append("Subnet %s found for network %s, updated attributes" - % (exists[0].resource_id, n.name)) - - # Need to create a Fabric/Vlan for this network - if (subnet is None or (subnet is not None and subnet.vlan is None)): - fabric_list = maas_fabric.Fabrics(self.maas_client) - fabric_list.refresh() - matching_fabrics = fabric_list.query({'name': n.name}) - - fabric = None - vlan = None - - if len(matching_fabrics) > 0: - # Fabric exists, update VLAN - fabric = matching_fabrics[0] - - vlan_list = maas_vlan.Vlans(self.maas_client, fabric_id=fabric.resource_id) - vlan_list.refresh() - vlan_id = n.vlan_id if n.vlan_id is not None else 0 - matching_vlans = vlan_list.query({'vid': vlan_id}) - - if len(matching_vlans) > 0: - vlan = matching_vlans[0] - - vlan.name = n.name - if getattr(n, 'mtu', None) is not None: - vlan.mtu = n.mtu - - if subnet is not None: - subnet.vlan = vlan.resource_id - subnet.update() - vlan.update() - else: - vlan = maas_vlan.Vlan(self.maas_client, name=n.name, vid=vlan_id, - mtu=getattr(n, 'mtu', None),fabric_id=fabric.resource_id) - vlan = vlan_list.add(vlan) - - if subnet is not None: - subnet.vlan = vlan.resource_id - subnet.update() - - else: - new_fabric = maas_fabric.Fabric(self.maas_client, name=n.name) - new_fabric = fabric_list.add(new_fabric) - new_fabric.refresh() - fabric = new_fabric - - vlan_list = maas_vlan.Vlans(self.maas_client, fabric_id=new_fabric.resource_id) - vlan_list.refresh() - vlan = vlan_list.single() - - vlan.name = n.name - vlan.vid = n.vlan_id if n.vlan_id is not None else 0 - if getattr(n, 'mtu', None) is not None: - vlan.mtu = n.mtu - - vlan.update() - - if subnet is not None: - subnet.vlan = vlan.resource_id - subnet.update() - - if subnet is None: - subnet = maas_subnet.Subnet(self.maas_client, name=n.name, cidr=n.cidr, fabric=fabric.resource_id, - vlan=vlan.resource_id, gateway_ip=n.get_default_gateway()) - - subnet_list = maas_subnet.Subnets(self.maas_client) - subnet = subnet_list.add(subnet) - - subnet_list = maas_subnet.Subnets(self.maas_client) - subnet_list.refresh() - - action_result = hd_fields.ActionResult.Incomplete - - success_rate = 0 - - for n in design_networks: - exists = subnet_list.query({'cidr': n.cidr}) - if len(exists) > 0: - subnet = exists[0] - if subnet.name == n.name: - success_rate = success_rate + 1 - else: - success_rate = success_rate + 1 - else: - success_rate = success_rate + 1 - - if success_rate == len(design_networks): - action_result = hd_fields.ActionResult.Success - elif success_rate == - (len(design_networks)): - action_result = hd_fields.ActionResult.Failure - else: - action_result = hd_fields.ActionResult.PartialSuccess - - self.orchestrator.task_field_update(self.task.get_id(), - status=hd_fields.TaskStatus.Complete, - result=action_result, - result_detail=result_detail) \ No newline at end of file diff --git a/helm_drydock/ingester/__init__.py b/helm_drydock/ingester/__init__.py deleted file mode 100644 index b2b69072..00000000 --- a/helm_drydock/ingester/__init__.py +++ /dev/null @@ -1,107 +0,0 @@ -# Copyright 2017 AT&T Intellectual Property. All other rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# ingester - Ingest host topologies to define site design and -# persist design to helm-drydock's statemgmt service - -import logging -import yaml -import uuid - -import helm_drydock.objects as objects -import helm_drydock.objects.site as site -import helm_drydock.objects.network as network -import helm_drydock.objects.hwprofile as hwprofile -import helm_drydock.objects.node as node -import helm_drydock.objects.hostprofile as hostprofile - -from helm_drydock.statemgmt import DesignState - -class Ingester(object): - - def __init__(self): - logging.basicConfig(format="%(asctime)-15s [%(levelname)] %(module)s %(process)d %(message)s") - self.log = logging.Logger("ingester") - self.registered_plugins = {} - - def enable_plugins(self, plugins=[]): - if len(plugins) == 0: - self.log.error("Cannot have an empty plugin list.") - - for plugin in plugins: - try: - new_plugin = plugin() - plugin_name = new_plugin.get_name() - self.registered_plugins[plugin_name] = new_plugin - except: - self.log.error("Could not enable plugin %s" % (plugin.__name__)) - - if len(self.registered_plugins) == 0: - self.log.error("Could not enable at least one plugin") - raise Exception("Could not enable at least one plugin") - """ - enable_plugins - - params: plugins - A list of class objects denoting the ingester plugins to be enabled - - Enable plugins that can be used for ingest_data calls. Each plugin should use - helm_drydock.ingester.plugins.IngesterPlugin as its base class. As long as one - enabled plugin successfully initializes, the call is considered successful. Otherwise - it will throw an exception - """ - - def ingest_data(self, plugin_name='', design_state=None, **kwargs): - if design_state is None: - self.log.error("ingest_data called without valid DesignState handler") - raise Exception("Invalid design_state handler") - - design_data = None - - # If no design_id is specified, instantiate a new one - if 'design_id' not in kwargs.keys(): - design_id = str(uuid.uuid4()) - design_data = objects.SiteDesign(id=design_id) - design_state.post_design(design_data) - else: - design_id = kwargs.get('design_id') - design_data = design_state.get_design(design_id) - - if plugin_name in self.registered_plugins: - design_items = self.registered_plugins[plugin_name].ingest_data(**kwargs) - for m in design_items: - if type(m) is site.Site: - design_data.set_site(m) - elif type(m) is network.Network: - design_data.add_network(m) - elif type(m) is network.NetworkLink: - design_data.add_network_link(m) - elif type(m) is hostprofile.HostProfile: - design_data.add_host_profile(m) - elif type(m) is hwprofile.HardwareProfile: - design_data.add_hardware_profile(m) - elif type(m) is node.BaremetalNode: - design_data.add_baremetal_node(m) - design_state.put_design(design_data) - else: - self.log.error("Could not find plugin %s to ingest data." % (plugin_name)) - raise LookupError("Could not find plugin %s" % plugin_name) - """ - ingest_data - - params: plugin_name - Which plugin should be used for ingestion - params: params - A map of parameters that will be passed to the plugin's ingest_data method - - Execute a data ingestion using the named plugin (assuming it is enabled) - """ - diff --git a/setup.py b/setup.py index 01bbff5f..b090c19c 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# helm_drydock - A tool to consume a host topology and orchestrate +# drydock_provisioner - A tool to consume a host topology and orchestrate # and monitor the provisioning of those hosts and execution of bootstrap # scripts # @@ -32,26 +32,27 @@ from setuptools import setup -setup(name='helm_drydock', +setup(name='drydock_provisioner', version='0.1a1', description='Bootstrapper for Kubernetes infrastructure', url='http://github.com/att-comdev/drydock', author='Scott Hussey - AT&T', author_email='sh8121@att.com', license='Apache 2.0', - packages=['helm_drydock', - 'helm_drydock.objects', - 'helm_drydock.ingester', - 'helm_drydock.ingester.plugins', - 'helm_drydock.statemgmt', - 'helm_drydock.orchestrator', - 'helm_drydock.control', - 'helm_drydock.drivers', - 'helm_drydock.drivers.oob', - 'helm_drydock.drivers.oob.pyghmi_driver', - 'helm_drydock.drivers.node', - 'helm_drydock.drivers.node.maasdriver', - 'helm_drydock.drivers.node.maasdriver.models'], + packages=['drydock_provisioner', + 'drydock_provisioner.objects', + 'drydock_provisioner.ingester', + 'drydock_provisioner.ingester.plugins', + 'drydock_provisioner.statemgmt', + 'drydock_provisioner.orchestrator', + 'drydock_provisioner.control', + 'drydock_provisioner.drivers', + 'drydock_provisioner.drivers.oob', + 'drydock_provisioner.drivers.oob.pyghmi_driver', + 'drydock_provisioner.drivers.node', + 'drydock_provisioner.drivers.node.maasdriver', + 'drydock_provisioner.drivers.node.maasdriver.models', + 'drydock_provisioner.control'], install_requires=[ 'PyYAML', 'pyghmi>=1.0.18', @@ -60,6 +61,8 @@ setup(name='helm_drydock', 'oslo.versionedobjects>=1.23.0', 'requests', 'oauthlib', + 'uwsgi>1.4', + 'bson===0.4.7' ] ) diff --git a/tests/integration/test_maasdriver_client.py b/tests/integration/test_maasdriver_client.py index 88b86e95..3c174018 100644 --- a/tests/integration/test_maasdriver_client.py +++ b/tests/integration/test_maasdriver_client.py @@ -13,8 +13,8 @@ # limitations under the License. import json -import helm_drydock.config as config -import helm_drydock.drivers.node.maasdriver.api_client as client +import drydock_provisioner.config as config +import drydock_provisioner.drivers.node.maasdriver.api_client as client class TestClass(object): diff --git a/tests/integration/test_maasdriver_network.py b/tests/integration/test_maasdriver_network.py index 36c8b324..ba5f5114 100644 --- a/tests/integration/test_maasdriver_network.py +++ b/tests/integration/test_maasdriver_network.py @@ -14,10 +14,10 @@ import json import uuid -import helm_drydock.config as config -import helm_drydock.drivers.node.maasdriver.api_client as client -import helm_drydock.drivers.node.maasdriver.models.fabric as maas_fabric -import helm_drydock.drivers.node.maasdriver.models.subnet as maas_subnet +import drydock_provisioner.config as config +import drydock_provisioner.drivers.node.maasdriver.api_client as client +import drydock_provisioner.drivers.node.maasdriver.models.fabric as maas_fabric +import drydock_provisioner.drivers.node.maasdriver.models.subnet as maas_subnet class TestClass(object): diff --git a/tests/integration/test_orch_node_networks.py b/tests/integration/test_orch_node_networks.py index 85619a30..15c2f811 100644 --- a/tests/integration/test_orch_node_networks.py +++ b/tests/integration/test_orch_node_networks.py @@ -17,23 +17,23 @@ import shutil import os import uuid -import helm_drydock.config as config -import helm_drydock.drivers.node.maasdriver.api_client as client -import helm_drydock.ingester.plugins.yaml -import helm_drydock.statemgmt as statemgmt -import helm_drydock.objects as objects -import helm_drydock.orchestrator as orch -import helm_drydock.objects.fields as hd_fields -import helm_drydock.objects.task as task -import helm_drydock.drivers as drivers -from helm_drydock.ingester import Ingester +import drydock_provisioner.config as config +import drydock_provisioner.drivers.node.maasdriver.api_client as client +import drydock_provisioner.ingester.plugins.yaml +import drydock_provisioner.statemgmt as statemgmt +import drydock_provisioner.objects as objects +import drydock_provisioner.orchestrator as orch +import drydock_provisioner.objects.fields as hd_fields +import drydock_provisioner.objects.task as task +import drydock_provisioner.drivers as drivers +from drydock_provisioner.ingester import Ingester class TestClass(object): def test_client_verify(self): design_state = statemgmt.DesignState() orchestrator = orch.Orchestrator(state_manager=design_state, - enabled_drivers={'node': 'helm_drydock.drivers.node.maasdriver.driver.MaasNodeDriver'}) + enabled_drivers={'node': 'drydock_provisioner.drivers.node.maasdriver.driver.MaasNodeDriver'}) orch_task = orchestrator.create_task(task.OrchestratorTask, site='sitename', @@ -57,14 +57,14 @@ class TestClass(object): design_state.post_design(design_data) ingester = Ingester() - ingester.enable_plugins([helm_drydock.ingester.plugins.yaml.YamlIngester]) + ingester.enable_plugins([drydock_provisioner.ingester.plugins.yaml.YamlIngester]) ingester.ingest_data(plugin_name='yaml', design_state=design_state, filenames=[str(input_file)], design_id=design_id) design_data = design_state.get_design(design_id) orchestrator = orch.Orchestrator(state_manager=design_state, - enabled_drivers={'node': 'helm_drydock.drivers.node.maasdriver.driver.MaasNodeDriver'}) + enabled_drivers={'node': 'drydock_provisioner.drivers.node.maasdriver.driver.MaasNodeDriver'}) orch_task = orchestrator.create_task(task.OrchestratorTask, site='sitename', diff --git a/tests/unit/test_design_inheritance.py b/tests/unit/test_design_inheritance.py index e5c57ce8..dad63a0f 100644 --- a/tests/unit/test_design_inheritance.py +++ b/tests/unit/test_design_inheritance.py @@ -12,16 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -from helm_drydock.ingester import Ingester -from helm_drydock.statemgmt import DesignState -from helm_drydock.orchestrator import Orchestrator +from drydock_provisioner.ingester import Ingester +from drydock_provisioner.statemgmt import DesignState +from drydock_provisioner.orchestrator import Orchestrator from copy import deepcopy import pytest import shutil import os -import helm_drydock.ingester.plugins.yaml +import drydock_provisioner.ingester.plugins.yaml import yaml class TestClass(object): @@ -32,7 +32,7 @@ class TestClass(object): def test_design_inheritance(self, loaded_design): orchestrator = Orchestrator(state_manager=loaded_design, - enabled_drivers={'oob': 'helm_drydock.drivers.oob.pyghmi_driver.PyghmiDriver'}) + enabled_drivers={'oob': 'drydock_provisioner.drivers.oob.pyghmi_driver.PyghmiDriver'}) design_data = orchestrator.load_design_data("sitename") @@ -63,7 +63,7 @@ class TestClass(object): design_state.post_design_base(design_data) ingester = Ingester() - ingester.enable_plugins([helm_drydock.ingester.plugins.yaml.YamlIngester]) + ingester.enable_plugins([drydock_provisioner.ingester.plugins.yaml.YamlIngester]) ingester.ingest_data(plugin_name='yaml', design_state=design_state, filenames=[str(input_file)]) return design_state diff --git a/tests/unit/test_ingester.py b/tests/unit/test_ingester.py index a719ad6c..9b083322 100644 --- a/tests/unit/test_ingester.py +++ b/tests/unit/test_ingester.py @@ -12,14 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -from helm_drydock.ingester import Ingester -from helm_drydock.statemgmt import DesignState -import helm_drydock.objects as objects +from drydock_provisioner.ingester import Ingester +from drydock_provisioner.statemgmt import DesignState +import drydock_provisioner.objects as objects import pytest import shutil import os -import helm_drydock.ingester.plugins.yaml +import drydock_provisioner.ingester.plugins.yaml class TestClass(object): @@ -37,7 +37,7 @@ class TestClass(object): design_state.post_design(design_data) ingester = Ingester() - ingester.enable_plugins([helm_drydock.ingester.plugins.yaml.YamlIngester]) + ingester.enable_plugins([drydock_provisioner.ingester.plugins.yaml.YamlIngester]) ingester.ingest_data(plugin_name='yaml', design_state=design_state, filenames=[str(input_file)], design_id=design_id) @@ -59,7 +59,7 @@ class TestClass(object): design_state.post_design(design_data) ingester = Ingester() - ingester.enable_plugins([helm_drydock.ingester.plugins.yaml.YamlIngester]) + ingester.enable_plugins([drydock_provisioner.ingester.plugins.yaml.YamlIngester]) ingester.ingest_data(plugin_name='yaml', design_state=design_state, design_id=design_id, filenames=[str(profiles_file), str(networks_file), str(nodes_file)]) diff --git a/tests/unit/test_ingester_yaml.py b/tests/unit/test_ingester_yaml.py index 3be5db08..c078401f 100644 --- a/tests/unit/test_ingester_yaml.py +++ b/tests/unit/test_ingester_yaml.py @@ -16,7 +16,7 @@ import shutil import os import uuid -from helm_drydock.ingester.plugins.yaml import YamlIngester +from drydock_provisioner.ingester.plugins.yaml import YamlIngester class TestClass(object): diff --git a/tests/unit/test_models.py b/tests/unit/test_models.py index e5014444..25c5128a 100644 --- a/tests/unit/test_models.py +++ b/tests/unit/test_models.py @@ -14,8 +14,8 @@ import pytest -import helm_drydock.objects as objects -from helm_drydock.objects import fields +import drydock_provisioner.objects as objects +from drydock_provisioner.objects import fields class TestClass(object): @@ -23,7 +23,7 @@ class TestClass(object): objects.register_all() model_attr = { - 'versioned_object.namespace': 'helm_drydock.objects', + 'versioned_object.namespace': 'drydock_provisioner.objects', 'versioned_object.name': 'HardwareProfile', 'versioned_object.version': '1.0', 'versioned_object.data': { @@ -38,13 +38,13 @@ class TestClass(object): 'bootstrap_protocol': 'pxe', 'pxe_interface': '0', 'devices': { - 'versioned_object.namespace': 'helm_drydock.objects', + 'versioned_object.namespace': 'drydock_provisioner.objects', 'versioned_object.name': 'HardwareDeviceAliasList', 'versioned_object.version': '1.0', 'versioned_object.data': { 'objects': [ { - 'versioned_object.namespace': 'helm_drydock.objects', + 'versioned_object.namespace': 'drydock_provisioner.objects', 'versioned_object.name': 'HardwareDeviceAlias', 'versioned_object.version': '1.0', 'versioned_object.data': { @@ -56,7 +56,7 @@ class TestClass(object): } }, { - 'versioned_object.namespace': 'helm_drydock.objects', + 'versioned_object.namespace': 'drydock_provisioner.objects', 'versioned_object.name': 'HardwareDeviceAlias', 'versioned_object.version': '1.0', 'versioned_object.data': { diff --git a/tests/unit/test_orch_generic.py b/tests/unit/test_orch_generic.py index 86739965..82940686 100644 --- a/tests/unit/test_orch_generic.py +++ b/tests/unit/test_orch_generic.py @@ -18,11 +18,11 @@ import threading import time -import helm_drydock.orchestrator as orch -import helm_drydock.objects.fields as hd_fields -import helm_drydock.statemgmt as statemgmt -import helm_drydock.objects.task as task -import helm_drydock.drivers as drivers +import drydock_provisioner.orchestrator as orch +import drydock_provisioner.objects.fields as hd_fields +import drydock_provisioner.statemgmt as statemgmt +import drydock_provisioner.objects.task as task +import drydock_provisioner.drivers as drivers class TestClass(object): diff --git a/tests/unit/test_orch_oob.py b/tests/unit/test_orch_oob.py index 6c10d8f1..b510b935 100644 --- a/tests/unit/test_orch_oob.py +++ b/tests/unit/test_orch_oob.py @@ -23,15 +23,15 @@ import os import shutil import uuid -from helm_drydock.ingester import Ingester +from drydock_provisioner.ingester import Ingester -import helm_drydock.orchestrator as orch -import helm_drydock.objects.fields as hd_fields -import helm_drydock.statemgmt as statemgmt -import helm_drydock.objects as objects -import helm_drydock.objects.task as task -import helm_drydock.drivers as drivers -import helm_drydock.ingester.plugins.yaml as yaml_ingester +import drydock_provisioner.orchestrator as orch +import drydock_provisioner.objects.fields as hd_fields +import drydock_provisioner.statemgmt as statemgmt +import drydock_provisioner.objects as objects +import drydock_provisioner.objects.task as task +import drydock_provisioner.drivers as drivers +import drydock_provisioner.ingester.plugins.yaml as yaml_ingester class TestClass(object): @@ -44,7 +44,7 @@ class TestClass(object): #mocker.patch.object('pyghmi.ipmi.command.Command','get_asset_tag') orchestrator = orch.Orchestrator(state_manager=loaded_design, - enabled_drivers={'oob': 'helm_drydock.drivers.oob.pyghmi_driver.PyghmiDriver'}) + enabled_drivers={'oob': 'drydock_provisioner.drivers.oob.pyghmi_driver.PyghmiDriver'}) orch_task = orchestrator.create_task(task.OrchestratorTask, site='sitename', @@ -63,7 +63,7 @@ class TestClass(object): #mocker.patch.object('pyghmi.ipmi.command.Command','set_bootdev') orchestrator = orch.Orchestrator(state_manager=loaded_design, - enabled_drivers={'oob': 'helm_drydock.drivers.oob.pyghmi_driver.PyghmiDriver'}) + enabled_drivers={'oob': 'drydock_provisioner.drivers.oob.pyghmi_driver.PyghmiDriver'}) orch_task = orchestrator.create_task(task.OrchestratorTask, site='sitename', diff --git a/tests/unit/test_statemgmt.py b/tests/unit/test_statemgmt.py index edb82764..17c90899 100644 --- a/tests/unit/test_statemgmt.py +++ b/tests/unit/test_statemgmt.py @@ -15,8 +15,8 @@ import pytest import shutil -import helm_drydock.objects as objects -import helm_drydock.statemgmt as statemgmt +import drydock_provisioner.objects as objects +import drydock_provisioner.statemgmt as statemgmt class TestClass(object):