From 839c83651f780bd47adc63f2164d6049c8da32c5 Mon Sep 17 00:00:00 2001 From: Scott Hussey Date: Wed, 15 Mar 2017 14:53:46 -0500 Subject: [PATCH] Last commit to push changes to master. Will use feature branches going forward Added readmes to submodules Refactored model into multiple files Standardized on 'node' for physical server Pinned requirements.txt to versions Began orchestration action outline in conjunction with DAG --- README.md | 10 +- helm_drydock/config.py | 22 +- helm_drydock/control/readme.md | 6 + helm_drydock/drivers/__init__.py | 2 + .../root.py => drivers/node/__init__.py} | 28 +- .../{server => node}/maasdriver/__init__.py | 7 +- helm_drydock/drivers/oob/__init__.py | 42 ++ helm_drydock/drivers/readme.md | 26 + .../{drivers/server/__init__.py => error.py} | 6 +- helm_drydock/ingester/__init__.py | 88 +-- helm_drydock/ingester/plugins/yaml.py | 19 +- helm_drydock/ingester/readme.md | 13 + helm_drydock/introspection/readme.md | 9 + helm_drydock/model/__init__.py | 620 ------------------ helm_drydock/model/hostprofile.py | 378 +++++++++++ helm_drydock/model/hwprofile.py | 95 +++ helm_drydock/model/network.py | 133 ++++ helm_drydock/model/node.py | 154 +++++ helm_drydock/model/site.py | 122 ++++ helm_drydock/orchestrator/__init__.py | 60 +- helm_drydock/orchestrator/designdata.py | 42 +- helm_drydock/orchestrator/enum.py | 64 ++ helm_drydock/orchestrator/readme.md | 83 +++ helm_drydock/statemgmt/__init__.py | 212 +++++- helm_drydock/statemgmt/readme.md | 41 ++ helm_drydock/tox.ini | 11 - requirements.txt | 12 +- setup.py | 2 +- tests/test_design_inheritance.py | 15 +- tests/test_ingester.py | 17 +- tests/test_models.py | 2 +- tests/yaml_samples/fullsite.yaml | 5 +- tests/yaml_samples/fullsite_networks.yaml | 2 +- tests/yaml_samples/fullsite_profiles.yaml | 5 +- 34 files changed, 1584 insertions(+), 769 deletions(-) create mode 100644 helm_drydock/control/readme.md rename helm_drydock/{control/root.py => drivers/node/__init__.py} (51%) rename helm_drydock/drivers/{server => node}/maasdriver/__init__.py (78%) create mode 100644 helm_drydock/drivers/oob/__init__.py create mode 100644 helm_drydock/drivers/readme.md rename helm_drydock/{drivers/server/__init__.py => error.py} (88%) create mode 100644 helm_drydock/ingester/readme.md create mode 100644 helm_drydock/introspection/readme.md create mode 100644 helm_drydock/model/hostprofile.py create mode 100644 helm_drydock/model/hwprofile.py create mode 100644 helm_drydock/model/network.py create mode 100644 helm_drydock/model/node.py create mode 100644 helm_drydock/model/site.py create mode 100644 helm_drydock/orchestrator/enum.py create mode 100644 helm_drydock/orchestrator/readme.md create mode 100644 helm_drydock/statemgmt/readme.md delete mode 100644 helm_drydock/tox.ini diff --git a/README.md b/README.md index b97df8b9..dc619477 100644 --- a/README.md +++ b/README.md @@ -32,9 +32,15 @@ aka orchestrator Handle validation of complete design, ordering and managing downstream API calls for hardware provisioning/bootstrapping -### Server Driver ### +### OOB Driver ### -aka maasdriver +Pluggable provider for server OOB (ILO) management + +aka driver/oob + +### Node Driver ### + +aka driver/node Pluggable provisioner for server bootstrapping. Initial implementation is MaaS client. diff --git a/helm_drydock/config.py b/helm_drydock/config.py index 8bd363bc..92b3d8ac 100644 --- a/helm_drydock/config.py +++ b/helm_drydock/config.py @@ -21,7 +21,7 @@ class DrydockConfig(object): - def __init__(self): + def __init__(self): self.server_driver_config = { selected_driver = helm_drydock.drivers.server.maasdriver, params = { @@ -29,13 +29,13 @@ class DrydockConfig(object): maas_api_url = "" } } - self.selected_network_driver = helm_drydock.drivers.network.noopdriver - self.control_config = {} - self.ingester_config = { - plugins = [helm_drydock.ingester.plugins.aicyaml.AicYamlIngester] - } - self.introspection_config = {} - self.orchestrator_config = {} - self.statemgmt_config = { - backend_driver = helm_drydock.drivers.statemgmt.etcd, - } + self.selected_network_driver = helm_drydock.drivers.network.noopdriver + self.control_config = {} + self.ingester_config = { + plugins = [helm_drydock.ingester.plugins.aicyaml.AicYamlIngester] + } + self.introspection_config = {} + self.orchestrator_config = {} + self.statemgmt_config = { + backend_driver = helm_drydock.drivers.statemgmt.etcd, + } diff --git a/helm_drydock/control/readme.md b/helm_drydock/control/readme.md new file mode 100644 index 00000000..f3c602fb --- /dev/null +++ b/helm_drydock/control/readme.md @@ -0,0 +1,6 @@ +# Control # + +This is the external facing API service to control the rest +of Drydock and query Drydock-managed data. + +Anticipate basing this service on the falcon Python library \ No newline at end of file diff --git a/helm_drydock/drivers/__init__.py b/helm_drydock/drivers/__init__.py index 5c73a1fa..a0e27676 100644 --- a/helm_drydock/drivers/__init__.py +++ b/helm_drydock/drivers/__init__.py @@ -17,3 +17,5 @@ class ProviderDriver(object): __init__(self): pass + +class DriverTask(object): \ No newline at end of file diff --git a/helm_drydock/control/root.py b/helm_drydock/drivers/node/__init__.py similarity index 51% rename from helm_drydock/control/root.py rename to helm_drydock/drivers/node/__init__.py index eb4bb8af..bae95f38 100644 --- a/helm_drydock/control/root.py +++ b/helm_drydock/drivers/node/__init__.py @@ -11,25 +11,19 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# -from pecan import expose -from webob.exc import status_map +from helm_drydock.drivers import ProviderDriver -class RootController(object): +class NodeDriver(ProviderDriver): - @expose(generic=True, template='index.html') - def index(self): - return dict() +class NodeAction(Enum): + PrepareNode = 'prepare_node' + ApplyNetworkConfig = 'apply_network_config' + ApplyStorageConfig = 'apply_storage_config' + InterrogateNode = 'interrogate_node' + DeployNode = 'deploy_node' - @index.when(method='POST') - def index_post(self, q): - redirect('https://pecan.readthedocs.io/en/latest/search.html?q=%s' % q) - @expose('error.html') - def error(self, status): - try: - status = int(status) - except ValueError: - status = 0 - message = getattr(status_map.get(status), 'explanation', '') - return dict(status=status, message=message) \ No newline at end of file + + \ No newline at end of file diff --git a/helm_drydock/drivers/server/maasdriver/__init__.py b/helm_drydock/drivers/node/maasdriver/__init__.py similarity index 78% rename from helm_drydock/drivers/server/maasdriver/__init__.py rename to helm_drydock/drivers/node/maasdriver/__init__.py index 21cbcd26..7c0c2b74 100644 --- a/helm_drydock/drivers/server/maasdriver/__init__.py +++ b/helm_drydock/drivers/node/maasdriver/__init__.py @@ -11,7 +11,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import helm_drydock.drivers.server.ServerDriver +from helm_drydock.drivers.node import NodeDriver -class MaasServerDriver(object): +class MaasNodeDriver(NodeDriver): + + def __init__(self, kwargs): + super(MaasNodeDriver, self).__init__(**kwargs) \ No newline at end of file diff --git a/helm_drydock/drivers/oob/__init__.py b/helm_drydock/drivers/oob/__init__.py new file mode 100644 index 00000000..9dd08ff8 --- /dev/null +++ b/helm_drydock/drivers/oob/__init__.py @@ -0,0 +1,42 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# OOB: +# sync_hardware_clock +# collect_chassis_sysinfo +# enable_netboot +# initiate_reboot +# set_power_off +# set_power_on + +from helm_drydock.drivers import ProviderDriver + +class OobDriver(ProviderDriver): + + def __init__(self): + pass + + def execute_action(self, action, **kwargs): + if action == + + + +class OobAction(Enum): + ConfigNodePxe = 'config_node_pxe' + SetNodeBoot = 'set_node_boot' + PowerOffNode = 'power_off_node' + PowerOnNode = 'power_on_node' + PowerCycleNode = 'power_cycle_node' + InterrogateNode = 'interrogate_node' + diff --git a/helm_drydock/drivers/readme.md b/helm_drydock/drivers/readme.md new file mode 100644 index 00000000..0a663a16 --- /dev/null +++ b/helm_drydock/drivers/readme.md @@ -0,0 +1,26 @@ +# Drivers # + +Drivers are downstream actors that Drydock will use to actually execute +orchestration actions. It is intended to be a pluggable architecture +so that various downstream automation can be used. + +## oob ## + +The oob drivers will interface with physical servers' out-of-band +management system (e.g. Dell iDRAC, HP iLO, etc...). OOB management +will be used for setting a system to use PXE boot and power cycling +servers. + +## node ## + +The node drivers will interface with an external bootstrapping system +for loading the base OS on a server and configuring hardware, network, +and storage. + +## network ## + +The network drivers will interface with switches for managing port +configuration to support the bootstrapping of physical nodes. This is not +intended to be a network provisioner, but instead is a support driver +for node bootstrapping where temporary changes to network configurations +are required. \ No newline at end of file diff --git a/helm_drydock/drivers/server/__init__.py b/helm_drydock/error.py similarity index 88% rename from helm_drydock/drivers/server/__init__.py rename to helm_drydock/error.py index febcdad1..9319eeca 100644 --- a/helm_drydock/drivers/server/__init__.py +++ b/helm_drydock/error.py @@ -12,7 +12,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -import helm_drydock.drivers.ProviderDriver - -class ServerDriver(ProviderDriver): - \ No newline at end of file +class DesignError(Exception): + pass diff --git a/helm_drydock/ingester/__init__.py b/helm_drydock/ingester/__init__.py index beae66a3..48553c2d 100644 --- a/helm_drydock/ingester/__init__.py +++ b/helm_drydock/ingester/__init__.py @@ -17,9 +17,14 @@ import logging import yaml -import helm_drydock.model as model -from helm_drydock.statemgmt import DesignState +import helm_drydock.model.site as site +import helm_drydock.model.network as network +import helm_drydock.model.hwprofile as hwprofile +import helm_drydock.model.node as node +import helm_drydock.model.hostprofile as hostprofile + +from helm_drydock.statemgmt import DesignState, SiteDesign, DesignError class Ingester(object): @@ -28,16 +33,6 @@ class Ingester(object): self.log = logging.Logger("ingester") self.registered_plugins = {} - """ - enable_plugins - - params: plugins - A list of class objects denoting the ingester plugins to be enabled - - Enable plugins that can be used for ingest_data calls. Each plugin should use - helm_drydock.ingester.plugins.IngesterPlugin as its base class. As long as one - enabled plugin successfully initializes, the call is considered successful. Otherwise - it will throw an exception - """ def enable_plugins(self, plugins=[]): if len(plugins) == 0: self.log.error("Cannot have an empty plugin list.") @@ -53,7 +48,51 @@ class Ingester(object): if len(self.registered_plugins) == 0: self.log.error("Could not enable at least one plugin") raise Exception("Could not enable at least one plugin") + """ + enable_plugins + params: plugins - A list of class objects denoting the ingester plugins to be enabled + + Enable plugins that can be used for ingest_data calls. Each plugin should use + helm_drydock.ingester.plugins.IngesterPlugin as its base class. As long as one + enabled plugin successfully initializes, the call is considered successful. Otherwise + it will throw an exception + """ + + def ingest_data(self, plugin_name='', design_state=None, **kwargs): + if design_state is None: + self.log.error("ingest_data called without valid DesignState handler") + raise Exception("Invalid design_state handler") + + # TODO this method needs refactored to handle design base vs change + + design_data = None + + try: + design_data = design_state.get_design_base() + except DesignError: + design_data = SiteDesign() + + if plugin_name in self.registered_plugins: + design_items = self.registered_plugins[plugin_name].ingest_data(**kwargs) + # Need to persist data here, but we don't yet have the statemgmt service working + for m in design_items: + if type(m) is site.Site: + design_data.add_site(m) + elif type(m) is network.Network: + design_data.add_network(m) + elif type(m) is network.NetworkLink: + design_data.add_network_link(m) + elif type(m) is hostprofile.HostProfile: + design_data.add_host_profile(m) + elif type(m) is hwprofile.HardwareProfile: + design_data.add_hardware_profile(m) + elif type(m) is node.BaremetalNode: + design_data.add_baremetal_node(m) + design_state.put_design_base(design_data) + else: + self.log.error("Could not find plugin %s to ingest data." % (plugin_name)) + raise LookupError("Could not find plugin %s" % plugin_name) """ ingest_data @@ -62,29 +101,4 @@ class Ingester(object): Execute a data ingestion using the named plugin (assuming it is enabled) """ - def ingest_data(self, plugin_name='', design_state=None, **kwargs): - if design_state is None: - self.log.error("ingest_data called without valid DesignState handler") - raise Exception("Invalid design_state handler") - - if plugin_name in self.registered_plugins: - design_data = self.registered_plugins[plugin_name].ingest_data(**kwargs) - # Need to persist data here, but we don't yet have the statemgmt service working - for m in design_data: - if type(m) is model.Site: - design_state.add_site(m) - elif type(m) is model.Network: - design_state.add_network(m) - elif type(m) is model.NetworkLink: - design_state.add_network_link(m) - elif type(m) is model.HostProfile: - design_state.add_host_profile(m) - elif type(m) is model.HardwareProfile: - design_state.add_hardware_profile(m) - elif type(m) is model.BaremetalNode: - design_state.add_baremetal_node(m) - else: - self.log.error("Could not find plugin %s to ingest data." % (plugin_name)) - raise LookupError("Could not find plugin %s" % plugin_name) - diff --git a/helm_drydock/ingester/plugins/yaml.py b/helm_drydock/ingester/plugins/yaml.py index 44664c8c..20a4d902 100644 --- a/helm_drydock/ingester/plugins/yaml.py +++ b/helm_drydock/ingester/plugins/yaml.py @@ -19,18 +19,23 @@ import yaml import logging -import helm_drydock.model as model +import helm_drydock.model.hwprofile as hwprofile +import helm_drydock.model.node as node +import helm_drydock.model.site as site +import helm_drydock.model.hostprofile as hostprofile +import helm_drydock.model.network as network + from helm_drydock.ingester.plugins import IngesterPlugin class YamlIngester(IngesterPlugin): kind_map = { - "Region": model.Site, - "NetworkLink": model.NetworkLink, - "HardwareProfile": model.HardwareProfile, - "Network": model.Network, - "HostProfile": model.HostProfile, - "BaremetalNode": model.BaremetalNode, + "Region": site.Site, + "NetworkLink": network.NetworkLink, + "HardwareProfile": hwprofile.HardwareProfile, + "Network": network.Network, + "HostProfile": hostprofile.HostProfile, + "BaremetalNode": node.BaremetalNode, } def __init__(self): diff --git a/helm_drydock/ingester/readme.md b/helm_drydock/ingester/readme.md new file mode 100644 index 00000000..1437c216 --- /dev/null +++ b/helm_drydock/ingester/readme.md @@ -0,0 +1,13 @@ +# Ingester # + +Ingester is a pluggable consumer of site design data. It +will support consuming data in different formats from +different sources. + +Each ingester plugin should be able source data +based on user-provided parameters and parse that data +into the Drydock internal model (helm_drydock.model). + +Each plugin does not need to support every type of design +data as a single site design may be federated from multiple +sources. \ No newline at end of file diff --git a/helm_drydock/introspection/readme.md b/helm_drydock/introspection/readme.md new file mode 100644 index 00000000..b62f79d5 --- /dev/null +++ b/helm_drydock/introspection/readme.md @@ -0,0 +1,9 @@ +# Introspection # + +Introspection is a cloud-init compatible metadata service +that is used to make a node self-aware. After a full +deployment by the node driver, the newly installed OS +will contact the introspection API to gain a package of +declaritive data defining the node's role in the site and +enough initial data to start the promenade process of +Kubernetes assimilation \ No newline at end of file diff --git a/helm_drydock/model/__init__.py b/helm_drydock/model/__init__.py index 05bb67d9..e682cefd 100644 --- a/helm_drydock/model/__init__.py +++ b/helm_drydock/model/__init__.py @@ -18,626 +18,6 @@ import logging from copy import deepcopy - -class HardwareProfile(object): - - def __init__(self, **kwargs): - self.log = logging.Logger('model') - - self.api_version = kwargs.get('apiVersion', '') - - if self.api_version == "v1.0": - metadata = kwargs.get('metadata', {}) - spec = kwargs.get('spec', {}) - - # Need to add validation logic, we'll assume the input is - # valid for now - self.name = metadata.get('name', '') - self.site = metadata.get('region', '') - - self.vendor = spec.get('vendor', None) - self.generation = spec.get('generation', None) - self.hw_version = spec.get('hw_version', None) - self.bios_version = spec.get('bios_version', None) - self.boot_mode = spec.get('boot_mode', None) - self.bootstrap_protocol = spec.get('bootstrap_protocol', None) - self.pxe_interface = spec.get('pxe_interface', None) - self.devices = [] - - device_aliases = spec.get('device_aliases', {}) - - pci_devices = device_aliases.get('pci', []) - scsi_devices = device_aliases.get('scsi', []) - - for d in pci_devices: - d['bus_type'] = 'pci' - self.devices.append( - HardwareDeviceAlias(self.api_version, **d)) - - for d in scsi_devices: - d['bus_type'] = 'scsi' - self.devices.append( - HardwareDeviceAlias(self.api_version, **d)) - else: - self.log.error("Unknown API version %s of %s" % - (self.api_version, self.__class__)) - raise ValueError('Unknown API version of object') - - return - - def resolve_alias(self, alias_type, alias): - selector = {} - for d in self.devices: - if d.alias == alias and d.bus_type == alias_type: - selector['address'] = d.address - selector['device_type'] = d.type - return selector - - return None - - -class HardwareDeviceAlias(object): - - def __init__(self, api_version, **kwargs): - self.log = logging.Logger('model') - - self.api_version = api_version - - if self.api_version == "v1.0": - self.bus_type = kwargs.get('bus_type', None) - self.address = kwargs.get('address', None) - self.alias = kwargs.get('alias', None) - self.type = kwargs.get('type', None) - else: - self.log.error("Unknown API version %s of %s" % - (self.api_version, self.__class__)) - raise ValueError('Unknown API version of object') - - -class Site(object): - - def __init__(self, **kwargs): - self.log = logging.Logger('model') - - self.api_version = kwargs.get('apiVersion', '') - - if self.api_version == "v1.0": - metadata = kwargs.get('metadata', {}) - - # Need to add validation logic, we'll assume the input is - # valid for now - self.name = metadata.get('name', '') - - self.networks = [] - self.network_links = [] - self.host_profiles = [] - self.hardware_profiles = [] - self.baremetal_nodes = [] - - else: - self.log.error("Unknown API version %s of %s" % - (self.api_version, self.__class__)) - raise ValueError('Unknown API version of object') - - def get_network(self, network_name): - for n in self.networks: - if n.name == network_name: - return n - - return None - - def get_network_link(self, link_name): - for l in self.network_links: - if l.name == link_name: - return l - - return None - - def get_host_profile(self, profile_name): - for p in self.host_profiles: - if p.name == profile_name: - return p - - return None - - def get_hardware_profile(self, profile_name): - for p in self.hardware_profiles: - if p.name == profile_name: - return p - - return None - - def get_baremetal_node(self, node_name): - for n in self.baremetal_nodes: - if n.name == node_name: - return n - - return None - -class NetworkLink(object): - - def __init__(self, **kwargs): - self.log = logging.Logger('model') - - self.api_version = kwargs.get('apiVersion', '') - - if self.api_version == "v1.0": - metadata = kwargs.get('metadata', {}) - spec = kwargs.get('spec', {}) - - self.name = metadata.get('name', '') - self.site = metadata.get('region', '') - - bonding = spec.get('bonding', {}) - self.bonding_mode = bonding.get('mode', 'none') - - # How should we define defaults for CIs not in the input? - if self.bonding_mode == '802.3ad': - self.bonding_xmit_hash = bonding.get('hash', 'layer3+4') - self.bonding_peer_rate = bonding.get('peer_rate', 'fast') - self.bonding_mon_rate = bonding.get('mon_rate', '100') - self.bonding_up_delay = bonding.get('up_delay', '200') - self.bonding_down_delay = bonding.get('down_delay', '200') - - self.mtu = spec.get('mtu', 1500) - self.linkspeed = spec.get('linkspeed', 'auto') - - trunking = spec.get('trunking', {}) - self.trunk_mode = trunking.get('mode', 'none') - - self.native_network = spec.get('default_network', '') - else: - self.log.error("Unknown API version %s of %s" % - (self.api_version, self.__class__)) - raise ValueError('Unknown API version of object') - - -class Network(object): - - def __init__(self, **kwargs): - self.log = logging.Logger('model') - - self.api_version = kwargs.get('apiVersion', '') - - if self.api_version == "v1.0": - metadata = kwargs.get('metadata', {}) - spec = kwargs.get('spec', {}) - - self.name = metadata.get('name', '') - self.site = metadata.get('region', '') - - self.cidr = spec.get('cidr', None) - self.allocation_strategy = spec.get('allocation', 'static') - self.vlan_id = spec.get('vlan_id', 1) - self.mtu = spec.get('mtu', 0) - - dns = spec.get('dns', {}) - self.dns_domain = dns.get('domain', 'local') - self.dns_servers = dns.get('servers', None) - - ranges = spec.get('ranges', []) - self.ranges = [] - - for r in ranges: - self.ranges.append(NetworkAddressRange(self.api_version, **r)) - - routes = spec.get('routes', []) - self.routes = [] - - for r in routes: - self.routes.append(NetworkRoute(self.api_version, **r)) - else: - self.log.error("Unknown API version %s of %s" % - (self.api_version, self.__class__)) - raise ValueError('Unknown API version of object') - - -class NetworkAddressRange(object): - - def __init__(self, api_version, **kwargs): - self.log = logging.Logger('model') - - self.api_version = api_version - - if self.api_version == "v1.0": - self.type = kwargs.get('type', None) - self.start = kwargs.get('start', None) - self.end = kwargs.get('end', None) - else: - self.log.error("Unknown API version %s of %s" % - (self.api_version, self.__class__)) - raise ValueError('Unknown API version of object') - - -class NetworkRoute(object): - - def __init__(self, api_version, **kwargs): - self.log = logging.Logger('model') - - self.api_version = api_version - - if self.api_version == "v1.0": - self.type = kwargs.get('subnet', None) - self.start = kwargs.get('gateway', None) - self.end = kwargs.get('metric', 100) - else: - self.log.error("Unknown API version %s of %s" % - (self.api_version, self.__class__)) - raise ValueError('Unknown API version of object') - - -class HostProfile(object): - - def __init__(self, **kwargs): - self.log = logging.Logger('model') - - self.api_version = kwargs.get('apiVersion', '') - - if self.api_version == "v1.0": - metadata = kwargs.get('metadata', {}) - spec = kwargs.get('spec', {}) - - self.name = metadata.get('name', '') - self.site = metadata.get('region', '') - - self.parent_profile = spec.get('host_profile', None) - self.hardware_profile = spec.get('hardware_profile', None) - - oob = spec.get('oob', {}) - self.oob_type = oob.get('type', None) - self.oob_network = oob.get('network', None) - self.oob_account = oob.get('account', None) - self.oob_credential = oob.get('credential', None) - - storage = spec.get('storage', {}) - self.storage_layout = storage.get('layout', 'lvm') - - bootdisk = storage.get('bootdisk', {}) - self.bootdisk_device = bootdisk.get('device', None) - self.bootdisk_root_size = bootdisk.get('root_size', None) - self.bootdisk_boot_size = bootdisk.get('boot_size', None) - - partitions = storage.get('partitions', []) - self.partitions = [] - - for p in partitions: - self.partitions.append(HostPartition(self.api_version, **p)) - - interfaces = spec.get('interfaces', []) - self.interfaces = [] - - for i in interfaces: - self.interfaces.append(HostInterface(self.api_version, **i)) - - metadata = spec.get('metadata', {}) - - metadata_tags = metadata.get('tags', []) - self.tags = [] - - for t in metadata_tags: - self.tags.append(t) - - owner_data = metadata.get('owner_data', {}) - self.owner_data = {} - - for k, v in owner_data.items(): - self.owner_data[k] = v - - self.rack = metadata.get('rack', None) - - else: - self.log.error("Unknown API version %s of %s" % - (self.api_version, self.__class__)) - raise ValueError('Unknown API version of object') - - def apply_inheritance(self, site): - # We return a deep copy of the profile so as not to corrupt - # the original model - self_copy = deepcopy(self) - - if self.parent_profile is None: - return self_copy - - parent = site.get_host_profile(self.parent_profile) - - if parent is None: - raise NameError("Cannot find parent profile %s for %s" - % (self.parent_profile, self.name)) - - parent = parent.apply_inheritance(site) - - # First compute inheritance for simple fields - inheritable_field_list = [ - "hardware_profile", "oob_type", "oob_network", - "oob_credential", "oob_account", "storage_layout", - "bootdisk_device", "bootdisk_root_size", "bootdisk_boot_size", - "rack"] - - for f in inheritable_field_list: - setattr(self_copy, f, - Utils.apply_field_inheritance(getattr(self, f, None), - getattr(parent, f, None))) - - # Now compute inheritance for complex types - self_copy.tags = Utils.merge_lists(self.tags, parent.tags) - - self_copy.owner_data = Utils.merge_dicts( - self.owner_data, parent.owner_data) - - self_copy.interfaces = HostInterface.merge_lists( - self.interfaces, parent.interfaces) - - self_copy.partitions = HostPartition.merge_lists( - self.partitions, parent.partitions) - - return self_copy - - -class HostInterface(object): - - def __init__(self, api_version, **kwargs): - self.log = logging.Logger('model') - - self.api_version = api_version - - if self.api_version == "v1.0": - self.device_name = kwargs.get('device_name', None) - self.network_link = kwargs.get('device_link', None) - - self.hardware_slaves = [] - slaves = kwargs.get('slaves', []) - - for s in slaves: - self.hardware_slaves.append(s) - - self.networks = [] - networks = kwargs.get('networks', []) - - for n in networks: - self.networks.append(n) - else: - self.log.error("Unknown API version %s of %s" % - (self.api_version, self.__class__)) - raise ValueError('Unknown API version of object') - - # The device attribute may be hardware alias that translates to a - # physical device address. If the device attribute does not match an - # alias, we assume it directly identifies a OS device name. When the - # apply_hardware_profile method is called on the parent Node of this - # device, the selector will be decided and applied - - def add_selector(self, sel_type, address='', dev_type=''): - if getattr(self, 'selectors', None) is None: - self.selectors = [] - - new_selector = {} - new_selector['selector_type'] = sel_type - new_selector['address'] = address - new_selector['device_type'] = dev_type - - self.selectors.append(new_selector) - - def get_slave_selectors(self): - return self.selectors - - # Return number of slaves for this interface - def get_slave_count(self): - return len(self.hardware_slaves) - - """ - Merge two lists of HostInterface models with child_list taking - priority when conflicts. If a member of child_list has a device_name - beginning with '!' it indicates that HostInterface should be - removed from the merged list - """ - - @staticmethod - def merge_lists(child_list, parent_list): - if len(child_list) == 0: - return deepcopy(parent_list) - - effective_list = [] - if len(parent_list) == 0: - for i in child_list: - if i.device_name.startswith('!'): - continue - else: - effective_list.append(deepcopy(i)) - return effective_list - - parent_interfaces = [] - for i in parent_list: - parent_name = i.device_name - parent_interfaces.append(parent_name) - add = True - for j in child_list: - if j.device_name == ("!" + parent_name): - add = False - break - elif j.device_name == parent_name: - m = HostInterface(j.api_version) - m.device_name = j.device_name - m.network_link = \ - Utils.apply_field_inheritance(j.network_link, - i.network_link) - s = filter(lambda x: ("!" + x) not in j.hardware_slaves, - i.hardware_slaves) - s = list(s) - - s.extend(filter(lambda x: not x.startswith("!"), - j.hardware_slaves)) - m.hardware_slaves = s - - n = filter(lambda x: ("!" + x) not in j.networks, - i.networks) - n = list(n) - - n.extend(filter(lambda x: not x.startswith("!"), - j.networks)) - m.networks = n - - effective_list.append(m) - add = False - break - - if add: - effective_list.append(deepcopy(i)) - - for j in child_list: - if (j.device_name not in parent_interfaces - and not j.device_name.startswith("!")): - effective_list.append(deepcopy(j)) - - return effective_list - - -class HostPartition(object): - - def __init__(self, api_version, **kwargs): - self.api_version = api_version - - if self.api_version == "v1.0": - self.name = kwargs.get('name', None) - self.device = kwargs.get('device', None) - self.part_uuid = kwargs.get('part_uuid', None) - self.size = kwargs.get('size', None) - self.mountpoint = kwargs.get('mountpoint', None) - self.fstype = kwargs.get('fstype', 'ext4') - self.mount_options = kwargs.get('mount_options', 'defaults') - self.fs_uuid = kwargs.get('fs_uuid', None) - self.fs_label = kwargs.get('fs_label', None) - else: - raise ValueError('Unknown API version of object') - - # The device attribute may be hardware alias that translates to a - # physical device address. If the device attribute does not match an - # alias, we assume it directly identifies a OS device name. When the - # apply_hardware_profile method is called on the parent Node of this - # device, the selector will be decided and applied - - def set_selector(self, sel_type, address='', dev_type=''): - selector = {} - selector['type'] = sel_type - selector['address'] = address - selector['device_type'] = dev_type - - self.selector = selector - - def get_selector(self): - return self.selector - - """ - Merge two lists of HostPartition models with child_list taking - priority when conflicts. If a member of child_list has a name - beginning with '!' it indicates that HostPartition should be - removed from the merged list - """ - - @staticmethod - def merge_lists(child_list, parent_list): - if len(child_list) == 0: - return deepcopy(parent_list) - - effective_list = [] - if len(parent_list) == 0: - for i in child_list: - if i.name.startswith('!'): - continue - else: - effective_list.append(deepcopy(i)) - - inherit_field_list = ["device", "part_uuid", "size", - "mountpoint", "fstype", "mount_options", - "fs_uuid", "fs_label"] - - parent_partitions = [] - for i in parent_list: - parent_name = i.name - parent_partitions.append(parent_name) - add = True - for j in child_list: - if j.name == ("!" + parent_name): - add = False - break - elif j.name == parent_name: - p = HostPartition(j.api_version) - p.name = j.name - - for f in inherit_field_list: - setattr(p, Utils.apply_field_inheritance(getattr(j, f), - getattr(i, f)) - ) - add = False - effective_list.append(p) - if add: - effective_list.append(deepcopy(i)) - - for j in child_list: - if j.name not in parent_list: - effective_list.append(deepcopy(j)) - - return effective_list - -# A BaremetalNode is really nothing more than a physical -# instantiation of a HostProfile, so they both represent -# the same set of CIs -class BaremetalNode(HostProfile): - - def __init__(self, **kwargs): - super(BaremetalNode, self).__init__(**kwargs) - - def apply_host_profile(self, site): - return self.apply_inheritance(site) - - # Translate device alises to physical selectors and copy - # other hardware attributes into this object - def apply_hardware_profile(self, site): - self_copy = deepcopy(self) - - if self.hardware_profile is None: - raise ValueError("Hardware profile not set") - - hw_profile = site.get_hardware_profile(self.hardware_profile) - - for i in self_copy.interfaces: - for s in i.hardware_slaves: - selector = hw_profile.resolve_alias("pci", s) - if selector is None: - i.add_selector("name", address=p.device) - else: - i.add_selector("address", address=selector['address'], - dev_type=selector['device_type']) - - for p in self_copy.partitions: - selector = hw_profile.resolve_alias("scsi", p.device) - if selector is None: - p.set_selector("name", address=p.device) - else: - p.set_selector("address", address=selector['address'], - dev_type=selector['device_type']) - - - hardware = {"vendor": getattr(hw_profile, 'vendor', None), - "generation": getattr(hw_profile, 'generation', None), - "hw_version": getattr(hw_profile, 'hw_version', None), - "bios_version": getattr(hw_profile, 'bios_version', None), - "boot_mode": getattr(hw_profile, 'boot_mode', None), - "bootstrap_protocol": getattr(hw_profile, - 'bootstrap_protocol', - None), - "pxe_interface": getattr(hw_profile, 'pxe_interface', None) - } - - self_copy.hardware = hardware - - return self_copy - - def get_interface(self, iface_name): - for i in self.interfaces: - if i.device_name == iface_name: - return i - return None - # Utility class for calculating inheritance class Utils(object): diff --git a/helm_drydock/model/hostprofile.py b/helm_drydock/model/hostprofile.py new file mode 100644 index 00000000..b239219c --- /dev/null +++ b/helm_drydock/model/hostprofile.py @@ -0,0 +1,378 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Models for helm_drydock +# +import logging + +from copy import deepcopy + +from helm_drydock.orchestrator.enum import SiteStatus +from helm_drydock.orchestrator.enum import NodeStatus +from helm_drydock.model.network import Network +from helm_drydock.model.network import NetworkLink +from helm_drydock.model import Utils + +class HostProfile(object): + + def __init__(self, **kwargs): + self.log = logging.Logger('model') + + self.api_version = kwargs.get('apiVersion', '') + + if self.api_version == "v1.0": + metadata = kwargs.get('metadata', {}) + spec = kwargs.get('spec', {}) + + self.name = metadata.get('name', '') + self.site = metadata.get('region', '') + + self.parent_profile = spec.get('host_profile', None) + self.hardware_profile = spec.get('hardware_profile', None) + + oob = spec.get('oob', {}) + self.oob_type = oob.get('type', None) + self.oob_network = oob.get('network', None) + self.oob_account = oob.get('account', None) + self.oob_credential = oob.get('credential', None) + + storage = spec.get('storage', {}) + self.storage_layout = storage.get('layout', 'lvm') + + bootdisk = storage.get('bootdisk', {}) + self.bootdisk_device = bootdisk.get('device', None) + self.bootdisk_root_size = bootdisk.get('root_size', None) + self.bootdisk_boot_size = bootdisk.get('boot_size', None) + + partitions = storage.get('partitions', []) + self.partitions = [] + + for p in partitions: + self.partitions.append(HostPartition(self.api_version, **p)) + + interfaces = spec.get('interfaces', []) + self.interfaces = [] + + for i in interfaces: + self.interfaces.append(HostInterface(self.api_version, **i)) + + node_metadata = spec.get('metadata', {}) + + metadata_tags = node_metadata.get('tags', []) + self.tags = [] + + for t in metadata_tags: + self.tags.append(t) + + owner_data = node_metadata.get('owner_data', {}) + self.owner_data = {} + + for k, v in owner_data.items(): + self.owner_data[k] = v + + self.rack = node_metadata.get('rack', None) + + else: + self.log.error("Unknown API version %s of %s" % + (self.api_version, self.__class__)) + raise ValueError('Unknown API version of object') + + def get_rack(self): + return self.rack + + def get_name(self): + return self.name + + def has_tag(self, tag): + if tag in self.tags: + return True + + return False + + def apply_inheritance(self, site): + # We return a deep copy of the profile so as not to corrupt + # the original model + self_copy = deepcopy(self) + + if self.parent_profile is None: + return self_copy + + parent = site.get_host_profile(self.parent_profile) + + if parent is None: + raise NameError("Cannot find parent profile %s for %s" + % (self.parent_profile, self.name)) + + parent = parent.apply_inheritance(site) + + # First compute inheritance for simple fields + inheritable_field_list = [ + "hardware_profile", "oob_type", "oob_network", + "oob_credential", "oob_account", "storage_layout", + "bootdisk_device", "bootdisk_root_size", "bootdisk_boot_size", + "rack"] + + for f in inheritable_field_list: + setattr(self_copy, f, + Utils.apply_field_inheritance(getattr(self, f, None), + getattr(parent, f, None))) + + # Now compute inheritance for complex types + self_copy.tags = Utils.merge_lists(self.tags, parent.tags) + + self_copy.owner_data = Utils.merge_dicts( + self.owner_data, parent.owner_data) + + self_copy.interfaces = HostInterface.merge_lists( + self.interfaces, parent.interfaces) + + self_copy.partitions = HostPartition.merge_lists( + self.partitions, parent.partitions) + + return self_copy + + +class HostInterface(object): + + def __init__(self, api_version, **kwargs): + self.log = logging.Logger('model') + + self.api_version = api_version + + if self.api_version == "v1.0": + self.device_name = kwargs.get('device_name', None) + self.network_link = kwargs.get('device_link', None) + + self.hardware_slaves = [] + slaves = kwargs.get('slaves', []) + + for s in slaves: + self.hardware_slaves.append(s) + + self.networks = [] + networks = kwargs.get('networks', []) + + for n in networks: + self.networks.append(n) + else: + self.log.error("Unknown API version %s of %s" % + (self.api_version, self.__class__)) + raise ValueError('Unknown API version of object') + + # The device attribute may be hardware alias that translates to a + # physical device address. If the device attribute does not match an + # alias, we assume it directly identifies a OS device name. When the + # apply_hardware_profile method is called on the parent Node of this + # device, the selector will be decided and applied + + def add_selector(self, sel_type, address='', dev_type=''): + if getattr(self, 'selectors', None) is None: + self.selectors = [] + + new_selector = {} + new_selector['selector_type'] = sel_type + new_selector['address'] = address + new_selector['device_type'] = dev_type + + self.selectors.append(new_selector) + + def get_slave_selectors(self): + return self.selectors + + # Return number of slaves for this interface + def get_slave_count(self): + return len(self.hardware_slaves) + + def apply_link_config(self, net_link): + if (net_link is not None and + isinstance(net_link, NetworkLink) and + net_link.name == self.network_link): + + self.attached_link = deepcopy(net_link) + return True + return False + + def apply_network_config(self, network): + if network in self.networks: + if getattr(self, 'attached_networks', None) is None: + self.attached_networks = [] + self.attached_networks.append(deepcopy(network)) + return True + else: + return False + + def set_network_address(self, network_name, address): + if getattr(self, 'attached_networks', None) is None: + return False + + for n in self.attached_neteworks: + if n.name == network_name: + n.assigned_address = address + + def get_network_configs(self): + return self.attached_networks + + """ + Merge two lists of HostInterface models with child_list taking + priority when conflicts. If a member of child_list has a device_name + beginning with '!' it indicates that HostInterface should be + removed from the merged list + """ + + @staticmethod + def merge_lists(child_list, parent_list): + if len(child_list) == 0: + return deepcopy(parent_list) + + effective_list = [] + if len(parent_list) == 0: + for i in child_list: + if i.device_name.startswith('!'): + continue + else: + effective_list.append(deepcopy(i)) + return effective_list + + parent_interfaces = [] + for i in parent_list: + parent_name = i.device_name + parent_interfaces.append(parent_name) + add = True + for j in child_list: + if j.device_name == ("!" + parent_name): + add = False + break + elif j.device_name == parent_name: + m = HostInterface(j.api_version) + m.device_name = j.device_name + m.network_link = \ + Utils.apply_field_inheritance(j.network_link, + i.network_link) + s = filter(lambda x: ("!" + x) not in j.hardware_slaves, + i.hardware_slaves) + s = list(s) + + s.extend(filter(lambda x: not x.startswith("!"), + j.hardware_slaves)) + m.hardware_slaves = s + + n = filter(lambda x: ("!" + x) not in j.networks, + i.networks) + n = list(n) + + n.extend(filter(lambda x: not x.startswith("!"), + j.networks)) + m.networks = n + + effective_list.append(m) + add = False + break + + if add: + effective_list.append(deepcopy(i)) + + for j in child_list: + if (j.device_name not in parent_interfaces + and not j.device_name.startswith("!")): + effective_list.append(deepcopy(j)) + + return effective_list + + +class HostPartition(object): + + def __init__(self, api_version, **kwargs): + self.api_version = api_version + + if self.api_version == "v1.0": + self.name = kwargs.get('name', None) + self.device = kwargs.get('device', None) + self.part_uuid = kwargs.get('part_uuid', None) + self.size = kwargs.get('size', None) + self.mountpoint = kwargs.get('mountpoint', None) + self.fstype = kwargs.get('fstype', 'ext4') + self.mount_options = kwargs.get('mount_options', 'defaults') + self.fs_uuid = kwargs.get('fs_uuid', None) + self.fs_label = kwargs.get('fs_label', None) + else: + raise ValueError('Unknown API version of object') + + # The device attribute may be hardware alias that translates to a + # physical device address. If the device attribute does not match an + # alias, we assume it directly identifies a OS device name. When the + # apply_hardware_profile method is called on the parent Node of this + # device, the selector will be decided and applied + + def set_selector(self, sel_type, address='', dev_type=''): + selector = {} + selector['type'] = sel_type + selector['address'] = address + selector['device_type'] = dev_type + + self.selector = selector + + def get_selector(self): + return self.selector + + """ + Merge two lists of HostPartition models with child_list taking + priority when conflicts. If a member of child_list has a name + beginning with '!' it indicates that HostPartition should be + removed from the merged list + """ + + @staticmethod + def merge_lists(child_list, parent_list): + if len(child_list) == 0: + return deepcopy(parent_list) + + effective_list = [] + if len(parent_list) == 0: + for i in child_list: + if i.name.startswith('!'): + continue + else: + effective_list.append(deepcopy(i)) + + inherit_field_list = ["device", "part_uuid", "size", + "mountpoint", "fstype", "mount_options", + "fs_uuid", "fs_label"] + + parent_partitions = [] + for i in parent_list: + parent_name = i.name + parent_partitions.append(parent_name) + add = True + for j in child_list: + if j.name == ("!" + parent_name): + add = False + break + elif j.name == parent_name: + p = HostPartition(j.api_version) + p.name = j.name + + for f in inherit_field_list: + setattr(p, Utils.apply_field_inheritance(getattr(j, f), + getattr(i, f)) + ) + add = False + effective_list.append(p) + if add: + effective_list.append(deepcopy(i)) + + for j in child_list: + if j.name not in parent_list: + effective_list.append(deepcopy(j)) + + return effective_list diff --git a/helm_drydock/model/hwprofile.py b/helm_drydock/model/hwprofile.py new file mode 100644 index 00000000..cb8d3779 --- /dev/null +++ b/helm_drydock/model/hwprofile.py @@ -0,0 +1,95 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Models for helm_drydock +# +import logging + +from copy import deepcopy + +from helm_drydock.orchestrator.enum import SiteStatus +from helm_drydock.orchestrator.enum import NodeStatus + +class HardwareProfile(object): + + def __init__(self, **kwargs): + self.log = logging.Logger('model') + + self.api_version = kwargs.get('apiVersion', '') + + if self.api_version == "v1.0": + metadata = kwargs.get('metadata', {}) + spec = kwargs.get('spec', {}) + + # Need to add validation logic, we'll assume the input is + # valid for now + self.name = metadata.get('name', '') + self.site = metadata.get('region', '') + + self.vendor = spec.get('vendor', None) + self.generation = spec.get('generation', None) + self.hw_version = spec.get('hw_version', None) + self.bios_version = spec.get('bios_version', None) + self.boot_mode = spec.get('boot_mode', None) + self.bootstrap_protocol = spec.get('bootstrap_protocol', None) + self.pxe_interface = spec.get('pxe_interface', None) + self.devices = [] + + device_aliases = spec.get('device_aliases', {}) + + pci_devices = device_aliases.get('pci', []) + scsi_devices = device_aliases.get('scsi', []) + + for d in pci_devices: + d['bus_type'] = 'pci' + self.devices.append( + HardwareDeviceAlias(self.api_version, **d)) + + for d in scsi_devices: + d['bus_type'] = 'scsi' + self.devices.append( + HardwareDeviceAlias(self.api_version, **d)) + else: + self.log.error("Unknown API version %s of %s" % + (self.api_version, self.__class__)) + raise ValueError('Unknown API version of object') + + return + + def resolve_alias(self, alias_type, alias): + selector = {} + for d in self.devices: + if d.alias == alias and d.bus_type == alias_type: + selector['address'] = d.address + selector['device_type'] = d.type + return selector + + return None + +class HardwareDeviceAlias(object): + + def __init__(self, api_version, **kwargs): + self.log = logging.Logger('model') + + self.api_version = api_version + + if self.api_version == "v1.0": + self.bus_type = kwargs.get('bus_type', None) + self.address = kwargs.get('address', None) + self.alias = kwargs.get('alias', None) + self.type = kwargs.get('type', None) + else: + self.log.error("Unknown API version %s of %s" % + (self.api_version, self.__class__)) + raise ValueError('Unknown API version of object') diff --git a/helm_drydock/model/network.py b/helm_drydock/model/network.py new file mode 100644 index 00000000..ef784114 --- /dev/null +++ b/helm_drydock/model/network.py @@ -0,0 +1,133 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Models for helm_drydock +# +import logging + +from copy import deepcopy + +from helm_drydock.orchestrator.enum import SiteStatus +from helm_drydock.orchestrator.enum import NodeStatus + +class NetworkLink(object): + + def __init__(self, **kwargs): + self.log = logging.Logger('model') + + self.api_version = kwargs.get('apiVersion', '') + + if self.api_version == "v1.0": + metadata = kwargs.get('metadata', {}) + spec = kwargs.get('spec', {}) + + self.name = metadata.get('name', '') + self.site = metadata.get('region', '') + + bonding = spec.get('bonding', {}) + self.bonding_mode = bonding.get('mode', 'none') + + # How should we define defaults for CIs not in the input? + if self.bonding_mode == '802.3ad': + self.bonding_xmit_hash = bonding.get('hash', 'layer3+4') + self.bonding_peer_rate = bonding.get('peer_rate', 'fast') + self.bonding_mon_rate = bonding.get('mon_rate', '100') + self.bonding_up_delay = bonding.get('up_delay', '200') + self.bonding_down_delay = bonding.get('down_delay', '200') + + self.mtu = spec.get('mtu', 1500) + self.linkspeed = spec.get('linkspeed', 'auto') + + trunking = spec.get('trunking', {}) + self.trunk_mode = trunking.get('mode', 'none') + + self.native_network = spec.get('default_network', '') + else: + self.log.error("Unknown API version %s of %s" % + (self.api_version, self.__class__)) + raise ValueError('Unknown API version of object') + + +class Network(object): + + def __init__(self, **kwargs): + self.log = logging.Logger('model') + + self.api_version = kwargs.get('apiVersion', '') + + if self.api_version == "v1.0": + metadata = kwargs.get('metadata', {}) + spec = kwargs.get('spec', {}) + + self.name = metadata.get('name', '') + self.site = metadata.get('region', '') + + self.cidr = spec.get('cidr', None) + self.allocation_strategy = spec.get('allocation', 'static') + self.vlan_id = spec.get('vlan_id', 1) + self.mtu = spec.get('mtu', 0) + + dns = spec.get('dns', {}) + self.dns_domain = dns.get('domain', 'local') + self.dns_servers = dns.get('servers', None) + + ranges = spec.get('ranges', []) + self.ranges = [] + + for r in ranges: + self.ranges.append(NetworkAddressRange(self.api_version, **r)) + + routes = spec.get('routes', []) + self.routes = [] + + for r in routes: + self.routes.append(NetworkRoute(self.api_version, **r)) + else: + self.log.error("Unknown API version %s of %s" % + (self.api_version, self.__class__)) + raise ValueError('Unknown API version of object') + + +class NetworkAddressRange(object): + + def __init__(self, api_version, **kwargs): + self.log = logging.Logger('model') + + self.api_version = api_version + + if self.api_version == "v1.0": + self.type = kwargs.get('type', None) + self.start = kwargs.get('start', None) + self.end = kwargs.get('end', None) + else: + self.log.error("Unknown API version %s of %s" % + (self.api_version, self.__class__)) + raise ValueError('Unknown API version of object') + + +class NetworkRoute(object): + + def __init__(self, api_version, **kwargs): + self.log = logging.Logger('model') + + self.api_version = api_version + + if self.api_version == "v1.0": + self.type = kwargs.get('subnet', None) + self.start = kwargs.get('gateway', None) + self.end = kwargs.get('metric', 100) + else: + self.log.error("Unknown API version %s of %s" % + (self.api_version, self.__class__)) + raise ValueError('Unknown API version of object') diff --git a/helm_drydock/model/node.py b/helm_drydock/model/node.py new file mode 100644 index 00000000..22d41b2e --- /dev/null +++ b/helm_drydock/model/node.py @@ -0,0 +1,154 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Models for helm_drydock +# +import logging + +from copy import deepcopy + + +from helm_drydock.orchestrator.enum import SiteStatus +from helm_drydock.orchestrator.enum import NodeStatus +from helm_drydock.model.hostprofile import HostProfile +from helm_drydock.model import Utils + +class BaremetalNode(HostProfile): + + # A BaremetalNode is really nothing more than a physical + # instantiation of a HostProfile, so they both represent + # the same set of CIs + def __init__(self, **kwargs): + super(BaremetalNode, self).__init__(**kwargs) + + if self.api_version == "v1.0": + self.addressing = [] + + spec = kwargs.get('spec', {}) + addresses = spec.get('addressing', []) + + if len(addresses) == 0: + raise ValueError('BaremetalNode needs at least' \ + ' 1 assigned address') + for a in addresses: + assignment = {} + address = a.get('address', '') + if address == 'dhcp': + assignment['type'] = 'dhcp' + assignment['address'] = None + assignment['network'] = a.get('network') + self.addressing.append(assignment) + elif address != '': + assignment['type'] = 'static' + assignment['address'] = a.get('address') + assignment['network'] = a.get('network') + self.addressing.append(assignment) + else: + self.log.error("Invalid address assignment %s on Node %s" + % (address, self.name)) + + self.build = kwargs.get('build', {}) + + def start_build(self): + if self.build.get('status','') == '': + self.build['status'] = NodeStatus.Unknown + + def apply_host_profile(self, site): + return self.apply_inheritance(site) + + # Translate device alises to physical selectors and copy + # other hardware attributes into this object + def apply_hardware_profile(self, site): + self_copy = deepcopy(self) + + if self.hardware_profile is None: + raise ValueError("Hardware profile not set") + + hw_profile = site.get_hardware_profile(self.hardware_profile) + + for i in self_copy.interfaces: + for s in i.hardware_slaves: + selector = hw_profile.resolve_alias("pci", s) + if selector is None: + i.add_selector("name", address=p.device) + else: + i.add_selector("address", address=selector['address'], + dev_type=selector['device_type']) + + for p in self_copy.partitions: + selector = hw_profile.resolve_alias("scsi", p.device) + if selector is None: + p.set_selector("name", address=p.device) + else: + p.set_selector("address", address=selector['address'], + dev_type=selector['device_type']) + + + hardware = {"vendor": getattr(hw_profile, 'vendor', None), + "generation": getattr(hw_profile, 'generation', None), + "hw_version": getattr(hw_profile, 'hw_version', None), + "bios_version": getattr(hw_profile, 'bios_version', None), + "boot_mode": getattr(hw_profile, 'boot_mode', None), + "bootstrap_protocol": getattr(hw_profile, + 'bootstrap_protocol', + None), + "pxe_interface": getattr(hw_profile, 'pxe_interface', None) + } + + self_copy.hardware = hardware + + return self_copy + + def apply_network_connections(self, site): + self_copy = deepcopy(self) + + for n in site.network_links: + for i in self_copy.interfaces: + i.apply_link_config(n) + + for n in site.networks: + for i in self_copy.interfaces: + i.apply_network_config(n) + + for a in self_copy.addressing: + for i in self_copy.interfaces: + i.set_network_address(a.get('network'), a.get('address')) + + return self_copy + + def get_interface(self, iface_name): + for i in self.interfaces: + if i.device_name == iface_name: + return i + return None + + def get_status(self): + return self.build['status'] + + def set_status(self, status): + if isinstance(status, NodeStatus): + self.build['status'] = status + + def get_last_build_action(self): + return self.build.get('last_action', None) + + def set_last_build_action(self, action, result, detail=None): + last_action = self.build.get('last_action', None) + if last_action is None: + self.build['last_action'] = {} + last_action = self.build['last_action'] + last_action['action'] = action + last_action['result'] = result + if detail is not None: + last_action['detail'] = detail diff --git a/helm_drydock/model/site.py b/helm_drydock/model/site.py new file mode 100644 index 00000000..0bddb45a --- /dev/null +++ b/helm_drydock/model/site.py @@ -0,0 +1,122 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Models for helm_drydock +# +import logging + +from copy import deepcopy + +from helm_drydock.orchestrator.enum import SiteStatus +from helm_drydock.orchestrator.enum import NodeStatus + +class Site(object): + + def __init__(self, **kwargs): + self.log = logging.Logger('model') + + if kwargs is None: + raise ValueError("Empty arguments") + + self.api_version = kwargs.get('apiVersion', '') + + self.build = kwargs.get('build', {}) + + if self.api_version == "v1.0": + metadata = kwargs.get('metadata', {}) + + # Need to add validation logic, we'll assume the input is + # valid for now + self.name = metadata.get('name', '') + + spec = kwargs.get('spec', {}) + + self.tag_definitions = [] + tag_defs = spec.get('tag_definitions', []) + + for t in tag_defs: + self.tag_definitions.append( + NodeTagDefinition(self.api_version, **t)) + + self.networks = [] + self.network_links = [] + self.host_profiles = [] + self.hardware_profiles = [] + self.baremetal_nodes = [] + + else: + self.log.error("Unknown API version %s of %s" % + (self.api_version, self.__class__)) + raise ValueError('Unknown API version of object') + + def start_build(self): + if self.build.get('status', '') == '': + self.build['status'] = SiteStatus.Unknown + + def get_network(self, network_name): + for n in self.networks: + if n.name == network_name: + return n + + return None + + def get_network_link(self, link_name): + for l in self.network_links: + if l.name == link_name: + return l + + return None + + def get_host_profile(self, profile_name): + for p in self.host_profiles: + if p.name == profile_name: + return p + + return None + + def get_hardware_profile(self, profile_name): + for p in self.hardware_profiles: + if p.name == profile_name: + return p + + return None + + def get_baremetal_node(self, node_name): + for n in self.baremetal_nodes: + if n.name == node_name: + return n + + return None + + def set_status(self, status): + if isinstance(status, SiteStatus): + self.build['status'] = status + +class NodeTagDefinition(object): + + def __init__(self, api_version, **kwargs): + self.api_version = api_version + + if self.api_version == "v1.0": + self.tag = kwargs.get('tag', '') + self.definition_type = kwargs.get('definition_type', '') + self.definition = kwargs.get('definition', '') + + if self.definition_type not in ['lshw_xpath']: + raise ValueError('Unknown definition type in ' \ + 'NodeTagDefinition: %s' % (self.definition_type)) + else: + self.log.error("Unknown API version %s of %s" % + (self.api_version, self.__class__)) + raise ValueError('Unknown API version of object') \ No newline at end of file diff --git a/helm_drydock/orchestrator/__init__.py b/helm_drydock/orchestrator/__init__.py index 2a385a45..6ae6f0fb 100644 --- a/helm_drydock/orchestrator/__init__.py +++ b/helm_drydock/orchestrator/__init__.py @@ -10,4 +10,62 @@ # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and -# limitations under the License. \ No newline at end of file +# limitations under the License. +from enum import Enum, unique + +import uuid + +class Orchestrator(object): + + # enabled_drivers is a map which provider drivers + # should be enabled for use by this orchestrator + + def __init__(self, enabled_drivers=None, design_state=None): + self.enabled_drivers = {} + + self.enabled_drivers['oob'] = enabled_drivers.get('oob', None) + self.enabled_drivers['server'] = enabled_drivers.get('server', None) + self.enabled_drivers['network'] = enabled_drivers.get('network', None) + + self.design_state = design_state + + """ + execute_task + + This is the core of the orchestrator. The task will describe the action + to take and the context/scope of the command. We will then source + the current designed state and current built state from the statemgmt + module. Based on those 3 inputs, we'll decide what is needed next. + """ + def execute_task(self, task): + if design_state is None: + raise Exception("Cannot execute task without initialized state manager") + + +class OrchestrationTask(object): + + def __init__(self, action, **kwargs): + self.taskid = uuid.uuid4() + + self.action = action + + parent_task = kwargs.get('parent_task','') + + # Validate parameters based on action + self.site = kwargs.get('site', '') + + + if self.site == '': + raise ValueError("Task requires 'site' parameter") + + if action in [Action.VerifyNode, Action.PrepareNode, + Action.DeployNode, Action.DestroyNode]: + self.node_filter = kwargs.get('node_filter', None) + + def child_task(self, action, **kwargs): + child_task = OrchestrationTask(action, parent_task=self.taskid, site=self.site, **kwargs) + return child_task + + + + diff --git a/helm_drydock/orchestrator/designdata.py b/helm_drydock/orchestrator/designdata.py index a26ebb1d..bc2cfbba 100644 --- a/helm_drydock/orchestrator/designdata.py +++ b/helm_drydock/orchestrator/designdata.py @@ -16,6 +16,8 @@ import logging from copy import deepcopy +from helm_drydock.error import DesignError + class DesignStateClient(object): def __init__(self): @@ -31,34 +33,47 @@ class DesignStateClient(object): return a Site model populated with all components from the design state """ - def load_design_data(self, site_name, design_state=None): - site = design_state.get_site(site_name) + def load_design_data(self, site_name, design_state=None, change_id=None): + if design_state is None: + raise ValueError("Design state is None") - networks = design_state.get_networks() + design_data = None + + if change_id is None: + try: + design_data = design_state.get_design_base() + except DesignError(e): + raise e + else: + design_data = design_state.get_design_change(change_id) + + site = design_data.get_site(site_name) + + networks = design_data.get_networks() for n in networks: if n.site == site_name: site.networks.append(n) - network_links = design_state.get_network_links() + network_links = design_data.get_network_links() for l in network_links: if l.site == site_name: site.network_links.append(l) - host_profiles = design_state.get_host_profiles() + host_profiles = design_data.get_host_profiles() for p in host_profiles: if p.site == site_name: site.host_profiles.append(p) - hardware_profiles = design_state.get_hardware_profiles() + hardware_profiles = design_data.get_hardware_profiles() for p in hardware_profiles: if p.site == site_name: site.hardware_profiles.append(p) - baremetal_nodes = design_state.get_baremetal_nodes() + baremetal_nodes = design_data.get_baremetal_nodes() for n in baremetal_nodes: if n.site == site_name: @@ -66,12 +81,6 @@ class DesignStateClient(object): return site - """ - compute_model_inheritance - given a fully populated Site model, compute the effecitve - design by applying inheritance and references - - return a Site model reflecting the effective design for the site - """ def compute_model_inheritance(self, site_root): # For now the only thing that really incorporates inheritance is @@ -86,8 +95,15 @@ class DesignStateClient(object): for n in site_copy.baremetal_nodes: resolved = n.apply_host_profile(site_copy) resolved = resolved.apply_hardware_profile(site_copy) + resolved = resolved.apply_network_connections(site_copy) effective_nodes.append(resolved) site_copy.baremetal_nodes = effective_nodes return site_copy + """ + compute_model_inheritance - given a fully populated Site model, + compute the effecitve design by applying inheritance and references + + return a Site model reflecting the effective design for the site + """ diff --git a/helm_drydock/orchestrator/enum.py b/helm_drydock/orchestrator/enum.py new file mode 100644 index 00000000..ffbf958a --- /dev/null +++ b/helm_drydock/orchestrator/enum.py @@ -0,0 +1,64 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from enum import Enum, unique + +@unique +class Action(Enum): + Noop = 'noop' + ValidateDesign = 'validate_design' + VerifySite = 'verify_site' + PrepareSite = 'prepare_site' + VerifyNode = 'verify_node' + PrepareNode = 'prepare_node' + DeployNode = 'deploy_node' + DestroyNode = 'destroy_node' + +@unique +class ActionResult(Enum): + Success = 'success' + PartialSuccess = 'partial_success' + Failure = 'failure' + DependentFailure = 'dependent_failure' + +@unique +class SiteStatus(Enum): + Unknown = 'unknown' + DesignStarted = 'design_started' + DesignAvailable = 'design_available' + DesignValidated = 'design_validated' + Deploying = 'deploying' + Deployed = 'deployed' + DesignUpdated = 'design_updated' + +@unique +class NodeStatus(Enum): + Unknown = 'unknown' + Designed = 'designed' + Present = 'present' # IPMI access verified + BasicVerifying = 'basic_verifying' # Base node verification in process + FailedBasicVerify = 'failed_basic_verify' # Base node verification failed + BasicVerified = 'basic_verified' # Base node verification successful + Preparing = 'preparing' # Node preparation in progress + FailedPrepare = 'failed_prepare' # Node preparation failed + Prepared = 'prepared' # Node preparation complete + FullyVerifying = 'fully_verifying' # Node full verification in progress + FailedFullVerify = 'failed_full_verify' # Node full verification failed + FullyVerified = 'fully_verified' # Deeper verification successful + Deploying = 'deploy' # Node deployment in progress + FailedDeploy = 'failed_deploy' # Node deployment failed + Deployed = 'deployed' # Node deployed successfully + Bootstrapping = 'bootstrapping' # Node bootstrapping + FailedBootstrap = 'failed_bootstrap' # Node bootstrapping failed + Bootstrapped = 'bootstrapped' # Node fully bootstrapped + Complete = 'complete' # Node is complete diff --git a/helm_drydock/orchestrator/readme.md b/helm_drydock/orchestrator/readme.md new file mode 100644 index 00000000..fd11755f --- /dev/null +++ b/helm_drydock/orchestrator/readme.md @@ -0,0 +1,83 @@ +# Orchestrator # + +The orchestrator is the core of drydock and will manage +the ordering of driver actions to implement the main Drydock +actions. Each of these actions will be started by the +external cLCP orchestrator with different parameters to +control the scope of the action. + +Orchestrator should persist the state of each task +such that on failure the task can retried and only the +steps needed will be executed. + +Bullet points listed below are not exhaustive and will +change as we move through testing + +## ValidateDesign ## + +Load design data from the statemgmt persistent store and +validate that the current state of design data represents +a valid site design. No claim is made that the design data +is compatible with the physical state of the site. + +## VerifySite ## + +Verify site-wide resources are in a useful state + +* Driver downstream resources are reachable (e.g. MaaS) +* OS images needed for bootstrapping are available +* Promenade or other next-step services are up and available +* Verify credentials are available + +## PrepareSite ## + +Begin preparing site-wide resources for bootstrapping. This +action will lock site design data for changes. + +* Configure bootstrapper with site network configs +* Shuffle images so they are correctly configured for bootstrapping + +## VerifyNode ## + +Verification of per-node configurations within the context +of the current node status + +* Status: Present + * Basic hardware verification as available via OOB driver + - BIOS firmware + - PCI layout + - Drives + - Hardware alarms + * IPMI connectivity +* Status: Prepared + - Full hardware manifest + - Possibly network connectivity + - Firmware versions + +## PrepareNode ## + +Prepare a node for bootstrapping + +* Configure network port for PXE +* Configure a node for PXE boot +* Power-cycle the node +* Setup commissioning configuration + - Hardware drivers + - Hardware configuration (e.g. RAID) +* Configure node networking +* Configure node storage + +## DeployNode ## + +Begin bootstrapping the node and monitor +success + +* Initialize the Introspection service for the node +* Bootstrap the node (i.e. Write persistent OS install) +* Ensure network port is returned to production configuration +* Reboot node from local disk +* Monitor platform bootstrapping + +## DestroyNode ## + +Destroy current node configuration and rebootstrap from scratch \ No newline at end of file diff --git a/helm_drydock/statemgmt/__init__.py b/helm_drydock/statemgmt/__init__.py index 2fb13a51..b630cdfd 100644 --- a/helm_drydock/statemgmt/__init__.py +++ b/helm_drydock/statemgmt/__init__.py @@ -11,23 +11,132 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from copy import deepcopy +from datetime import datetime +from datetime import timezone -import helm_drydock.model as model +import uuid + +import helm_drydock.model.node as node +import helm_drydock.model.hostprofile as hostprofile +import helm_drydock.model.network as network +import helm_drydock.model.site as site +import helm_drydock.model.hwprofile as hwprofile + +from helm_drydock.error import DesignError class DesignState(object): def __init__(self): + self.design_base = None + + self.design_changes = [] + + self.builds = [] + return + + # TODO Need to lock a design base or change once implementation + # has started + def get_design_base(self): + if self.design_base is None: + raise DesignError("No design base submitted") + + return deepcopy(self.design_base) + + def post_design_base(self, site_design): + if site_design is not None and isinstance(site_design, SiteDesign): + self.design_base = deepcopy(site_design) + return True + + def put_design_base(self, site_design): + # TODO Support merging + if site_design is not None and isinstance(site_design, SiteDesign): + self.design_base = deepcopy(site_design) + return True + + def get_design_change(self, changeid): + match = [x for x in self.design_changes if x.changeid == changeid] + + if len(match) == 0: + raise DesignError("No design change %s found." % (changeid)) + else: + return deepcopy(match[0]) + + def post_design_change(self, site_design): + if site_design is not None and isinstance(site_design, SiteDesign): + exists = [(x) for x + in self.design_changes + if x.changeid == site_design.changeid] + if len(exists) > 0: + raise DesignError("Existing change %s found" % + (site_design.changeid)) + self.design_changes.append(deepcopy(site_design)) + return True + else: + raise DesignError("Design change must be a SiteDesign instance") + + def put_design_change(self, site_design): + # TODO Support merging + if site_design is not None and isinstance(site_design, SiteDesign): + design_copy = deepcopy(site_design) + self.design_changes = [design_copy + if x.changeid == design_copy.changeid + else x + for x + in self.design_changes] + return True + else: + raise DesignError("Design change must be a SiteDesign instance") + + def get_current_build(self): + latest_stamp = 0 + current_build = None + + for b in self.builds: + if b.build_id > latest_stamp: + latest_stamp = b.build_id + current_build = b + + return deepcopy(current_build) + + def get_build(self, build_id): + for b in self.builds: + if b.build_id == build_id: + return b + + return None + + def post_build(self, site_build): + if site_build is not None and isinstance(site_build, SiteBuild): + exists = [b for b in self.builds + if b.build_id == site_build.build_id] + + if len(exists) > 0: + raise DesignError("Already a site build with ID %s" % + (str(site_build.build_id))) + else: + self.builds.append(deepcopy(site_build)) + return True + + +class SiteDesign(object): + + def __init__(self, ischange=False): + if ischange: + self.changeid = uuid.uuid4() + else: + self.changeid = 0 + self.sites = [] self.networks = [] self.network_links = [] self.host_profiles = [] self.hardware_profiles = [] self.baremetal_nodes = [] - return def add_site(self, new_site): - if new_site is None or not isinstance(new_site, model.Site): - raise Exception("Invalid Site model") + if new_site is None or not isinstance(new_site, site.Site): + raise DesignError("Invalid Site model") self.sites.append(new_site) @@ -39,11 +148,11 @@ class DesignState(object): if s.name == site_name: return s - raise NameError("Site %s not found in design state" % site_name) + raise DesignError("Site %s not found in design state" % site_name) def add_network(self, new_network): - if new_network is None or not isinstance(new_network, model.Network): - raise Exception("Invalid Network model") + if new_network is None or not isinstance(new_network, network.Network): + raise DesignError("Invalid Network model") self.networks.append(new_network) @@ -55,11 +164,13 @@ class DesignState(object): if n.name == network_name: return n - raise NameError("Network %s not found in design state" % network_name) + raise DesignError("Network %s not found in design state" + % network_name) def add_network_link(self, new_network_link): - if new_network_link is None or not isinstance(new_network_link, model.NetworkLink): - raise Exception("Invalid NetworkLink model") + if new_network_link is None or not isinstance(new_network_link, + network.NetworkLink): + raise DesignError("Invalid NetworkLink model") self.network_links.append(new_network_link) @@ -71,11 +182,13 @@ class DesignState(object): if l.name == link_name: return l - raise NameError("NetworkLink %s not found in design state" % link_name) + raise DesignError("NetworkLink %s not found in design state" + % link_name) def add_host_profile(self, new_host_profile): - if new_host_profile is None or not isinstance(new_host_profile, model.HostProfile): - raise Exception("Invalid HostProfile model") + if new_host_profile is None or not isinstance(new_host_profile, + hostprofile.HostProfile): + raise DesignError("Invalid HostProfile model") self.host_profiles.append(new_host_profile) @@ -87,11 +200,13 @@ class DesignState(object): if p.name == profile_name: return p - raise NameError("HostProfile %s not found in design state" % profile_name) + raise DesignError("HostProfile %s not found in design state" + % profile_name) def add_hardware_profile(self, new_hardware_profile): - if new_hardware_profile is None or not isinstance(new_hardware_profile, model.HardwareProfile): - raise Exception("Invalid HardwareProfile model") + if (new_hardware_profile is None or + not isinstance(new_hardware_profile, hwprofile.HardwareProfile)): + raise DesignError("Invalid HardwareProfile model") self.hardware_profiles.append(new_hardware_profile) @@ -103,11 +218,13 @@ class DesignState(object): if p.name == profile_name: return p - raise NameError("HardwareProfile %s not found in design state" % profile_name) + raise DesignError("HardwareProfile %s not found in design state" + % profile_name) def add_baremetal_node(self, new_baremetal_node): - if new_baremetal_node is None or not isinstance(new_baremetal_node, model.BaremetalNode): - raise Exception("Invalid BaremetalNode model") + if (new_baremetal_node is None or + not isinstance(new_baremetal_node, node.BaremetalNode)): + raise DesignError("Invalid BaremetalNode model") self.baremetal_nodes.append(new_baremetal_node) @@ -119,4 +236,59 @@ class DesignState(object): if n.name == node_name: return n - raise NameError("BaremetalNode %s not found in design state" % node_name) + raise DesignError("BaremetalNode %s not found in design state" + % node_name) + + +class SiteBuild(SiteDesign): + + def __init__(self, build_id=None): + super(SiteBuild, self).__init__() + + if build_id is None: + self.build_id = datetime.datetime.now(timezone.utc).timestamp() + else: + self.build_id = build_id + + def get_filtered_nodes(self, node_filter): + effective_nodes = self.get_baremetal_nodes() + + # filter by rack + rack_filter = node_filter.get('rackname', None) + + if rack_filter is not None: + rack_list = rack_filter.split(',') + effective_nodes = [x + for x in effective_nodes + if x.get_rack() in rack_list] + # filter by name + name_filter = node_filter.get('nodename', None) + + if name_filter is not None: + name_list = name_filter.split(',') + effective_nodes = [x + for x in effective_nodes + if x.get_name() in name_list] + # filter by tag + tag_filter = node_filter.get('tags', None) + + if tag_filter is not None: + tag_list = tag_filter.split(',') + effective_nodes = [x + for x in effective_nodes + for t in tag_list + if x.has_tag(t)] + + return effective_nodes + """ + Support filtering on rack name, node name or node tag + for now. Each filter can be a comma-delimited list of + values. The final result is an intersection of all the + filters + """ + + def set_nodes_status(self, node_filter, status): + target_nodes = self.get_filtered_nodes(node_filter) + + for n in target_nodes: + n.set_status(status) diff --git a/helm_drydock/statemgmt/readme.md b/helm_drydock/statemgmt/readme.md new file mode 100644 index 00000000..85e4e0d5 --- /dev/null +++ b/helm_drydock/statemgmt/readme.md @@ -0,0 +1,41 @@ +# Statemgmt # + +Statemgmt is the interface to the persistence store +for holding site design data as well as build status data + +/drydock - Base namespace for drydock data + +## As Designed ## + +Serialization of Drydock internal model as ingested. Not externally writable. + +/drydock/design +/drydock/design/base - The base site design used for the first deployment +/drydock/design/[changeID] - A change layer on top of the base site design. Chrono ordered + +## As Built ## + +Serialization of Drydock internal model as rendered to effective implementation including build status. Not externally writable. + +/drydock/build +/drydock/build/[datestamp] - A point-in-time view of what was deployed with deployment results + +## Node data ## + +Per-node data that can drive introspection as well as accept updates from nodes + +/drydock/nodedata +/drydock/nodedata/[nodename] - Per-node data, can be seeded with Ingested metadata but can be updated during by orchestrator or node + +## Service data ## + +Per-service data (may not be needed) + +/drydock/servicedata +/drydock/servicedata/[servicename] + +## Global data ## + +Generic k:v store that can be produced/consumed by non-Drydock services via the Drydock API + +/drydock/globaldata \ No newline at end of file diff --git a/helm_drydock/tox.ini b/helm_drydock/tox.ini deleted file mode 100644 index ae414018..00000000 --- a/helm_drydock/tox.ini +++ /dev/null @@ -1,11 +0,0 @@ -[tox] -envlist = py35 - -[testenv] -deps= - -rrequirements.txt -setenv= - PYTHONWARNING=all - -[flake8] -ignore=E302,H306 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 9c39a3da..1a33ec11 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,5 @@ -PyYAML -oauth -requests-oauthlib -pyipmi -netaddr -pecan -python-libmaas \ No newline at end of file +PyYAML==3.12 +oauth==1.0.1 +requests-oauthlib==0.8.0 +netaddr==0.7.19 +python-libmaas==0.4.1 \ No newline at end of file diff --git a/setup.py b/setup.py index c16c862c..69a958b3 100644 --- a/setup.py +++ b/setup.py @@ -50,7 +50,7 @@ setup(name='helm_drydock', 'PyYAML', 'oauth', 'requests-oauthlib', - 'pyipmi', + 'pyghmi', 'netaddr', 'pecan', 'webob' diff --git a/tests/test_design_inheritance.py b/tests/test_design_inheritance.py index 8691646d..0e13e8f1 100644 --- a/tests/test_design_inheritance.py +++ b/tests/test_design_inheritance.py @@ -13,7 +13,7 @@ # limitations under the License. from helm_drydock.ingester import Ingester -from helm_drydock.statemgmt import DesignState +from helm_drydock.statemgmt import DesignState, SiteDesign from helm_drydock.orchestrator.designdata import DesignStateClient from copy import deepcopy @@ -34,6 +34,11 @@ class TestClass(object): client = DesignStateClient() design_data = client.load_design_data("sitename", design_state=loaded_design) + + assert len(design_data.baremetal_nodes) == 2 + + print(yaml.dump(design_data, default_flow_style=False)) + design_data = client.compute_model_inheritance(design_data) node = design_data.get_baremetal_node("controller01") @@ -54,13 +59,15 @@ class TestClass(object): def loaded_design(self, input_files): input_file = input_files.join("fullsite.yaml") - module_design_state = DesignState() + design_state = DesignState() + design_data = SiteDesign() + design_state.post_design_base(design_data) ingester = Ingester() ingester.enable_plugins([helm_drydock.ingester.plugins.yaml.YamlIngester]) - ingester.ingest_data(plugin_name='yaml', design_state=module_design_state, filenames=[str(input_file)]) + ingester.ingest_data(plugin_name='yaml', design_state=design_state, filenames=[str(input_file)]) - return module_design_state + return design_state diff --git a/tests/test_ingester.py b/tests/test_ingester.py index 2081521b..e9db185f 100644 --- a/tests/test_ingester.py +++ b/tests/test_ingester.py @@ -13,7 +13,7 @@ # limitations under the License. from helm_drydock.ingester import Ingester -from helm_drydock.statemgmt import DesignState +from helm_drydock.statemgmt import DesignState, SiteDesign import pytest import shutil @@ -27,14 +27,19 @@ class TestClass(object): def test_ingest_full_site(self, input_files): input_file = input_files.join("fullsite.yaml") + design_state = DesignState() + design_data = SiteDesign() + design_state.post_design_base(design_data) ingester = Ingester() ingester.enable_plugins([helm_drydock.ingester.plugins.yaml.YamlIngester]) ingester.ingest_data(plugin_name='yaml', design_state=design_state, filenames=[str(input_file)]) - assert len(design_state.get_host_profiles()) == 3 - assert len(design_state.get_baremetal_nodes()) == 2 + design_data = design_state.get_design_base() + + assert len(design_data.get_host_profiles()) == 3 + assert len(design_data.get_baremetal_nodes()) == 2 def test_ingest_federated_design(self, input_files): profiles_file = input_files.join("fullsite_profiles.yaml") @@ -42,13 +47,17 @@ class TestClass(object): nodes_file = input_files.join("fullsite_nodes.yaml") design_state = DesignState() + design_data = SiteDesign() + design_state.post_design_base(design_data) ingester = Ingester() ingester.enable_plugins([helm_drydock.ingester.plugins.yaml.YamlIngester]) ingester.ingest_data(plugin_name='yaml', design_state=design_state, filenames=[str(profiles_file), str(networks_file), str(nodes_file)]) - assert len(design_state.host_profiles) == 3 + design_data = design_state.get_design_base() + + assert len(design_data.host_profiles) == 3 @pytest.fixture(scope='module') def input_files(self, tmpdir_factory, request): diff --git a/tests/test_models.py b/tests/test_models.py index 3e993c44..a2d6fe8f 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -14,7 +14,7 @@ import pytest import yaml -from helm_drydock.model import HardwareProfile +from helm_drydock.model.hwprofile import HardwareProfile class TestClass(object): diff --git a/tests/yaml_samples/fullsite.yaml b/tests/yaml_samples/fullsite.yaml index 3a7f8ede..df2f39ba 100644 --- a/tests/yaml_samples/fullsite.yaml +++ b/tests/yaml_samples/fullsite.yaml @@ -25,8 +25,7 @@ metadata: date: 17-FEB-2017 description: Sample site design author: sh8121@att.com -spec: - # Not sure if we have site wide data that doesn't fall into another 'Kind' +# Not sure if we have site wide data that doesn't fall into another 'Kind' --- apiVersion: 'v1.0' kind: NetworkLink @@ -454,4 +453,4 @@ spec: scsi: - address: scsi@2:0.0.0 alias: primary_boot - type: 'VBOX HARDDISK' \ No newline at end of file + type: 'VBOX HARDDISK' diff --git a/tests/yaml_samples/fullsite_networks.yaml b/tests/yaml_samples/fullsite_networks.yaml index 54770ac5..37af1412 100644 --- a/tests/yaml_samples/fullsite_networks.yaml +++ b/tests/yaml_samples/fullsite_networks.yaml @@ -225,4 +225,4 @@ spec: metric: 9 dns: domain: sitename.example.com - servers: 8.8.8.8 \ No newline at end of file + servers: 8.8.8.8 diff --git a/tests/yaml_samples/fullsite_profiles.yaml b/tests/yaml_samples/fullsite_profiles.yaml index 7ad9a73d..c5430085 100644 --- a/tests/yaml_samples/fullsite_profiles.yaml +++ b/tests/yaml_samples/fullsite_profiles.yaml @@ -25,8 +25,7 @@ metadata: date: 17-FEB-2017 description: Sample site design author: sh8121@att.com -spec: - # Not sure if we have site wide data that doesn't fall into another 'Kind' +# Not sure if we have site wide data that doesn't fall into another 'Kind' --- apiVersion: 'v1.0' kind: HostProfile @@ -194,4 +193,4 @@ spec: scsi: - address: scsi@2:0.0.0 alias: primary_boot - type: 'VBOX HARDDISK' \ No newline at end of file + type: 'VBOX HARDDISK'