diff --git a/README.md b/README.md index b97df8b9..dc619477 100644 --- a/README.md +++ b/README.md @@ -32,9 +32,15 @@ aka orchestrator Handle validation of complete design, ordering and managing downstream API calls for hardware provisioning/bootstrapping -### Server Driver ### +### OOB Driver ### -aka maasdriver +Pluggable provider for server OOB (ILO) management + +aka driver/oob + +### Node Driver ### + +aka driver/node Pluggable provisioner for server bootstrapping. Initial implementation is MaaS client. diff --git a/helm_drydock/config.py b/helm_drydock/config.py index 8bd363bc..92b3d8ac 100644 --- a/helm_drydock/config.py +++ b/helm_drydock/config.py @@ -21,7 +21,7 @@ class DrydockConfig(object): - def __init__(self): + def __init__(self): self.server_driver_config = { selected_driver = helm_drydock.drivers.server.maasdriver, params = { @@ -29,13 +29,13 @@ class DrydockConfig(object): maas_api_url = "" } } - self.selected_network_driver = helm_drydock.drivers.network.noopdriver - self.control_config = {} - self.ingester_config = { - plugins = [helm_drydock.ingester.plugins.aicyaml.AicYamlIngester] - } - self.introspection_config = {} - self.orchestrator_config = {} - self.statemgmt_config = { - backend_driver = helm_drydock.drivers.statemgmt.etcd, - } + self.selected_network_driver = helm_drydock.drivers.network.noopdriver + self.control_config = {} + self.ingester_config = { + plugins = [helm_drydock.ingester.plugins.aicyaml.AicYamlIngester] + } + self.introspection_config = {} + self.orchestrator_config = {} + self.statemgmt_config = { + backend_driver = helm_drydock.drivers.statemgmt.etcd, + } diff --git a/helm_drydock/control/readme.md b/helm_drydock/control/readme.md new file mode 100644 index 00000000..f3c602fb --- /dev/null +++ b/helm_drydock/control/readme.md @@ -0,0 +1,6 @@ +# Control # + +This is the external facing API service to control the rest +of Drydock and query Drydock-managed data. + +Anticipate basing this service on the falcon Python library \ No newline at end of file diff --git a/helm_drydock/drivers/__init__.py b/helm_drydock/drivers/__init__.py index 5c73a1fa..a0e27676 100644 --- a/helm_drydock/drivers/__init__.py +++ b/helm_drydock/drivers/__init__.py @@ -17,3 +17,5 @@ class ProviderDriver(object): __init__(self): pass + +class DriverTask(object): \ No newline at end of file diff --git a/helm_drydock/control/root.py b/helm_drydock/drivers/node/__init__.py similarity index 51% rename from helm_drydock/control/root.py rename to helm_drydock/drivers/node/__init__.py index eb4bb8af..bae95f38 100644 --- a/helm_drydock/control/root.py +++ b/helm_drydock/drivers/node/__init__.py @@ -11,25 +11,19 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +# -from pecan import expose -from webob.exc import status_map +from helm_drydock.drivers import ProviderDriver -class RootController(object): +class NodeDriver(ProviderDriver): - @expose(generic=True, template='index.html') - def index(self): - return dict() +class NodeAction(Enum): + PrepareNode = 'prepare_node' + ApplyNetworkConfig = 'apply_network_config' + ApplyStorageConfig = 'apply_storage_config' + InterrogateNode = 'interrogate_node' + DeployNode = 'deploy_node' - @index.when(method='POST') - def index_post(self, q): - redirect('https://pecan.readthedocs.io/en/latest/search.html?q=%s' % q) - @expose('error.html') - def error(self, status): - try: - status = int(status) - except ValueError: - status = 0 - message = getattr(status_map.get(status), 'explanation', '') - return dict(status=status, message=message) \ No newline at end of file + + \ No newline at end of file diff --git a/helm_drydock/drivers/server/maasdriver/__init__.py b/helm_drydock/drivers/node/maasdriver/__init__.py similarity index 78% rename from helm_drydock/drivers/server/maasdriver/__init__.py rename to helm_drydock/drivers/node/maasdriver/__init__.py index 21cbcd26..7c0c2b74 100644 --- a/helm_drydock/drivers/server/maasdriver/__init__.py +++ b/helm_drydock/drivers/node/maasdriver/__init__.py @@ -11,7 +11,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import helm_drydock.drivers.server.ServerDriver +from helm_drydock.drivers.node import NodeDriver -class MaasServerDriver(object): +class MaasNodeDriver(NodeDriver): + + def __init__(self, kwargs): + super(MaasNodeDriver, self).__init__(**kwargs) \ No newline at end of file diff --git a/helm_drydock/drivers/oob/__init__.py b/helm_drydock/drivers/oob/__init__.py new file mode 100644 index 00000000..9dd08ff8 --- /dev/null +++ b/helm_drydock/drivers/oob/__init__.py @@ -0,0 +1,42 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# OOB: +# sync_hardware_clock +# collect_chassis_sysinfo +# enable_netboot +# initiate_reboot +# set_power_off +# set_power_on + +from helm_drydock.drivers import ProviderDriver + +class OobDriver(ProviderDriver): + + def __init__(self): + pass + + def execute_action(self, action, **kwargs): + if action == + + + +class OobAction(Enum): + ConfigNodePxe = 'config_node_pxe' + SetNodeBoot = 'set_node_boot' + PowerOffNode = 'power_off_node' + PowerOnNode = 'power_on_node' + PowerCycleNode = 'power_cycle_node' + InterrogateNode = 'interrogate_node' + diff --git a/helm_drydock/drivers/readme.md b/helm_drydock/drivers/readme.md new file mode 100644 index 00000000..0a663a16 --- /dev/null +++ b/helm_drydock/drivers/readme.md @@ -0,0 +1,26 @@ +# Drivers # + +Drivers are downstream actors that Drydock will use to actually execute +orchestration actions. It is intended to be a pluggable architecture +so that various downstream automation can be used. + +## oob ## + +The oob drivers will interface with physical servers' out-of-band +management system (e.g. Dell iDRAC, HP iLO, etc...). OOB management +will be used for setting a system to use PXE boot and power cycling +servers. + +## node ## + +The node drivers will interface with an external bootstrapping system +for loading the base OS on a server and configuring hardware, network, +and storage. + +## network ## + +The network drivers will interface with switches for managing port +configuration to support the bootstrapping of physical nodes. This is not +intended to be a network provisioner, but instead is a support driver +for node bootstrapping where temporary changes to network configurations +are required. \ No newline at end of file diff --git a/helm_drydock/drivers/server/__init__.py b/helm_drydock/error.py similarity index 88% rename from helm_drydock/drivers/server/__init__.py rename to helm_drydock/error.py index febcdad1..9319eeca 100644 --- a/helm_drydock/drivers/server/__init__.py +++ b/helm_drydock/error.py @@ -12,7 +12,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -import helm_drydock.drivers.ProviderDriver - -class ServerDriver(ProviderDriver): - \ No newline at end of file +class DesignError(Exception): + pass diff --git a/helm_drydock/ingester/__init__.py b/helm_drydock/ingester/__init__.py index beae66a3..48553c2d 100644 --- a/helm_drydock/ingester/__init__.py +++ b/helm_drydock/ingester/__init__.py @@ -17,9 +17,14 @@ import logging import yaml -import helm_drydock.model as model -from helm_drydock.statemgmt import DesignState +import helm_drydock.model.site as site +import helm_drydock.model.network as network +import helm_drydock.model.hwprofile as hwprofile +import helm_drydock.model.node as node +import helm_drydock.model.hostprofile as hostprofile + +from helm_drydock.statemgmt import DesignState, SiteDesign, DesignError class Ingester(object): @@ -28,16 +33,6 @@ class Ingester(object): self.log = logging.Logger("ingester") self.registered_plugins = {} - """ - enable_plugins - - params: plugins - A list of class objects denoting the ingester plugins to be enabled - - Enable plugins that can be used for ingest_data calls. Each plugin should use - helm_drydock.ingester.plugins.IngesterPlugin as its base class. As long as one - enabled plugin successfully initializes, the call is considered successful. Otherwise - it will throw an exception - """ def enable_plugins(self, plugins=[]): if len(plugins) == 0: self.log.error("Cannot have an empty plugin list.") @@ -53,7 +48,51 @@ class Ingester(object): if len(self.registered_plugins) == 0: self.log.error("Could not enable at least one plugin") raise Exception("Could not enable at least one plugin") + """ + enable_plugins + params: plugins - A list of class objects denoting the ingester plugins to be enabled + + Enable plugins that can be used for ingest_data calls. Each plugin should use + helm_drydock.ingester.plugins.IngesterPlugin as its base class. As long as one + enabled plugin successfully initializes, the call is considered successful. Otherwise + it will throw an exception + """ + + def ingest_data(self, plugin_name='', design_state=None, **kwargs): + if design_state is None: + self.log.error("ingest_data called without valid DesignState handler") + raise Exception("Invalid design_state handler") + + # TODO this method needs refactored to handle design base vs change + + design_data = None + + try: + design_data = design_state.get_design_base() + except DesignError: + design_data = SiteDesign() + + if plugin_name in self.registered_plugins: + design_items = self.registered_plugins[plugin_name].ingest_data(**kwargs) + # Need to persist data here, but we don't yet have the statemgmt service working + for m in design_items: + if type(m) is site.Site: + design_data.add_site(m) + elif type(m) is network.Network: + design_data.add_network(m) + elif type(m) is network.NetworkLink: + design_data.add_network_link(m) + elif type(m) is hostprofile.HostProfile: + design_data.add_host_profile(m) + elif type(m) is hwprofile.HardwareProfile: + design_data.add_hardware_profile(m) + elif type(m) is node.BaremetalNode: + design_data.add_baremetal_node(m) + design_state.put_design_base(design_data) + else: + self.log.error("Could not find plugin %s to ingest data." % (plugin_name)) + raise LookupError("Could not find plugin %s" % plugin_name) """ ingest_data @@ -62,29 +101,4 @@ class Ingester(object): Execute a data ingestion using the named plugin (assuming it is enabled) """ - def ingest_data(self, plugin_name='', design_state=None, **kwargs): - if design_state is None: - self.log.error("ingest_data called without valid DesignState handler") - raise Exception("Invalid design_state handler") - - if plugin_name in self.registered_plugins: - design_data = self.registered_plugins[plugin_name].ingest_data(**kwargs) - # Need to persist data here, but we don't yet have the statemgmt service working - for m in design_data: - if type(m) is model.Site: - design_state.add_site(m) - elif type(m) is model.Network: - design_state.add_network(m) - elif type(m) is model.NetworkLink: - design_state.add_network_link(m) - elif type(m) is model.HostProfile: - design_state.add_host_profile(m) - elif type(m) is model.HardwareProfile: - design_state.add_hardware_profile(m) - elif type(m) is model.BaremetalNode: - design_state.add_baremetal_node(m) - else: - self.log.error("Could not find plugin %s to ingest data." % (plugin_name)) - raise LookupError("Could not find plugin %s" % plugin_name) - diff --git a/helm_drydock/ingester/plugins/yaml.py b/helm_drydock/ingester/plugins/yaml.py index 44664c8c..20a4d902 100644 --- a/helm_drydock/ingester/plugins/yaml.py +++ b/helm_drydock/ingester/plugins/yaml.py @@ -19,18 +19,23 @@ import yaml import logging -import helm_drydock.model as model +import helm_drydock.model.hwprofile as hwprofile +import helm_drydock.model.node as node +import helm_drydock.model.site as site +import helm_drydock.model.hostprofile as hostprofile +import helm_drydock.model.network as network + from helm_drydock.ingester.plugins import IngesterPlugin class YamlIngester(IngesterPlugin): kind_map = { - "Region": model.Site, - "NetworkLink": model.NetworkLink, - "HardwareProfile": model.HardwareProfile, - "Network": model.Network, - "HostProfile": model.HostProfile, - "BaremetalNode": model.BaremetalNode, + "Region": site.Site, + "NetworkLink": network.NetworkLink, + "HardwareProfile": hwprofile.HardwareProfile, + "Network": network.Network, + "HostProfile": hostprofile.HostProfile, + "BaremetalNode": node.BaremetalNode, } def __init__(self): diff --git a/helm_drydock/ingester/readme.md b/helm_drydock/ingester/readme.md new file mode 100644 index 00000000..1437c216 --- /dev/null +++ b/helm_drydock/ingester/readme.md @@ -0,0 +1,13 @@ +# Ingester # + +Ingester is a pluggable consumer of site design data. It +will support consuming data in different formats from +different sources. + +Each ingester plugin should be able source data +based on user-provided parameters and parse that data +into the Drydock internal model (helm_drydock.model). + +Each plugin does not need to support every type of design +data as a single site design may be federated from multiple +sources. \ No newline at end of file diff --git a/helm_drydock/introspection/readme.md b/helm_drydock/introspection/readme.md new file mode 100644 index 00000000..b62f79d5 --- /dev/null +++ b/helm_drydock/introspection/readme.md @@ -0,0 +1,9 @@ +# Introspection # + +Introspection is a cloud-init compatible metadata service +that is used to make a node self-aware. After a full +deployment by the node driver, the newly installed OS +will contact the introspection API to gain a package of +declaritive data defining the node's role in the site and +enough initial data to start the promenade process of +Kubernetes assimilation \ No newline at end of file diff --git a/helm_drydock/model/__init__.py b/helm_drydock/model/__init__.py index 05bb67d9..e682cefd 100644 --- a/helm_drydock/model/__init__.py +++ b/helm_drydock/model/__init__.py @@ -18,626 +18,6 @@ import logging from copy import deepcopy - -class HardwareProfile(object): - - def __init__(self, **kwargs): - self.log = logging.Logger('model') - - self.api_version = kwargs.get('apiVersion', '') - - if self.api_version == "v1.0": - metadata = kwargs.get('metadata', {}) - spec = kwargs.get('spec', {}) - - # Need to add validation logic, we'll assume the input is - # valid for now - self.name = metadata.get('name', '') - self.site = metadata.get('region', '') - - self.vendor = spec.get('vendor', None) - self.generation = spec.get('generation', None) - self.hw_version = spec.get('hw_version', None) - self.bios_version = spec.get('bios_version', None) - self.boot_mode = spec.get('boot_mode', None) - self.bootstrap_protocol = spec.get('bootstrap_protocol', None) - self.pxe_interface = spec.get('pxe_interface', None) - self.devices = [] - - device_aliases = spec.get('device_aliases', {}) - - pci_devices = device_aliases.get('pci', []) - scsi_devices = device_aliases.get('scsi', []) - - for d in pci_devices: - d['bus_type'] = 'pci' - self.devices.append( - HardwareDeviceAlias(self.api_version, **d)) - - for d in scsi_devices: - d['bus_type'] = 'scsi' - self.devices.append( - HardwareDeviceAlias(self.api_version, **d)) - else: - self.log.error("Unknown API version %s of %s" % - (self.api_version, self.__class__)) - raise ValueError('Unknown API version of object') - - return - - def resolve_alias(self, alias_type, alias): - selector = {} - for d in self.devices: - if d.alias == alias and d.bus_type == alias_type: - selector['address'] = d.address - selector['device_type'] = d.type - return selector - - return None - - -class HardwareDeviceAlias(object): - - def __init__(self, api_version, **kwargs): - self.log = logging.Logger('model') - - self.api_version = api_version - - if self.api_version == "v1.0": - self.bus_type = kwargs.get('bus_type', None) - self.address = kwargs.get('address', None) - self.alias = kwargs.get('alias', None) - self.type = kwargs.get('type', None) - else: - self.log.error("Unknown API version %s of %s" % - (self.api_version, self.__class__)) - raise ValueError('Unknown API version of object') - - -class Site(object): - - def __init__(self, **kwargs): - self.log = logging.Logger('model') - - self.api_version = kwargs.get('apiVersion', '') - - if self.api_version == "v1.0": - metadata = kwargs.get('metadata', {}) - - # Need to add validation logic, we'll assume the input is - # valid for now - self.name = metadata.get('name', '') - - self.networks = [] - self.network_links = [] - self.host_profiles = [] - self.hardware_profiles = [] - self.baremetal_nodes = [] - - else: - self.log.error("Unknown API version %s of %s" % - (self.api_version, self.__class__)) - raise ValueError('Unknown API version of object') - - def get_network(self, network_name): - for n in self.networks: - if n.name == network_name: - return n - - return None - - def get_network_link(self, link_name): - for l in self.network_links: - if l.name == link_name: - return l - - return None - - def get_host_profile(self, profile_name): - for p in self.host_profiles: - if p.name == profile_name: - return p - - return None - - def get_hardware_profile(self, profile_name): - for p in self.hardware_profiles: - if p.name == profile_name: - return p - - return None - - def get_baremetal_node(self, node_name): - for n in self.baremetal_nodes: - if n.name == node_name: - return n - - return None - -class NetworkLink(object): - - def __init__(self, **kwargs): - self.log = logging.Logger('model') - - self.api_version = kwargs.get('apiVersion', '') - - if self.api_version == "v1.0": - metadata = kwargs.get('metadata', {}) - spec = kwargs.get('spec', {}) - - self.name = metadata.get('name', '') - self.site = metadata.get('region', '') - - bonding = spec.get('bonding', {}) - self.bonding_mode = bonding.get('mode', 'none') - - # How should we define defaults for CIs not in the input? - if self.bonding_mode == '802.3ad': - self.bonding_xmit_hash = bonding.get('hash', 'layer3+4') - self.bonding_peer_rate = bonding.get('peer_rate', 'fast') - self.bonding_mon_rate = bonding.get('mon_rate', '100') - self.bonding_up_delay = bonding.get('up_delay', '200') - self.bonding_down_delay = bonding.get('down_delay', '200') - - self.mtu = spec.get('mtu', 1500) - self.linkspeed = spec.get('linkspeed', 'auto') - - trunking = spec.get('trunking', {}) - self.trunk_mode = trunking.get('mode', 'none') - - self.native_network = spec.get('default_network', '') - else: - self.log.error("Unknown API version %s of %s" % - (self.api_version, self.__class__)) - raise ValueError('Unknown API version of object') - - -class Network(object): - - def __init__(self, **kwargs): - self.log = logging.Logger('model') - - self.api_version = kwargs.get('apiVersion', '') - - if self.api_version == "v1.0": - metadata = kwargs.get('metadata', {}) - spec = kwargs.get('spec', {}) - - self.name = metadata.get('name', '') - self.site = metadata.get('region', '') - - self.cidr = spec.get('cidr', None) - self.allocation_strategy = spec.get('allocation', 'static') - self.vlan_id = spec.get('vlan_id', 1) - self.mtu = spec.get('mtu', 0) - - dns = spec.get('dns', {}) - self.dns_domain = dns.get('domain', 'local') - self.dns_servers = dns.get('servers', None) - - ranges = spec.get('ranges', []) - self.ranges = [] - - for r in ranges: - self.ranges.append(NetworkAddressRange(self.api_version, **r)) - - routes = spec.get('routes', []) - self.routes = [] - - for r in routes: - self.routes.append(NetworkRoute(self.api_version, **r)) - else: - self.log.error("Unknown API version %s of %s" % - (self.api_version, self.__class__)) - raise ValueError('Unknown API version of object') - - -class NetworkAddressRange(object): - - def __init__(self, api_version, **kwargs): - self.log = logging.Logger('model') - - self.api_version = api_version - - if self.api_version == "v1.0": - self.type = kwargs.get('type', None) - self.start = kwargs.get('start', None) - self.end = kwargs.get('end', None) - else: - self.log.error("Unknown API version %s of %s" % - (self.api_version, self.__class__)) - raise ValueError('Unknown API version of object') - - -class NetworkRoute(object): - - def __init__(self, api_version, **kwargs): - self.log = logging.Logger('model') - - self.api_version = api_version - - if self.api_version == "v1.0": - self.type = kwargs.get('subnet', None) - self.start = kwargs.get('gateway', None) - self.end = kwargs.get('metric', 100) - else: - self.log.error("Unknown API version %s of %s" % - (self.api_version, self.__class__)) - raise ValueError('Unknown API version of object') - - -class HostProfile(object): - - def __init__(self, **kwargs): - self.log = logging.Logger('model') - - self.api_version = kwargs.get('apiVersion', '') - - if self.api_version == "v1.0": - metadata = kwargs.get('metadata', {}) - spec = kwargs.get('spec', {}) - - self.name = metadata.get('name', '') - self.site = metadata.get('region', '') - - self.parent_profile = spec.get('host_profile', None) - self.hardware_profile = spec.get('hardware_profile', None) - - oob = spec.get('oob', {}) - self.oob_type = oob.get('type', None) - self.oob_network = oob.get('network', None) - self.oob_account = oob.get('account', None) - self.oob_credential = oob.get('credential', None) - - storage = spec.get('storage', {}) - self.storage_layout = storage.get('layout', 'lvm') - - bootdisk = storage.get('bootdisk', {}) - self.bootdisk_device = bootdisk.get('device', None) - self.bootdisk_root_size = bootdisk.get('root_size', None) - self.bootdisk_boot_size = bootdisk.get('boot_size', None) - - partitions = storage.get('partitions', []) - self.partitions = [] - - for p in partitions: - self.partitions.append(HostPartition(self.api_version, **p)) - - interfaces = spec.get('interfaces', []) - self.interfaces = [] - - for i in interfaces: - self.interfaces.append(HostInterface(self.api_version, **i)) - - metadata = spec.get('metadata', {}) - - metadata_tags = metadata.get('tags', []) - self.tags = [] - - for t in metadata_tags: - self.tags.append(t) - - owner_data = metadata.get('owner_data', {}) - self.owner_data = {} - - for k, v in owner_data.items(): - self.owner_data[k] = v - - self.rack = metadata.get('rack', None) - - else: - self.log.error("Unknown API version %s of %s" % - (self.api_version, self.__class__)) - raise ValueError('Unknown API version of object') - - def apply_inheritance(self, site): - # We return a deep copy of the profile so as not to corrupt - # the original model - self_copy = deepcopy(self) - - if self.parent_profile is None: - return self_copy - - parent = site.get_host_profile(self.parent_profile) - - if parent is None: - raise NameError("Cannot find parent profile %s for %s" - % (self.parent_profile, self.name)) - - parent = parent.apply_inheritance(site) - - # First compute inheritance for simple fields - inheritable_field_list = [ - "hardware_profile", "oob_type", "oob_network", - "oob_credential", "oob_account", "storage_layout", - "bootdisk_device", "bootdisk_root_size", "bootdisk_boot_size", - "rack"] - - for f in inheritable_field_list: - setattr(self_copy, f, - Utils.apply_field_inheritance(getattr(self, f, None), - getattr(parent, f, None))) - - # Now compute inheritance for complex types - self_copy.tags = Utils.merge_lists(self.tags, parent.tags) - - self_copy.owner_data = Utils.merge_dicts( - self.owner_data, parent.owner_data) - - self_copy.interfaces = HostInterface.merge_lists( - self.interfaces, parent.interfaces) - - self_copy.partitions = HostPartition.merge_lists( - self.partitions, parent.partitions) - - return self_copy - - -class HostInterface(object): - - def __init__(self, api_version, **kwargs): - self.log = logging.Logger('model') - - self.api_version = api_version - - if self.api_version == "v1.0": - self.device_name = kwargs.get('device_name', None) - self.network_link = kwargs.get('device_link', None) - - self.hardware_slaves = [] - slaves = kwargs.get('slaves', []) - - for s in slaves: - self.hardware_slaves.append(s) - - self.networks = [] - networks = kwargs.get('networks', []) - - for n in networks: - self.networks.append(n) - else: - self.log.error("Unknown API version %s of %s" % - (self.api_version, self.__class__)) - raise ValueError('Unknown API version of object') - - # The device attribute may be hardware alias that translates to a - # physical device address. If the device attribute does not match an - # alias, we assume it directly identifies a OS device name. When the - # apply_hardware_profile method is called on the parent Node of this - # device, the selector will be decided and applied - - def add_selector(self, sel_type, address='', dev_type=''): - if getattr(self, 'selectors', None) is None: - self.selectors = [] - - new_selector = {} - new_selector['selector_type'] = sel_type - new_selector['address'] = address - new_selector['device_type'] = dev_type - - self.selectors.append(new_selector) - - def get_slave_selectors(self): - return self.selectors - - # Return number of slaves for this interface - def get_slave_count(self): - return len(self.hardware_slaves) - - """ - Merge two lists of HostInterface models with child_list taking - priority when conflicts. If a member of child_list has a device_name - beginning with '!' it indicates that HostInterface should be - removed from the merged list - """ - - @staticmethod - def merge_lists(child_list, parent_list): - if len(child_list) == 0: - return deepcopy(parent_list) - - effective_list = [] - if len(parent_list) == 0: - for i in child_list: - if i.device_name.startswith('!'): - continue - else: - effective_list.append(deepcopy(i)) - return effective_list - - parent_interfaces = [] - for i in parent_list: - parent_name = i.device_name - parent_interfaces.append(parent_name) - add = True - for j in child_list: - if j.device_name == ("!" + parent_name): - add = False - break - elif j.device_name == parent_name: - m = HostInterface(j.api_version) - m.device_name = j.device_name - m.network_link = \ - Utils.apply_field_inheritance(j.network_link, - i.network_link) - s = filter(lambda x: ("!" + x) not in j.hardware_slaves, - i.hardware_slaves) - s = list(s) - - s.extend(filter(lambda x: not x.startswith("!"), - j.hardware_slaves)) - m.hardware_slaves = s - - n = filter(lambda x: ("!" + x) not in j.networks, - i.networks) - n = list(n) - - n.extend(filter(lambda x: not x.startswith("!"), - j.networks)) - m.networks = n - - effective_list.append(m) - add = False - break - - if add: - effective_list.append(deepcopy(i)) - - for j in child_list: - if (j.device_name not in parent_interfaces - and not j.device_name.startswith("!")): - effective_list.append(deepcopy(j)) - - return effective_list - - -class HostPartition(object): - - def __init__(self, api_version, **kwargs): - self.api_version = api_version - - if self.api_version == "v1.0": - self.name = kwargs.get('name', None) - self.device = kwargs.get('device', None) - self.part_uuid = kwargs.get('part_uuid', None) - self.size = kwargs.get('size', None) - self.mountpoint = kwargs.get('mountpoint', None) - self.fstype = kwargs.get('fstype', 'ext4') - self.mount_options = kwargs.get('mount_options', 'defaults') - self.fs_uuid = kwargs.get('fs_uuid', None) - self.fs_label = kwargs.get('fs_label', None) - else: - raise ValueError('Unknown API version of object') - - # The device attribute may be hardware alias that translates to a - # physical device address. If the device attribute does not match an - # alias, we assume it directly identifies a OS device name. When the - # apply_hardware_profile method is called on the parent Node of this - # device, the selector will be decided and applied - - def set_selector(self, sel_type, address='', dev_type=''): - selector = {} - selector['type'] = sel_type - selector['address'] = address - selector['device_type'] = dev_type - - self.selector = selector - - def get_selector(self): - return self.selector - - """ - Merge two lists of HostPartition models with child_list taking - priority when conflicts. If a member of child_list has a name - beginning with '!' it indicates that HostPartition should be - removed from the merged list - """ - - @staticmethod - def merge_lists(child_list, parent_list): - if len(child_list) == 0: - return deepcopy(parent_list) - - effective_list = [] - if len(parent_list) == 0: - for i in child_list: - if i.name.startswith('!'): - continue - else: - effective_list.append(deepcopy(i)) - - inherit_field_list = ["device", "part_uuid", "size", - "mountpoint", "fstype", "mount_options", - "fs_uuid", "fs_label"] - - parent_partitions = [] - for i in parent_list: - parent_name = i.name - parent_partitions.append(parent_name) - add = True - for j in child_list: - if j.name == ("!" + parent_name): - add = False - break - elif j.name == parent_name: - p = HostPartition(j.api_version) - p.name = j.name - - for f in inherit_field_list: - setattr(p, Utils.apply_field_inheritance(getattr(j, f), - getattr(i, f)) - ) - add = False - effective_list.append(p) - if add: - effective_list.append(deepcopy(i)) - - for j in child_list: - if j.name not in parent_list: - effective_list.append(deepcopy(j)) - - return effective_list - -# A BaremetalNode is really nothing more than a physical -# instantiation of a HostProfile, so they both represent -# the same set of CIs -class BaremetalNode(HostProfile): - - def __init__(self, **kwargs): - super(BaremetalNode, self).__init__(**kwargs) - - def apply_host_profile(self, site): - return self.apply_inheritance(site) - - # Translate device alises to physical selectors and copy - # other hardware attributes into this object - def apply_hardware_profile(self, site): - self_copy = deepcopy(self) - - if self.hardware_profile is None: - raise ValueError("Hardware profile not set") - - hw_profile = site.get_hardware_profile(self.hardware_profile) - - for i in self_copy.interfaces: - for s in i.hardware_slaves: - selector = hw_profile.resolve_alias("pci", s) - if selector is None: - i.add_selector("name", address=p.device) - else: - i.add_selector("address", address=selector['address'], - dev_type=selector['device_type']) - - for p in self_copy.partitions: - selector = hw_profile.resolve_alias("scsi", p.device) - if selector is None: - p.set_selector("name", address=p.device) - else: - p.set_selector("address", address=selector['address'], - dev_type=selector['device_type']) - - - hardware = {"vendor": getattr(hw_profile, 'vendor', None), - "generation": getattr(hw_profile, 'generation', None), - "hw_version": getattr(hw_profile, 'hw_version', None), - "bios_version": getattr(hw_profile, 'bios_version', None), - "boot_mode": getattr(hw_profile, 'boot_mode', None), - "bootstrap_protocol": getattr(hw_profile, - 'bootstrap_protocol', - None), - "pxe_interface": getattr(hw_profile, 'pxe_interface', None) - } - - self_copy.hardware = hardware - - return self_copy - - def get_interface(self, iface_name): - for i in self.interfaces: - if i.device_name == iface_name: - return i - return None - # Utility class for calculating inheritance class Utils(object): diff --git a/helm_drydock/model/hostprofile.py b/helm_drydock/model/hostprofile.py new file mode 100644 index 00000000..b239219c --- /dev/null +++ b/helm_drydock/model/hostprofile.py @@ -0,0 +1,378 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Models for helm_drydock +# +import logging + +from copy import deepcopy + +from helm_drydock.orchestrator.enum import SiteStatus +from helm_drydock.orchestrator.enum import NodeStatus +from helm_drydock.model.network import Network +from helm_drydock.model.network import NetworkLink +from helm_drydock.model import Utils + +class HostProfile(object): + + def __init__(self, **kwargs): + self.log = logging.Logger('model') + + self.api_version = kwargs.get('apiVersion', '') + + if self.api_version == "v1.0": + metadata = kwargs.get('metadata', {}) + spec = kwargs.get('spec', {}) + + self.name = metadata.get('name', '') + self.site = metadata.get('region', '') + + self.parent_profile = spec.get('host_profile', None) + self.hardware_profile = spec.get('hardware_profile', None) + + oob = spec.get('oob', {}) + self.oob_type = oob.get('type', None) + self.oob_network = oob.get('network', None) + self.oob_account = oob.get('account', None) + self.oob_credential = oob.get('credential', None) + + storage = spec.get('storage', {}) + self.storage_layout = storage.get('layout', 'lvm') + + bootdisk = storage.get('bootdisk', {}) + self.bootdisk_device = bootdisk.get('device', None) + self.bootdisk_root_size = bootdisk.get('root_size', None) + self.bootdisk_boot_size = bootdisk.get('boot_size', None) + + partitions = storage.get('partitions', []) + self.partitions = [] + + for p in partitions: + self.partitions.append(HostPartition(self.api_version, **p)) + + interfaces = spec.get('interfaces', []) + self.interfaces = [] + + for i in interfaces: + self.interfaces.append(HostInterface(self.api_version, **i)) + + node_metadata = spec.get('metadata', {}) + + metadata_tags = node_metadata.get('tags', []) + self.tags = [] + + for t in metadata_tags: + self.tags.append(t) + + owner_data = node_metadata.get('owner_data', {}) + self.owner_data = {} + + for k, v in owner_data.items(): + self.owner_data[k] = v + + self.rack = node_metadata.get('rack', None) + + else: + self.log.error("Unknown API version %s of %s" % + (self.api_version, self.__class__)) + raise ValueError('Unknown API version of object') + + def get_rack(self): + return self.rack + + def get_name(self): + return self.name + + def has_tag(self, tag): + if tag in self.tags: + return True + + return False + + def apply_inheritance(self, site): + # We return a deep copy of the profile so as not to corrupt + # the original model + self_copy = deepcopy(self) + + if self.parent_profile is None: + return self_copy + + parent = site.get_host_profile(self.parent_profile) + + if parent is None: + raise NameError("Cannot find parent profile %s for %s" + % (self.parent_profile, self.name)) + + parent = parent.apply_inheritance(site) + + # First compute inheritance for simple fields + inheritable_field_list = [ + "hardware_profile", "oob_type", "oob_network", + "oob_credential", "oob_account", "storage_layout", + "bootdisk_device", "bootdisk_root_size", "bootdisk_boot_size", + "rack"] + + for f in inheritable_field_list: + setattr(self_copy, f, + Utils.apply_field_inheritance(getattr(self, f, None), + getattr(parent, f, None))) + + # Now compute inheritance for complex types + self_copy.tags = Utils.merge_lists(self.tags, parent.tags) + + self_copy.owner_data = Utils.merge_dicts( + self.owner_data, parent.owner_data) + + self_copy.interfaces = HostInterface.merge_lists( + self.interfaces, parent.interfaces) + + self_copy.partitions = HostPartition.merge_lists( + self.partitions, parent.partitions) + + return self_copy + + +class HostInterface(object): + + def __init__(self, api_version, **kwargs): + self.log = logging.Logger('model') + + self.api_version = api_version + + if self.api_version == "v1.0": + self.device_name = kwargs.get('device_name', None) + self.network_link = kwargs.get('device_link', None) + + self.hardware_slaves = [] + slaves = kwargs.get('slaves', []) + + for s in slaves: + self.hardware_slaves.append(s) + + self.networks = [] + networks = kwargs.get('networks', []) + + for n in networks: + self.networks.append(n) + else: + self.log.error("Unknown API version %s of %s" % + (self.api_version, self.__class__)) + raise ValueError('Unknown API version of object') + + # The device attribute may be hardware alias that translates to a + # physical device address. If the device attribute does not match an + # alias, we assume it directly identifies a OS device name. When the + # apply_hardware_profile method is called on the parent Node of this + # device, the selector will be decided and applied + + def add_selector(self, sel_type, address='', dev_type=''): + if getattr(self, 'selectors', None) is None: + self.selectors = [] + + new_selector = {} + new_selector['selector_type'] = sel_type + new_selector['address'] = address + new_selector['device_type'] = dev_type + + self.selectors.append(new_selector) + + def get_slave_selectors(self): + return self.selectors + + # Return number of slaves for this interface + def get_slave_count(self): + return len(self.hardware_slaves) + + def apply_link_config(self, net_link): + if (net_link is not None and + isinstance(net_link, NetworkLink) and + net_link.name == self.network_link): + + self.attached_link = deepcopy(net_link) + return True + return False + + def apply_network_config(self, network): + if network in self.networks: + if getattr(self, 'attached_networks', None) is None: + self.attached_networks = [] + self.attached_networks.append(deepcopy(network)) + return True + else: + return False + + def set_network_address(self, network_name, address): + if getattr(self, 'attached_networks', None) is None: + return False + + for n in self.attached_neteworks: + if n.name == network_name: + n.assigned_address = address + + def get_network_configs(self): + return self.attached_networks + + """ + Merge two lists of HostInterface models with child_list taking + priority when conflicts. If a member of child_list has a device_name + beginning with '!' it indicates that HostInterface should be + removed from the merged list + """ + + @staticmethod + def merge_lists(child_list, parent_list): + if len(child_list) == 0: + return deepcopy(parent_list) + + effective_list = [] + if len(parent_list) == 0: + for i in child_list: + if i.device_name.startswith('!'): + continue + else: + effective_list.append(deepcopy(i)) + return effective_list + + parent_interfaces = [] + for i in parent_list: + parent_name = i.device_name + parent_interfaces.append(parent_name) + add = True + for j in child_list: + if j.device_name == ("!" + parent_name): + add = False + break + elif j.device_name == parent_name: + m = HostInterface(j.api_version) + m.device_name = j.device_name + m.network_link = \ + Utils.apply_field_inheritance(j.network_link, + i.network_link) + s = filter(lambda x: ("!" + x) not in j.hardware_slaves, + i.hardware_slaves) + s = list(s) + + s.extend(filter(lambda x: not x.startswith("!"), + j.hardware_slaves)) + m.hardware_slaves = s + + n = filter(lambda x: ("!" + x) not in j.networks, + i.networks) + n = list(n) + + n.extend(filter(lambda x: not x.startswith("!"), + j.networks)) + m.networks = n + + effective_list.append(m) + add = False + break + + if add: + effective_list.append(deepcopy(i)) + + for j in child_list: + if (j.device_name not in parent_interfaces + and not j.device_name.startswith("!")): + effective_list.append(deepcopy(j)) + + return effective_list + + +class HostPartition(object): + + def __init__(self, api_version, **kwargs): + self.api_version = api_version + + if self.api_version == "v1.0": + self.name = kwargs.get('name', None) + self.device = kwargs.get('device', None) + self.part_uuid = kwargs.get('part_uuid', None) + self.size = kwargs.get('size', None) + self.mountpoint = kwargs.get('mountpoint', None) + self.fstype = kwargs.get('fstype', 'ext4') + self.mount_options = kwargs.get('mount_options', 'defaults') + self.fs_uuid = kwargs.get('fs_uuid', None) + self.fs_label = kwargs.get('fs_label', None) + else: + raise ValueError('Unknown API version of object') + + # The device attribute may be hardware alias that translates to a + # physical device address. If the device attribute does not match an + # alias, we assume it directly identifies a OS device name. When the + # apply_hardware_profile method is called on the parent Node of this + # device, the selector will be decided and applied + + def set_selector(self, sel_type, address='', dev_type=''): + selector = {} + selector['type'] = sel_type + selector['address'] = address + selector['device_type'] = dev_type + + self.selector = selector + + def get_selector(self): + return self.selector + + """ + Merge two lists of HostPartition models with child_list taking + priority when conflicts. If a member of child_list has a name + beginning with '!' it indicates that HostPartition should be + removed from the merged list + """ + + @staticmethod + def merge_lists(child_list, parent_list): + if len(child_list) == 0: + return deepcopy(parent_list) + + effective_list = [] + if len(parent_list) == 0: + for i in child_list: + if i.name.startswith('!'): + continue + else: + effective_list.append(deepcopy(i)) + + inherit_field_list = ["device", "part_uuid", "size", + "mountpoint", "fstype", "mount_options", + "fs_uuid", "fs_label"] + + parent_partitions = [] + for i in parent_list: + parent_name = i.name + parent_partitions.append(parent_name) + add = True + for j in child_list: + if j.name == ("!" + parent_name): + add = False + break + elif j.name == parent_name: + p = HostPartition(j.api_version) + p.name = j.name + + for f in inherit_field_list: + setattr(p, Utils.apply_field_inheritance(getattr(j, f), + getattr(i, f)) + ) + add = False + effective_list.append(p) + if add: + effective_list.append(deepcopy(i)) + + for j in child_list: + if j.name not in parent_list: + effective_list.append(deepcopy(j)) + + return effective_list diff --git a/helm_drydock/model/hwprofile.py b/helm_drydock/model/hwprofile.py new file mode 100644 index 00000000..cb8d3779 --- /dev/null +++ b/helm_drydock/model/hwprofile.py @@ -0,0 +1,95 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Models for helm_drydock +# +import logging + +from copy import deepcopy + +from helm_drydock.orchestrator.enum import SiteStatus +from helm_drydock.orchestrator.enum import NodeStatus + +class HardwareProfile(object): + + def __init__(self, **kwargs): + self.log = logging.Logger('model') + + self.api_version = kwargs.get('apiVersion', '') + + if self.api_version == "v1.0": + metadata = kwargs.get('metadata', {}) + spec = kwargs.get('spec', {}) + + # Need to add validation logic, we'll assume the input is + # valid for now + self.name = metadata.get('name', '') + self.site = metadata.get('region', '') + + self.vendor = spec.get('vendor', None) + self.generation = spec.get('generation', None) + self.hw_version = spec.get('hw_version', None) + self.bios_version = spec.get('bios_version', None) + self.boot_mode = spec.get('boot_mode', None) + self.bootstrap_protocol = spec.get('bootstrap_protocol', None) + self.pxe_interface = spec.get('pxe_interface', None) + self.devices = [] + + device_aliases = spec.get('device_aliases', {}) + + pci_devices = device_aliases.get('pci', []) + scsi_devices = device_aliases.get('scsi', []) + + for d in pci_devices: + d['bus_type'] = 'pci' + self.devices.append( + HardwareDeviceAlias(self.api_version, **d)) + + for d in scsi_devices: + d['bus_type'] = 'scsi' + self.devices.append( + HardwareDeviceAlias(self.api_version, **d)) + else: + self.log.error("Unknown API version %s of %s" % + (self.api_version, self.__class__)) + raise ValueError('Unknown API version of object') + + return + + def resolve_alias(self, alias_type, alias): + selector = {} + for d in self.devices: + if d.alias == alias and d.bus_type == alias_type: + selector['address'] = d.address + selector['device_type'] = d.type + return selector + + return None + +class HardwareDeviceAlias(object): + + def __init__(self, api_version, **kwargs): + self.log = logging.Logger('model') + + self.api_version = api_version + + if self.api_version == "v1.0": + self.bus_type = kwargs.get('bus_type', None) + self.address = kwargs.get('address', None) + self.alias = kwargs.get('alias', None) + self.type = kwargs.get('type', None) + else: + self.log.error("Unknown API version %s of %s" % + (self.api_version, self.__class__)) + raise ValueError('Unknown API version of object') diff --git a/helm_drydock/model/network.py b/helm_drydock/model/network.py new file mode 100644 index 00000000..ef784114 --- /dev/null +++ b/helm_drydock/model/network.py @@ -0,0 +1,133 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Models for helm_drydock +# +import logging + +from copy import deepcopy + +from helm_drydock.orchestrator.enum import SiteStatus +from helm_drydock.orchestrator.enum import NodeStatus + +class NetworkLink(object): + + def __init__(self, **kwargs): + self.log = logging.Logger('model') + + self.api_version = kwargs.get('apiVersion', '') + + if self.api_version == "v1.0": + metadata = kwargs.get('metadata', {}) + spec = kwargs.get('spec', {}) + + self.name = metadata.get('name', '') + self.site = metadata.get('region', '') + + bonding = spec.get('bonding', {}) + self.bonding_mode = bonding.get('mode', 'none') + + # How should we define defaults for CIs not in the input? + if self.bonding_mode == '802.3ad': + self.bonding_xmit_hash = bonding.get('hash', 'layer3+4') + self.bonding_peer_rate = bonding.get('peer_rate', 'fast') + self.bonding_mon_rate = bonding.get('mon_rate', '100') + self.bonding_up_delay = bonding.get('up_delay', '200') + self.bonding_down_delay = bonding.get('down_delay', '200') + + self.mtu = spec.get('mtu', 1500) + self.linkspeed = spec.get('linkspeed', 'auto') + + trunking = spec.get('trunking', {}) + self.trunk_mode = trunking.get('mode', 'none') + + self.native_network = spec.get('default_network', '') + else: + self.log.error("Unknown API version %s of %s" % + (self.api_version, self.__class__)) + raise ValueError('Unknown API version of object') + + +class Network(object): + + def __init__(self, **kwargs): + self.log = logging.Logger('model') + + self.api_version = kwargs.get('apiVersion', '') + + if self.api_version == "v1.0": + metadata = kwargs.get('metadata', {}) + spec = kwargs.get('spec', {}) + + self.name = metadata.get('name', '') + self.site = metadata.get('region', '') + + self.cidr = spec.get('cidr', None) + self.allocation_strategy = spec.get('allocation', 'static') + self.vlan_id = spec.get('vlan_id', 1) + self.mtu = spec.get('mtu', 0) + + dns = spec.get('dns', {}) + self.dns_domain = dns.get('domain', 'local') + self.dns_servers = dns.get('servers', None) + + ranges = spec.get('ranges', []) + self.ranges = [] + + for r in ranges: + self.ranges.append(NetworkAddressRange(self.api_version, **r)) + + routes = spec.get('routes', []) + self.routes = [] + + for r in routes: + self.routes.append(NetworkRoute(self.api_version, **r)) + else: + self.log.error("Unknown API version %s of %s" % + (self.api_version, self.__class__)) + raise ValueError('Unknown API version of object') + + +class NetworkAddressRange(object): + + def __init__(self, api_version, **kwargs): + self.log = logging.Logger('model') + + self.api_version = api_version + + if self.api_version == "v1.0": + self.type = kwargs.get('type', None) + self.start = kwargs.get('start', None) + self.end = kwargs.get('end', None) + else: + self.log.error("Unknown API version %s of %s" % + (self.api_version, self.__class__)) + raise ValueError('Unknown API version of object') + + +class NetworkRoute(object): + + def __init__(self, api_version, **kwargs): + self.log = logging.Logger('model') + + self.api_version = api_version + + if self.api_version == "v1.0": + self.type = kwargs.get('subnet', None) + self.start = kwargs.get('gateway', None) + self.end = kwargs.get('metric', 100) + else: + self.log.error("Unknown API version %s of %s" % + (self.api_version, self.__class__)) + raise ValueError('Unknown API version of object') diff --git a/helm_drydock/model/node.py b/helm_drydock/model/node.py new file mode 100644 index 00000000..22d41b2e --- /dev/null +++ b/helm_drydock/model/node.py @@ -0,0 +1,154 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Models for helm_drydock +# +import logging + +from copy import deepcopy + + +from helm_drydock.orchestrator.enum import SiteStatus +from helm_drydock.orchestrator.enum import NodeStatus +from helm_drydock.model.hostprofile import HostProfile +from helm_drydock.model import Utils + +class BaremetalNode(HostProfile): + + # A BaremetalNode is really nothing more than a physical + # instantiation of a HostProfile, so they both represent + # the same set of CIs + def __init__(self, **kwargs): + super(BaremetalNode, self).__init__(**kwargs) + + if self.api_version == "v1.0": + self.addressing = [] + + spec = kwargs.get('spec', {}) + addresses = spec.get('addressing', []) + + if len(addresses) == 0: + raise ValueError('BaremetalNode needs at least' \ + ' 1 assigned address') + for a in addresses: + assignment = {} + address = a.get('address', '') + if address == 'dhcp': + assignment['type'] = 'dhcp' + assignment['address'] = None + assignment['network'] = a.get('network') + self.addressing.append(assignment) + elif address != '': + assignment['type'] = 'static' + assignment['address'] = a.get('address') + assignment['network'] = a.get('network') + self.addressing.append(assignment) + else: + self.log.error("Invalid address assignment %s on Node %s" + % (address, self.name)) + + self.build = kwargs.get('build', {}) + + def start_build(self): + if self.build.get('status','') == '': + self.build['status'] = NodeStatus.Unknown + + def apply_host_profile(self, site): + return self.apply_inheritance(site) + + # Translate device alises to physical selectors and copy + # other hardware attributes into this object + def apply_hardware_profile(self, site): + self_copy = deepcopy(self) + + if self.hardware_profile is None: + raise ValueError("Hardware profile not set") + + hw_profile = site.get_hardware_profile(self.hardware_profile) + + for i in self_copy.interfaces: + for s in i.hardware_slaves: + selector = hw_profile.resolve_alias("pci", s) + if selector is None: + i.add_selector("name", address=p.device) + else: + i.add_selector("address", address=selector['address'], + dev_type=selector['device_type']) + + for p in self_copy.partitions: + selector = hw_profile.resolve_alias("scsi", p.device) + if selector is None: + p.set_selector("name", address=p.device) + else: + p.set_selector("address", address=selector['address'], + dev_type=selector['device_type']) + + + hardware = {"vendor": getattr(hw_profile, 'vendor', None), + "generation": getattr(hw_profile, 'generation', None), + "hw_version": getattr(hw_profile, 'hw_version', None), + "bios_version": getattr(hw_profile, 'bios_version', None), + "boot_mode": getattr(hw_profile, 'boot_mode', None), + "bootstrap_protocol": getattr(hw_profile, + 'bootstrap_protocol', + None), + "pxe_interface": getattr(hw_profile, 'pxe_interface', None) + } + + self_copy.hardware = hardware + + return self_copy + + def apply_network_connections(self, site): + self_copy = deepcopy(self) + + for n in site.network_links: + for i in self_copy.interfaces: + i.apply_link_config(n) + + for n in site.networks: + for i in self_copy.interfaces: + i.apply_network_config(n) + + for a in self_copy.addressing: + for i in self_copy.interfaces: + i.set_network_address(a.get('network'), a.get('address')) + + return self_copy + + def get_interface(self, iface_name): + for i in self.interfaces: + if i.device_name == iface_name: + return i + return None + + def get_status(self): + return self.build['status'] + + def set_status(self, status): + if isinstance(status, NodeStatus): + self.build['status'] = status + + def get_last_build_action(self): + return self.build.get('last_action', None) + + def set_last_build_action(self, action, result, detail=None): + last_action = self.build.get('last_action', None) + if last_action is None: + self.build['last_action'] = {} + last_action = self.build['last_action'] + last_action['action'] = action + last_action['result'] = result + if detail is not None: + last_action['detail'] = detail diff --git a/helm_drydock/model/site.py b/helm_drydock/model/site.py new file mode 100644 index 00000000..0bddb45a --- /dev/null +++ b/helm_drydock/model/site.py @@ -0,0 +1,122 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Models for helm_drydock +# +import logging + +from copy import deepcopy + +from helm_drydock.orchestrator.enum import SiteStatus +from helm_drydock.orchestrator.enum import NodeStatus + +class Site(object): + + def __init__(self, **kwargs): + self.log = logging.Logger('model') + + if kwargs is None: + raise ValueError("Empty arguments") + + self.api_version = kwargs.get('apiVersion', '') + + self.build = kwargs.get('build', {}) + + if self.api_version == "v1.0": + metadata = kwargs.get('metadata', {}) + + # Need to add validation logic, we'll assume the input is + # valid for now + self.name = metadata.get('name', '') + + spec = kwargs.get('spec', {}) + + self.tag_definitions = [] + tag_defs = spec.get('tag_definitions', []) + + for t in tag_defs: + self.tag_definitions.append( + NodeTagDefinition(self.api_version, **t)) + + self.networks = [] + self.network_links = [] + self.host_profiles = [] + self.hardware_profiles = [] + self.baremetal_nodes = [] + + else: + self.log.error("Unknown API version %s of %s" % + (self.api_version, self.__class__)) + raise ValueError('Unknown API version of object') + + def start_build(self): + if self.build.get('status', '') == '': + self.build['status'] = SiteStatus.Unknown + + def get_network(self, network_name): + for n in self.networks: + if n.name == network_name: + return n + + return None + + def get_network_link(self, link_name): + for l in self.network_links: + if l.name == link_name: + return l + + return None + + def get_host_profile(self, profile_name): + for p in self.host_profiles: + if p.name == profile_name: + return p + + return None + + def get_hardware_profile(self, profile_name): + for p in self.hardware_profiles: + if p.name == profile_name: + return p + + return None + + def get_baremetal_node(self, node_name): + for n in self.baremetal_nodes: + if n.name == node_name: + return n + + return None + + def set_status(self, status): + if isinstance(status, SiteStatus): + self.build['status'] = status + +class NodeTagDefinition(object): + + def __init__(self, api_version, **kwargs): + self.api_version = api_version + + if self.api_version == "v1.0": + self.tag = kwargs.get('tag', '') + self.definition_type = kwargs.get('definition_type', '') + self.definition = kwargs.get('definition', '') + + if self.definition_type not in ['lshw_xpath']: + raise ValueError('Unknown definition type in ' \ + 'NodeTagDefinition: %s' % (self.definition_type)) + else: + self.log.error("Unknown API version %s of %s" % + (self.api_version, self.__class__)) + raise ValueError('Unknown API version of object') \ No newline at end of file diff --git a/helm_drydock/orchestrator/__init__.py b/helm_drydock/orchestrator/__init__.py index 2a385a45..6ae6f0fb 100644 --- a/helm_drydock/orchestrator/__init__.py +++ b/helm_drydock/orchestrator/__init__.py @@ -10,4 +10,62 @@ # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and -# limitations under the License. \ No newline at end of file +# limitations under the License. +from enum import Enum, unique + +import uuid + +class Orchestrator(object): + + # enabled_drivers is a map which provider drivers + # should be enabled for use by this orchestrator + + def __init__(self, enabled_drivers=None, design_state=None): + self.enabled_drivers = {} + + self.enabled_drivers['oob'] = enabled_drivers.get('oob', None) + self.enabled_drivers['server'] = enabled_drivers.get('server', None) + self.enabled_drivers['network'] = enabled_drivers.get('network', None) + + self.design_state = design_state + + """ + execute_task + + This is the core of the orchestrator. The task will describe the action + to take and the context/scope of the command. We will then source + the current designed state and current built state from the statemgmt + module. Based on those 3 inputs, we'll decide what is needed next. + """ + def execute_task(self, task): + if design_state is None: + raise Exception("Cannot execute task without initialized state manager") + + +class OrchestrationTask(object): + + def __init__(self, action, **kwargs): + self.taskid = uuid.uuid4() + + self.action = action + + parent_task = kwargs.get('parent_task','') + + # Validate parameters based on action + self.site = kwargs.get('site', '') + + + if self.site == '': + raise ValueError("Task requires 'site' parameter") + + if action in [Action.VerifyNode, Action.PrepareNode, + Action.DeployNode, Action.DestroyNode]: + self.node_filter = kwargs.get('node_filter', None) + + def child_task(self, action, **kwargs): + child_task = OrchestrationTask(action, parent_task=self.taskid, site=self.site, **kwargs) + return child_task + + + + diff --git a/helm_drydock/orchestrator/designdata.py b/helm_drydock/orchestrator/designdata.py index a26ebb1d..bc2cfbba 100644 --- a/helm_drydock/orchestrator/designdata.py +++ b/helm_drydock/orchestrator/designdata.py @@ -16,6 +16,8 @@ import logging from copy import deepcopy +from helm_drydock.error import DesignError + class DesignStateClient(object): def __init__(self): @@ -31,34 +33,47 @@ class DesignStateClient(object): return a Site model populated with all components from the design state """ - def load_design_data(self, site_name, design_state=None): - site = design_state.get_site(site_name) + def load_design_data(self, site_name, design_state=None, change_id=None): + if design_state is None: + raise ValueError("Design state is None") - networks = design_state.get_networks() + design_data = None + + if change_id is None: + try: + design_data = design_state.get_design_base() + except DesignError(e): + raise e + else: + design_data = design_state.get_design_change(change_id) + + site = design_data.get_site(site_name) + + networks = design_data.get_networks() for n in networks: if n.site == site_name: site.networks.append(n) - network_links = design_state.get_network_links() + network_links = design_data.get_network_links() for l in network_links: if l.site == site_name: site.network_links.append(l) - host_profiles = design_state.get_host_profiles() + host_profiles = design_data.get_host_profiles() for p in host_profiles: if p.site == site_name: site.host_profiles.append(p) - hardware_profiles = design_state.get_hardware_profiles() + hardware_profiles = design_data.get_hardware_profiles() for p in hardware_profiles: if p.site == site_name: site.hardware_profiles.append(p) - baremetal_nodes = design_state.get_baremetal_nodes() + baremetal_nodes = design_data.get_baremetal_nodes() for n in baremetal_nodes: if n.site == site_name: @@ -66,12 +81,6 @@ class DesignStateClient(object): return site - """ - compute_model_inheritance - given a fully populated Site model, compute the effecitve - design by applying inheritance and references - - return a Site model reflecting the effective design for the site - """ def compute_model_inheritance(self, site_root): # For now the only thing that really incorporates inheritance is @@ -86,8 +95,15 @@ class DesignStateClient(object): for n in site_copy.baremetal_nodes: resolved = n.apply_host_profile(site_copy) resolved = resolved.apply_hardware_profile(site_copy) + resolved = resolved.apply_network_connections(site_copy) effective_nodes.append(resolved) site_copy.baremetal_nodes = effective_nodes return site_copy + """ + compute_model_inheritance - given a fully populated Site model, + compute the effecitve design by applying inheritance and references + + return a Site model reflecting the effective design for the site + """ diff --git a/helm_drydock/orchestrator/enum.py b/helm_drydock/orchestrator/enum.py new file mode 100644 index 00000000..ffbf958a --- /dev/null +++ b/helm_drydock/orchestrator/enum.py @@ -0,0 +1,64 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from enum import Enum, unique + +@unique +class Action(Enum): + Noop = 'noop' + ValidateDesign = 'validate_design' + VerifySite = 'verify_site' + PrepareSite = 'prepare_site' + VerifyNode = 'verify_node' + PrepareNode = 'prepare_node' + DeployNode = 'deploy_node' + DestroyNode = 'destroy_node' + +@unique +class ActionResult(Enum): + Success = 'success' + PartialSuccess = 'partial_success' + Failure = 'failure' + DependentFailure = 'dependent_failure' + +@unique +class SiteStatus(Enum): + Unknown = 'unknown' + DesignStarted = 'design_started' + DesignAvailable = 'design_available' + DesignValidated = 'design_validated' + Deploying = 'deploying' + Deployed = 'deployed' + DesignUpdated = 'design_updated' + +@unique +class NodeStatus(Enum): + Unknown = 'unknown' + Designed = 'designed' + Present = 'present' # IPMI access verified + BasicVerifying = 'basic_verifying' # Base node verification in process + FailedBasicVerify = 'failed_basic_verify' # Base node verification failed + BasicVerified = 'basic_verified' # Base node verification successful + Preparing = 'preparing' # Node preparation in progress + FailedPrepare = 'failed_prepare' # Node preparation failed + Prepared = 'prepared' # Node preparation complete + FullyVerifying = 'fully_verifying' # Node full verification in progress + FailedFullVerify = 'failed_full_verify' # Node full verification failed + FullyVerified = 'fully_verified' # Deeper verification successful + Deploying = 'deploy' # Node deployment in progress + FailedDeploy = 'failed_deploy' # Node deployment failed + Deployed = 'deployed' # Node deployed successfully + Bootstrapping = 'bootstrapping' # Node bootstrapping + FailedBootstrap = 'failed_bootstrap' # Node bootstrapping failed + Bootstrapped = 'bootstrapped' # Node fully bootstrapped + Complete = 'complete' # Node is complete diff --git a/helm_drydock/orchestrator/readme.md b/helm_drydock/orchestrator/readme.md new file mode 100644 index 00000000..fd11755f --- /dev/null +++ b/helm_drydock/orchestrator/readme.md @@ -0,0 +1,83 @@ +# Orchestrator # + +The orchestrator is the core of drydock and will manage +the ordering of driver actions to implement the main Drydock +actions. Each of these actions will be started by the +external cLCP orchestrator with different parameters to +control the scope of the action. + +Orchestrator should persist the state of each task +such that on failure the task can retried and only the +steps needed will be executed. + +Bullet points listed below are not exhaustive and will +change as we move through testing + +## ValidateDesign ## + +Load design data from the statemgmt persistent store and +validate that the current state of design data represents +a valid site design. No claim is made that the design data +is compatible with the physical state of the site. + +## VerifySite ## + +Verify site-wide resources are in a useful state + +* Driver downstream resources are reachable (e.g. MaaS) +* OS images needed for bootstrapping are available +* Promenade or other next-step services are up and available +* Verify credentials are available + +## PrepareSite ## + +Begin preparing site-wide resources for bootstrapping. This +action will lock site design data for changes. + +* Configure bootstrapper with site network configs +* Shuffle images so they are correctly configured for bootstrapping + +## VerifyNode ## + +Verification of per-node configurations within the context +of the current node status + +* Status: Present + * Basic hardware verification as available via OOB driver + - BIOS firmware + - PCI layout + - Drives + - Hardware alarms + * IPMI connectivity +* Status: Prepared + - Full hardware manifest + - Possibly network connectivity + - Firmware versions + +## PrepareNode ## + +Prepare a node for bootstrapping + +* Configure network port for PXE +* Configure a node for PXE boot +* Power-cycle the node +* Setup commissioning configuration + - Hardware drivers + - Hardware configuration (e.g. RAID) +* Configure node networking +* Configure node storage + +## DeployNode ## + +Begin bootstrapping the node and monitor +success + +* Initialize the Introspection service for the node +* Bootstrap the node (i.e. Write persistent OS install) +* Ensure network port is returned to production configuration +* Reboot node from local disk +* Monitor platform bootstrapping + +## DestroyNode ## + +Destroy current node configuration and rebootstrap from scratch \ No newline at end of file diff --git a/helm_drydock/statemgmt/__init__.py b/helm_drydock/statemgmt/__init__.py index 2fb13a51..b630cdfd 100644 --- a/helm_drydock/statemgmt/__init__.py +++ b/helm_drydock/statemgmt/__init__.py @@ -11,23 +11,132 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from copy import deepcopy +from datetime import datetime +from datetime import timezone -import helm_drydock.model as model +import uuid + +import helm_drydock.model.node as node +import helm_drydock.model.hostprofile as hostprofile +import helm_drydock.model.network as network +import helm_drydock.model.site as site +import helm_drydock.model.hwprofile as hwprofile + +from helm_drydock.error import DesignError class DesignState(object): def __init__(self): + self.design_base = None + + self.design_changes = [] + + self.builds = [] + return + + # TODO Need to lock a design base or change once implementation + # has started + def get_design_base(self): + if self.design_base is None: + raise DesignError("No design base submitted") + + return deepcopy(self.design_base) + + def post_design_base(self, site_design): + if site_design is not None and isinstance(site_design, SiteDesign): + self.design_base = deepcopy(site_design) + return True + + def put_design_base(self, site_design): + # TODO Support merging + if site_design is not None and isinstance(site_design, SiteDesign): + self.design_base = deepcopy(site_design) + return True + + def get_design_change(self, changeid): + match = [x for x in self.design_changes if x.changeid == changeid] + + if len(match) == 0: + raise DesignError("No design change %s found." % (changeid)) + else: + return deepcopy(match[0]) + + def post_design_change(self, site_design): + if site_design is not None and isinstance(site_design, SiteDesign): + exists = [(x) for x + in self.design_changes + if x.changeid == site_design.changeid] + if len(exists) > 0: + raise DesignError("Existing change %s found" % + (site_design.changeid)) + self.design_changes.append(deepcopy(site_design)) + return True + else: + raise DesignError("Design change must be a SiteDesign instance") + + def put_design_change(self, site_design): + # TODO Support merging + if site_design is not None and isinstance(site_design, SiteDesign): + design_copy = deepcopy(site_design) + self.design_changes = [design_copy + if x.changeid == design_copy.changeid + else x + for x + in self.design_changes] + return True + else: + raise DesignError("Design change must be a SiteDesign instance") + + def get_current_build(self): + latest_stamp = 0 + current_build = None + + for b in self.builds: + if b.build_id > latest_stamp: + latest_stamp = b.build_id + current_build = b + + return deepcopy(current_build) + + def get_build(self, build_id): + for b in self.builds: + if b.build_id == build_id: + return b + + return None + + def post_build(self, site_build): + if site_build is not None and isinstance(site_build, SiteBuild): + exists = [b for b in self.builds + if b.build_id == site_build.build_id] + + if len(exists) > 0: + raise DesignError("Already a site build with ID %s" % + (str(site_build.build_id))) + else: + self.builds.append(deepcopy(site_build)) + return True + + +class SiteDesign(object): + + def __init__(self, ischange=False): + if ischange: + self.changeid = uuid.uuid4() + else: + self.changeid = 0 + self.sites = [] self.networks = [] self.network_links = [] self.host_profiles = [] self.hardware_profiles = [] self.baremetal_nodes = [] - return def add_site(self, new_site): - if new_site is None or not isinstance(new_site, model.Site): - raise Exception("Invalid Site model") + if new_site is None or not isinstance(new_site, site.Site): + raise DesignError("Invalid Site model") self.sites.append(new_site) @@ -39,11 +148,11 @@ class DesignState(object): if s.name == site_name: return s - raise NameError("Site %s not found in design state" % site_name) + raise DesignError("Site %s not found in design state" % site_name) def add_network(self, new_network): - if new_network is None or not isinstance(new_network, model.Network): - raise Exception("Invalid Network model") + if new_network is None or not isinstance(new_network, network.Network): + raise DesignError("Invalid Network model") self.networks.append(new_network) @@ -55,11 +164,13 @@ class DesignState(object): if n.name == network_name: return n - raise NameError("Network %s not found in design state" % network_name) + raise DesignError("Network %s not found in design state" + % network_name) def add_network_link(self, new_network_link): - if new_network_link is None or not isinstance(new_network_link, model.NetworkLink): - raise Exception("Invalid NetworkLink model") + if new_network_link is None or not isinstance(new_network_link, + network.NetworkLink): + raise DesignError("Invalid NetworkLink model") self.network_links.append(new_network_link) @@ -71,11 +182,13 @@ class DesignState(object): if l.name == link_name: return l - raise NameError("NetworkLink %s not found in design state" % link_name) + raise DesignError("NetworkLink %s not found in design state" + % link_name) def add_host_profile(self, new_host_profile): - if new_host_profile is None or not isinstance(new_host_profile, model.HostProfile): - raise Exception("Invalid HostProfile model") + if new_host_profile is None or not isinstance(new_host_profile, + hostprofile.HostProfile): + raise DesignError("Invalid HostProfile model") self.host_profiles.append(new_host_profile) @@ -87,11 +200,13 @@ class DesignState(object): if p.name == profile_name: return p - raise NameError("HostProfile %s not found in design state" % profile_name) + raise DesignError("HostProfile %s not found in design state" + % profile_name) def add_hardware_profile(self, new_hardware_profile): - if new_hardware_profile is None or not isinstance(new_hardware_profile, model.HardwareProfile): - raise Exception("Invalid HardwareProfile model") + if (new_hardware_profile is None or + not isinstance(new_hardware_profile, hwprofile.HardwareProfile)): + raise DesignError("Invalid HardwareProfile model") self.hardware_profiles.append(new_hardware_profile) @@ -103,11 +218,13 @@ class DesignState(object): if p.name == profile_name: return p - raise NameError("HardwareProfile %s not found in design state" % profile_name) + raise DesignError("HardwareProfile %s not found in design state" + % profile_name) def add_baremetal_node(self, new_baremetal_node): - if new_baremetal_node is None or not isinstance(new_baremetal_node, model.BaremetalNode): - raise Exception("Invalid BaremetalNode model") + if (new_baremetal_node is None or + not isinstance(new_baremetal_node, node.BaremetalNode)): + raise DesignError("Invalid BaremetalNode model") self.baremetal_nodes.append(new_baremetal_node) @@ -119,4 +236,59 @@ class DesignState(object): if n.name == node_name: return n - raise NameError("BaremetalNode %s not found in design state" % node_name) + raise DesignError("BaremetalNode %s not found in design state" + % node_name) + + +class SiteBuild(SiteDesign): + + def __init__(self, build_id=None): + super(SiteBuild, self).__init__() + + if build_id is None: + self.build_id = datetime.datetime.now(timezone.utc).timestamp() + else: + self.build_id = build_id + + def get_filtered_nodes(self, node_filter): + effective_nodes = self.get_baremetal_nodes() + + # filter by rack + rack_filter = node_filter.get('rackname', None) + + if rack_filter is not None: + rack_list = rack_filter.split(',') + effective_nodes = [x + for x in effective_nodes + if x.get_rack() in rack_list] + # filter by name + name_filter = node_filter.get('nodename', None) + + if name_filter is not None: + name_list = name_filter.split(',') + effective_nodes = [x + for x in effective_nodes + if x.get_name() in name_list] + # filter by tag + tag_filter = node_filter.get('tags', None) + + if tag_filter is not None: + tag_list = tag_filter.split(',') + effective_nodes = [x + for x in effective_nodes + for t in tag_list + if x.has_tag(t)] + + return effective_nodes + """ + Support filtering on rack name, node name or node tag + for now. Each filter can be a comma-delimited list of + values. The final result is an intersection of all the + filters + """ + + def set_nodes_status(self, node_filter, status): + target_nodes = self.get_filtered_nodes(node_filter) + + for n in target_nodes: + n.set_status(status) diff --git a/helm_drydock/statemgmt/readme.md b/helm_drydock/statemgmt/readme.md new file mode 100644 index 00000000..85e4e0d5 --- /dev/null +++ b/helm_drydock/statemgmt/readme.md @@ -0,0 +1,41 @@ +# Statemgmt # + +Statemgmt is the interface to the persistence store +for holding site design data as well as build status data + +/drydock - Base namespace for drydock data + +## As Designed ## + +Serialization of Drydock internal model as ingested. Not externally writable. + +/drydock/design +/drydock/design/base - The base site design used for the first deployment +/drydock/design/[changeID] - A change layer on top of the base site design. Chrono ordered + +## As Built ## + +Serialization of Drydock internal model as rendered to effective implementation including build status. Not externally writable. + +/drydock/build +/drydock/build/[datestamp] - A point-in-time view of what was deployed with deployment results + +## Node data ## + +Per-node data that can drive introspection as well as accept updates from nodes + +/drydock/nodedata +/drydock/nodedata/[nodename] - Per-node data, can be seeded with Ingested metadata but can be updated during by orchestrator or node + +## Service data ## + +Per-service data (may not be needed) + +/drydock/servicedata +/drydock/servicedata/[servicename] + +## Global data ## + +Generic k:v store that can be produced/consumed by non-Drydock services via the Drydock API + +/drydock/globaldata \ No newline at end of file diff --git a/helm_drydock/tox.ini b/helm_drydock/tox.ini deleted file mode 100644 index ae414018..00000000 --- a/helm_drydock/tox.ini +++ /dev/null @@ -1,11 +0,0 @@ -[tox] -envlist = py35 - -[testenv] -deps= - -rrequirements.txt -setenv= - PYTHONWARNING=all - -[flake8] -ignore=E302,H306 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 9c39a3da..1a33ec11 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,5 @@ -PyYAML -oauth -requests-oauthlib -pyipmi -netaddr -pecan -python-libmaas \ No newline at end of file +PyYAML==3.12 +oauth==1.0.1 +requests-oauthlib==0.8.0 +netaddr==0.7.19 +python-libmaas==0.4.1 \ No newline at end of file diff --git a/setup.py b/setup.py index c16c862c..69a958b3 100644 --- a/setup.py +++ b/setup.py @@ -50,7 +50,7 @@ setup(name='helm_drydock', 'PyYAML', 'oauth', 'requests-oauthlib', - 'pyipmi', + 'pyghmi', 'netaddr', 'pecan', 'webob' diff --git a/tests/test_design_inheritance.py b/tests/test_design_inheritance.py index 8691646d..0e13e8f1 100644 --- a/tests/test_design_inheritance.py +++ b/tests/test_design_inheritance.py @@ -13,7 +13,7 @@ # limitations under the License. from helm_drydock.ingester import Ingester -from helm_drydock.statemgmt import DesignState +from helm_drydock.statemgmt import DesignState, SiteDesign from helm_drydock.orchestrator.designdata import DesignStateClient from copy import deepcopy @@ -34,6 +34,11 @@ class TestClass(object): client = DesignStateClient() design_data = client.load_design_data("sitename", design_state=loaded_design) + + assert len(design_data.baremetal_nodes) == 2 + + print(yaml.dump(design_data, default_flow_style=False)) + design_data = client.compute_model_inheritance(design_data) node = design_data.get_baremetal_node("controller01") @@ -54,13 +59,15 @@ class TestClass(object): def loaded_design(self, input_files): input_file = input_files.join("fullsite.yaml") - module_design_state = DesignState() + design_state = DesignState() + design_data = SiteDesign() + design_state.post_design_base(design_data) ingester = Ingester() ingester.enable_plugins([helm_drydock.ingester.plugins.yaml.YamlIngester]) - ingester.ingest_data(plugin_name='yaml', design_state=module_design_state, filenames=[str(input_file)]) + ingester.ingest_data(plugin_name='yaml', design_state=design_state, filenames=[str(input_file)]) - return module_design_state + return design_state diff --git a/tests/test_ingester.py b/tests/test_ingester.py index 2081521b..e9db185f 100644 --- a/tests/test_ingester.py +++ b/tests/test_ingester.py @@ -13,7 +13,7 @@ # limitations under the License. from helm_drydock.ingester import Ingester -from helm_drydock.statemgmt import DesignState +from helm_drydock.statemgmt import DesignState, SiteDesign import pytest import shutil @@ -27,14 +27,19 @@ class TestClass(object): def test_ingest_full_site(self, input_files): input_file = input_files.join("fullsite.yaml") + design_state = DesignState() + design_data = SiteDesign() + design_state.post_design_base(design_data) ingester = Ingester() ingester.enable_plugins([helm_drydock.ingester.plugins.yaml.YamlIngester]) ingester.ingest_data(plugin_name='yaml', design_state=design_state, filenames=[str(input_file)]) - assert len(design_state.get_host_profiles()) == 3 - assert len(design_state.get_baremetal_nodes()) == 2 + design_data = design_state.get_design_base() + + assert len(design_data.get_host_profiles()) == 3 + assert len(design_data.get_baremetal_nodes()) == 2 def test_ingest_federated_design(self, input_files): profiles_file = input_files.join("fullsite_profiles.yaml") @@ -42,13 +47,17 @@ class TestClass(object): nodes_file = input_files.join("fullsite_nodes.yaml") design_state = DesignState() + design_data = SiteDesign() + design_state.post_design_base(design_data) ingester = Ingester() ingester.enable_plugins([helm_drydock.ingester.plugins.yaml.YamlIngester]) ingester.ingest_data(plugin_name='yaml', design_state=design_state, filenames=[str(profiles_file), str(networks_file), str(nodes_file)]) - assert len(design_state.host_profiles) == 3 + design_data = design_state.get_design_base() + + assert len(design_data.host_profiles) == 3 @pytest.fixture(scope='module') def input_files(self, tmpdir_factory, request): diff --git a/tests/test_models.py b/tests/test_models.py index 3e993c44..a2d6fe8f 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -14,7 +14,7 @@ import pytest import yaml -from helm_drydock.model import HardwareProfile +from helm_drydock.model.hwprofile import HardwareProfile class TestClass(object): diff --git a/tests/yaml_samples/fullsite.yaml b/tests/yaml_samples/fullsite.yaml index 3a7f8ede..df2f39ba 100644 --- a/tests/yaml_samples/fullsite.yaml +++ b/tests/yaml_samples/fullsite.yaml @@ -25,8 +25,7 @@ metadata: date: 17-FEB-2017 description: Sample site design author: sh8121@att.com -spec: - # Not sure if we have site wide data that doesn't fall into another 'Kind' +# Not sure if we have site wide data that doesn't fall into another 'Kind' --- apiVersion: 'v1.0' kind: NetworkLink @@ -454,4 +453,4 @@ spec: scsi: - address: scsi@2:0.0.0 alias: primary_boot - type: 'VBOX HARDDISK' \ No newline at end of file + type: 'VBOX HARDDISK' diff --git a/tests/yaml_samples/fullsite_networks.yaml b/tests/yaml_samples/fullsite_networks.yaml index 54770ac5..37af1412 100644 --- a/tests/yaml_samples/fullsite_networks.yaml +++ b/tests/yaml_samples/fullsite_networks.yaml @@ -225,4 +225,4 @@ spec: metric: 9 dns: domain: sitename.example.com - servers: 8.8.8.8 \ No newline at end of file + servers: 8.8.8.8 diff --git a/tests/yaml_samples/fullsite_profiles.yaml b/tests/yaml_samples/fullsite_profiles.yaml index 7ad9a73d..c5430085 100644 --- a/tests/yaml_samples/fullsite_profiles.yaml +++ b/tests/yaml_samples/fullsite_profiles.yaml @@ -25,8 +25,7 @@ metadata: date: 17-FEB-2017 description: Sample site design author: sh8121@att.com -spec: - # Not sure if we have site wide data that doesn't fall into another 'Kind' +# Not sure if we have site wide data that doesn't fall into another 'Kind' --- apiVersion: 'v1.0' kind: HostProfile @@ -194,4 +193,4 @@ spec: scsi: - address: scsi@2:0.0.0 alias: primary_boot - type: 'VBOX HARDDISK' \ No newline at end of file + type: 'VBOX HARDDISK'