Last commit to push changes to master. Will use feature branches

going forward

Added readmes to submodules
Refactored model into multiple files
Standardized on 'node' for physical server
Pinned requirements.txt to versions
Began orchestration action outline in conjunction with DAG
This commit is contained in:
Scott Hussey 2017-03-15 14:53:46 -05:00
parent ace0e47762
commit 839c83651f
34 changed files with 1584 additions and 769 deletions

View File

@ -32,9 +32,15 @@ aka orchestrator
Handle validation of complete design, ordering and managing downstream API calls for hardware
provisioning/bootstrapping
### Server Driver ###
### OOB Driver ###
aka maasdriver
Pluggable provider for server OOB (ILO) management
aka driver/oob
### Node Driver ###
aka driver/node
Pluggable provisioner for server bootstrapping. Initial implementation is MaaS client.

View File

@ -21,7 +21,7 @@
class DrydockConfig(object):
def __init__(self):
def __init__(self):
self.server_driver_config = {
selected_driver = helm_drydock.drivers.server.maasdriver,
params = {
@ -29,13 +29,13 @@ class DrydockConfig(object):
maas_api_url = ""
}
}
self.selected_network_driver = helm_drydock.drivers.network.noopdriver
self.control_config = {}
self.ingester_config = {
plugins = [helm_drydock.ingester.plugins.aicyaml.AicYamlIngester]
}
self.introspection_config = {}
self.orchestrator_config = {}
self.statemgmt_config = {
backend_driver = helm_drydock.drivers.statemgmt.etcd,
}
self.selected_network_driver = helm_drydock.drivers.network.noopdriver
self.control_config = {}
self.ingester_config = {
plugins = [helm_drydock.ingester.plugins.aicyaml.AicYamlIngester]
}
self.introspection_config = {}
self.orchestrator_config = {}
self.statemgmt_config = {
backend_driver = helm_drydock.drivers.statemgmt.etcd,
}

View File

@ -0,0 +1,6 @@
# Control #
This is the external facing API service to control the rest
of Drydock and query Drydock-managed data.
Anticipate basing this service on the falcon Python library

View File

@ -17,3 +17,5 @@ class ProviderDriver(object):
__init__(self):
pass
class DriverTask(object):

View File

@ -11,25 +11,19 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from pecan import expose
from webob.exc import status_map
from helm_drydock.drivers import ProviderDriver
class RootController(object):
class NodeDriver(ProviderDriver):
@expose(generic=True, template='index.html')
def index(self):
return dict()
class NodeAction(Enum):
PrepareNode = 'prepare_node'
ApplyNetworkConfig = 'apply_network_config'
ApplyStorageConfig = 'apply_storage_config'
InterrogateNode = 'interrogate_node'
DeployNode = 'deploy_node'
@index.when(method='POST')
def index_post(self, q):
redirect('https://pecan.readthedocs.io/en/latest/search.html?q=%s' % q)
@expose('error.html')
def error(self, status):
try:
status = int(status)
except ValueError:
status = 0
message = getattr(status_map.get(status), 'explanation', '')
return dict(status=status, message=message)

View File

@ -11,7 +11,10 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import helm_drydock.drivers.server.ServerDriver
from helm_drydock.drivers.node import NodeDriver
class MaasServerDriver(object):
class MaasNodeDriver(NodeDriver):
def __init__(self, kwargs):
super(MaasNodeDriver, self).__init__(**kwargs)

View File

@ -0,0 +1,42 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# OOB:
# sync_hardware_clock
# collect_chassis_sysinfo
# enable_netboot
# initiate_reboot
# set_power_off
# set_power_on
from helm_drydock.drivers import ProviderDriver
class OobDriver(ProviderDriver):
def __init__(self):
pass
def execute_action(self, action, **kwargs):
if action ==
class OobAction(Enum):
ConfigNodePxe = 'config_node_pxe'
SetNodeBoot = 'set_node_boot'
PowerOffNode = 'power_off_node'
PowerOnNode = 'power_on_node'
PowerCycleNode = 'power_cycle_node'
InterrogateNode = 'interrogate_node'

View File

@ -0,0 +1,26 @@
# Drivers #
Drivers are downstream actors that Drydock will use to actually execute
orchestration actions. It is intended to be a pluggable architecture
so that various downstream automation can be used.
## oob ##
The oob drivers will interface with physical servers' out-of-band
management system (e.g. Dell iDRAC, HP iLO, etc...). OOB management
will be used for setting a system to use PXE boot and power cycling
servers.
## node ##
The node drivers will interface with an external bootstrapping system
for loading the base OS on a server and configuring hardware, network,
and storage.
## network ##
The network drivers will interface with switches for managing port
configuration to support the bootstrapping of physical nodes. This is not
intended to be a network provisioner, but instead is a support driver
for node bootstrapping where temporary changes to network configurations
are required.

View File

@ -12,7 +12,5 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import helm_drydock.drivers.ProviderDriver
class ServerDriver(ProviderDriver):
class DesignError(Exception):
pass

View File

@ -17,9 +17,14 @@
import logging
import yaml
import helm_drydock.model as model
from helm_drydock.statemgmt import DesignState
import helm_drydock.model.site as site
import helm_drydock.model.network as network
import helm_drydock.model.hwprofile as hwprofile
import helm_drydock.model.node as node
import helm_drydock.model.hostprofile as hostprofile
from helm_drydock.statemgmt import DesignState, SiteDesign, DesignError
class Ingester(object):
@ -28,16 +33,6 @@ class Ingester(object):
self.log = logging.Logger("ingester")
self.registered_plugins = {}
"""
enable_plugins
params: plugins - A list of class objects denoting the ingester plugins to be enabled
Enable plugins that can be used for ingest_data calls. Each plugin should use
helm_drydock.ingester.plugins.IngesterPlugin as its base class. As long as one
enabled plugin successfully initializes, the call is considered successful. Otherwise
it will throw an exception
"""
def enable_plugins(self, plugins=[]):
if len(plugins) == 0:
self.log.error("Cannot have an empty plugin list.")
@ -53,7 +48,51 @@ class Ingester(object):
if len(self.registered_plugins) == 0:
self.log.error("Could not enable at least one plugin")
raise Exception("Could not enable at least one plugin")
"""
enable_plugins
params: plugins - A list of class objects denoting the ingester plugins to be enabled
Enable plugins that can be used for ingest_data calls. Each plugin should use
helm_drydock.ingester.plugins.IngesterPlugin as its base class. As long as one
enabled plugin successfully initializes, the call is considered successful. Otherwise
it will throw an exception
"""
def ingest_data(self, plugin_name='', design_state=None, **kwargs):
if design_state is None:
self.log.error("ingest_data called without valid DesignState handler")
raise Exception("Invalid design_state handler")
# TODO this method needs refactored to handle design base vs change
design_data = None
try:
design_data = design_state.get_design_base()
except DesignError:
design_data = SiteDesign()
if plugin_name in self.registered_plugins:
design_items = self.registered_plugins[plugin_name].ingest_data(**kwargs)
# Need to persist data here, but we don't yet have the statemgmt service working
for m in design_items:
if type(m) is site.Site:
design_data.add_site(m)
elif type(m) is network.Network:
design_data.add_network(m)
elif type(m) is network.NetworkLink:
design_data.add_network_link(m)
elif type(m) is hostprofile.HostProfile:
design_data.add_host_profile(m)
elif type(m) is hwprofile.HardwareProfile:
design_data.add_hardware_profile(m)
elif type(m) is node.BaremetalNode:
design_data.add_baremetal_node(m)
design_state.put_design_base(design_data)
else:
self.log.error("Could not find plugin %s to ingest data." % (plugin_name))
raise LookupError("Could not find plugin %s" % plugin_name)
"""
ingest_data
@ -62,29 +101,4 @@ class Ingester(object):
Execute a data ingestion using the named plugin (assuming it is enabled)
"""
def ingest_data(self, plugin_name='', design_state=None, **kwargs):
if design_state is None:
self.log.error("ingest_data called without valid DesignState handler")
raise Exception("Invalid design_state handler")
if plugin_name in self.registered_plugins:
design_data = self.registered_plugins[plugin_name].ingest_data(**kwargs)
# Need to persist data here, but we don't yet have the statemgmt service working
for m in design_data:
if type(m) is model.Site:
design_state.add_site(m)
elif type(m) is model.Network:
design_state.add_network(m)
elif type(m) is model.NetworkLink:
design_state.add_network_link(m)
elif type(m) is model.HostProfile:
design_state.add_host_profile(m)
elif type(m) is model.HardwareProfile:
design_state.add_hardware_profile(m)
elif type(m) is model.BaremetalNode:
design_state.add_baremetal_node(m)
else:
self.log.error("Could not find plugin %s to ingest data." % (plugin_name))
raise LookupError("Could not find plugin %s" % plugin_name)

View File

@ -19,18 +19,23 @@
import yaml
import logging
import helm_drydock.model as model
import helm_drydock.model.hwprofile as hwprofile
import helm_drydock.model.node as node
import helm_drydock.model.site as site
import helm_drydock.model.hostprofile as hostprofile
import helm_drydock.model.network as network
from helm_drydock.ingester.plugins import IngesterPlugin
class YamlIngester(IngesterPlugin):
kind_map = {
"Region": model.Site,
"NetworkLink": model.NetworkLink,
"HardwareProfile": model.HardwareProfile,
"Network": model.Network,
"HostProfile": model.HostProfile,
"BaremetalNode": model.BaremetalNode,
"Region": site.Site,
"NetworkLink": network.NetworkLink,
"HardwareProfile": hwprofile.HardwareProfile,
"Network": network.Network,
"HostProfile": hostprofile.HostProfile,
"BaremetalNode": node.BaremetalNode,
}
def __init__(self):

View File

@ -0,0 +1,13 @@
# Ingester #
Ingester is a pluggable consumer of site design data. It
will support consuming data in different formats from
different sources.
Each ingester plugin should be able source data
based on user-provided parameters and parse that data
into the Drydock internal model (helm_drydock.model).
Each plugin does not need to support every type of design
data as a single site design may be federated from multiple
sources.

View File

@ -0,0 +1,9 @@
# Introspection #
Introspection is a cloud-init compatible metadata service
that is used to make a node self-aware. After a full
deployment by the node driver, the newly installed OS
will contact the introspection API to gain a package of
declaritive data defining the node's role in the site and
enough initial data to start the promenade process of
Kubernetes assimilation

View File

@ -18,626 +18,6 @@ import logging
from copy import deepcopy
class HardwareProfile(object):
def __init__(self, **kwargs):
self.log = logging.Logger('model')
self.api_version = kwargs.get('apiVersion', '')
if self.api_version == "v1.0":
metadata = kwargs.get('metadata', {})
spec = kwargs.get('spec', {})
# Need to add validation logic, we'll assume the input is
# valid for now
self.name = metadata.get('name', '')
self.site = metadata.get('region', '')
self.vendor = spec.get('vendor', None)
self.generation = spec.get('generation', None)
self.hw_version = spec.get('hw_version', None)
self.bios_version = spec.get('bios_version', None)
self.boot_mode = spec.get('boot_mode', None)
self.bootstrap_protocol = spec.get('bootstrap_protocol', None)
self.pxe_interface = spec.get('pxe_interface', None)
self.devices = []
device_aliases = spec.get('device_aliases', {})
pci_devices = device_aliases.get('pci', [])
scsi_devices = device_aliases.get('scsi', [])
for d in pci_devices:
d['bus_type'] = 'pci'
self.devices.append(
HardwareDeviceAlias(self.api_version, **d))
for d in scsi_devices:
d['bus_type'] = 'scsi'
self.devices.append(
HardwareDeviceAlias(self.api_version, **d))
else:
self.log.error("Unknown API version %s of %s" %
(self.api_version, self.__class__))
raise ValueError('Unknown API version of object')
return
def resolve_alias(self, alias_type, alias):
selector = {}
for d in self.devices:
if d.alias == alias and d.bus_type == alias_type:
selector['address'] = d.address
selector['device_type'] = d.type
return selector
return None
class HardwareDeviceAlias(object):
def __init__(self, api_version, **kwargs):
self.log = logging.Logger('model')
self.api_version = api_version
if self.api_version == "v1.0":
self.bus_type = kwargs.get('bus_type', None)
self.address = kwargs.get('address', None)
self.alias = kwargs.get('alias', None)
self.type = kwargs.get('type', None)
else:
self.log.error("Unknown API version %s of %s" %
(self.api_version, self.__class__))
raise ValueError('Unknown API version of object')
class Site(object):
def __init__(self, **kwargs):
self.log = logging.Logger('model')
self.api_version = kwargs.get('apiVersion', '')
if self.api_version == "v1.0":
metadata = kwargs.get('metadata', {})
# Need to add validation logic, we'll assume the input is
# valid for now
self.name = metadata.get('name', '')
self.networks = []
self.network_links = []
self.host_profiles = []
self.hardware_profiles = []
self.baremetal_nodes = []
else:
self.log.error("Unknown API version %s of %s" %
(self.api_version, self.__class__))
raise ValueError('Unknown API version of object')
def get_network(self, network_name):
for n in self.networks:
if n.name == network_name:
return n
return None
def get_network_link(self, link_name):
for l in self.network_links:
if l.name == link_name:
return l
return None
def get_host_profile(self, profile_name):
for p in self.host_profiles:
if p.name == profile_name:
return p
return None
def get_hardware_profile(self, profile_name):
for p in self.hardware_profiles:
if p.name == profile_name:
return p
return None
def get_baremetal_node(self, node_name):
for n in self.baremetal_nodes:
if n.name == node_name:
return n
return None
class NetworkLink(object):
def __init__(self, **kwargs):
self.log = logging.Logger('model')
self.api_version = kwargs.get('apiVersion', '')
if self.api_version == "v1.0":
metadata = kwargs.get('metadata', {})
spec = kwargs.get('spec', {})
self.name = metadata.get('name', '')
self.site = metadata.get('region', '')
bonding = spec.get('bonding', {})
self.bonding_mode = bonding.get('mode', 'none')
# How should we define defaults for CIs not in the input?
if self.bonding_mode == '802.3ad':
self.bonding_xmit_hash = bonding.get('hash', 'layer3+4')
self.bonding_peer_rate = bonding.get('peer_rate', 'fast')
self.bonding_mon_rate = bonding.get('mon_rate', '100')
self.bonding_up_delay = bonding.get('up_delay', '200')
self.bonding_down_delay = bonding.get('down_delay', '200')
self.mtu = spec.get('mtu', 1500)
self.linkspeed = spec.get('linkspeed', 'auto')
trunking = spec.get('trunking', {})
self.trunk_mode = trunking.get('mode', 'none')
self.native_network = spec.get('default_network', '')
else:
self.log.error("Unknown API version %s of %s" %
(self.api_version, self.__class__))
raise ValueError('Unknown API version of object')
class Network(object):
def __init__(self, **kwargs):
self.log = logging.Logger('model')
self.api_version = kwargs.get('apiVersion', '')
if self.api_version == "v1.0":
metadata = kwargs.get('metadata', {})
spec = kwargs.get('spec', {})
self.name = metadata.get('name', '')
self.site = metadata.get('region', '')
self.cidr = spec.get('cidr', None)
self.allocation_strategy = spec.get('allocation', 'static')
self.vlan_id = spec.get('vlan_id', 1)
self.mtu = spec.get('mtu', 0)
dns = spec.get('dns', {})
self.dns_domain = dns.get('domain', 'local')
self.dns_servers = dns.get('servers', None)
ranges = spec.get('ranges', [])
self.ranges = []
for r in ranges:
self.ranges.append(NetworkAddressRange(self.api_version, **r))
routes = spec.get('routes', [])
self.routes = []
for r in routes:
self.routes.append(NetworkRoute(self.api_version, **r))
else:
self.log.error("Unknown API version %s of %s" %
(self.api_version, self.__class__))
raise ValueError('Unknown API version of object')
class NetworkAddressRange(object):
def __init__(self, api_version, **kwargs):
self.log = logging.Logger('model')
self.api_version = api_version
if self.api_version == "v1.0":
self.type = kwargs.get('type', None)
self.start = kwargs.get('start', None)
self.end = kwargs.get('end', None)
else:
self.log.error("Unknown API version %s of %s" %
(self.api_version, self.__class__))
raise ValueError('Unknown API version of object')
class NetworkRoute(object):
def __init__(self, api_version, **kwargs):
self.log = logging.Logger('model')
self.api_version = api_version
if self.api_version == "v1.0":
self.type = kwargs.get('subnet', None)
self.start = kwargs.get('gateway', None)
self.end = kwargs.get('metric', 100)
else:
self.log.error("Unknown API version %s of %s" %
(self.api_version, self.__class__))
raise ValueError('Unknown API version of object')
class HostProfile(object):
def __init__(self, **kwargs):
self.log = logging.Logger('model')
self.api_version = kwargs.get('apiVersion', '')
if self.api_version == "v1.0":
metadata = kwargs.get('metadata', {})
spec = kwargs.get('spec', {})
self.name = metadata.get('name', '')
self.site = metadata.get('region', '')
self.parent_profile = spec.get('host_profile', None)
self.hardware_profile = spec.get('hardware_profile', None)
oob = spec.get('oob', {})
self.oob_type = oob.get('type', None)
self.oob_network = oob.get('network', None)
self.oob_account = oob.get('account', None)
self.oob_credential = oob.get('credential', None)
storage = spec.get('storage', {})
self.storage_layout = storage.get('layout', 'lvm')
bootdisk = storage.get('bootdisk', {})
self.bootdisk_device = bootdisk.get('device', None)
self.bootdisk_root_size = bootdisk.get('root_size', None)
self.bootdisk_boot_size = bootdisk.get('boot_size', None)
partitions = storage.get('partitions', [])
self.partitions = []
for p in partitions:
self.partitions.append(HostPartition(self.api_version, **p))
interfaces = spec.get('interfaces', [])
self.interfaces = []
for i in interfaces:
self.interfaces.append(HostInterface(self.api_version, **i))
metadata = spec.get('metadata', {})
metadata_tags = metadata.get('tags', [])
self.tags = []
for t in metadata_tags:
self.tags.append(t)
owner_data = metadata.get('owner_data', {})
self.owner_data = {}
for k, v in owner_data.items():
self.owner_data[k] = v
self.rack = metadata.get('rack', None)
else:
self.log.error("Unknown API version %s of %s" %
(self.api_version, self.__class__))
raise ValueError('Unknown API version of object')
def apply_inheritance(self, site):
# We return a deep copy of the profile so as not to corrupt
# the original model
self_copy = deepcopy(self)
if self.parent_profile is None:
return self_copy
parent = site.get_host_profile(self.parent_profile)
if parent is None:
raise NameError("Cannot find parent profile %s for %s"
% (self.parent_profile, self.name))
parent = parent.apply_inheritance(site)
# First compute inheritance for simple fields
inheritable_field_list = [
"hardware_profile", "oob_type", "oob_network",
"oob_credential", "oob_account", "storage_layout",
"bootdisk_device", "bootdisk_root_size", "bootdisk_boot_size",
"rack"]
for f in inheritable_field_list:
setattr(self_copy, f,
Utils.apply_field_inheritance(getattr(self, f, None),
getattr(parent, f, None)))
# Now compute inheritance for complex types
self_copy.tags = Utils.merge_lists(self.tags, parent.tags)
self_copy.owner_data = Utils.merge_dicts(
self.owner_data, parent.owner_data)
self_copy.interfaces = HostInterface.merge_lists(
self.interfaces, parent.interfaces)
self_copy.partitions = HostPartition.merge_lists(
self.partitions, parent.partitions)
return self_copy
class HostInterface(object):
def __init__(self, api_version, **kwargs):
self.log = logging.Logger('model')
self.api_version = api_version
if self.api_version == "v1.0":
self.device_name = kwargs.get('device_name', None)
self.network_link = kwargs.get('device_link', None)
self.hardware_slaves = []
slaves = kwargs.get('slaves', [])
for s in slaves:
self.hardware_slaves.append(s)
self.networks = []
networks = kwargs.get('networks', [])
for n in networks:
self.networks.append(n)
else:
self.log.error("Unknown API version %s of %s" %
(self.api_version, self.__class__))
raise ValueError('Unknown API version of object')
# The device attribute may be hardware alias that translates to a
# physical device address. If the device attribute does not match an
# alias, we assume it directly identifies a OS device name. When the
# apply_hardware_profile method is called on the parent Node of this
# device, the selector will be decided and applied
def add_selector(self, sel_type, address='', dev_type=''):
if getattr(self, 'selectors', None) is None:
self.selectors = []
new_selector = {}
new_selector['selector_type'] = sel_type
new_selector['address'] = address
new_selector['device_type'] = dev_type
self.selectors.append(new_selector)
def get_slave_selectors(self):
return self.selectors
# Return number of slaves for this interface
def get_slave_count(self):
return len(self.hardware_slaves)
"""
Merge two lists of HostInterface models with child_list taking
priority when conflicts. If a member of child_list has a device_name
beginning with '!' it indicates that HostInterface should be
removed from the merged list
"""
@staticmethod
def merge_lists(child_list, parent_list):
if len(child_list) == 0:
return deepcopy(parent_list)
effective_list = []
if len(parent_list) == 0:
for i in child_list:
if i.device_name.startswith('!'):
continue
else:
effective_list.append(deepcopy(i))
return effective_list
parent_interfaces = []
for i in parent_list:
parent_name = i.device_name
parent_interfaces.append(parent_name)
add = True
for j in child_list:
if j.device_name == ("!" + parent_name):
add = False
break
elif j.device_name == parent_name:
m = HostInterface(j.api_version)
m.device_name = j.device_name
m.network_link = \
Utils.apply_field_inheritance(j.network_link,
i.network_link)
s = filter(lambda x: ("!" + x) not in j.hardware_slaves,
i.hardware_slaves)
s = list(s)
s.extend(filter(lambda x: not x.startswith("!"),
j.hardware_slaves))
m.hardware_slaves = s
n = filter(lambda x: ("!" + x) not in j.networks,
i.networks)
n = list(n)
n.extend(filter(lambda x: not x.startswith("!"),
j.networks))
m.networks = n
effective_list.append(m)
add = False
break
if add:
effective_list.append(deepcopy(i))
for j in child_list:
if (j.device_name not in parent_interfaces
and not j.device_name.startswith("!")):
effective_list.append(deepcopy(j))
return effective_list
class HostPartition(object):
def __init__(self, api_version, **kwargs):
self.api_version = api_version
if self.api_version == "v1.0":
self.name = kwargs.get('name', None)
self.device = kwargs.get('device', None)
self.part_uuid = kwargs.get('part_uuid', None)
self.size = kwargs.get('size', None)
self.mountpoint = kwargs.get('mountpoint', None)
self.fstype = kwargs.get('fstype', 'ext4')
self.mount_options = kwargs.get('mount_options', 'defaults')
self.fs_uuid = kwargs.get('fs_uuid', None)
self.fs_label = kwargs.get('fs_label', None)
else:
raise ValueError('Unknown API version of object')
# The device attribute may be hardware alias that translates to a
# physical device address. If the device attribute does not match an
# alias, we assume it directly identifies a OS device name. When the
# apply_hardware_profile method is called on the parent Node of this
# device, the selector will be decided and applied
def set_selector(self, sel_type, address='', dev_type=''):
selector = {}
selector['type'] = sel_type
selector['address'] = address
selector['device_type'] = dev_type
self.selector = selector
def get_selector(self):
return self.selector
"""
Merge two lists of HostPartition models with child_list taking
priority when conflicts. If a member of child_list has a name
beginning with '!' it indicates that HostPartition should be
removed from the merged list
"""
@staticmethod
def merge_lists(child_list, parent_list):
if len(child_list) == 0:
return deepcopy(parent_list)
effective_list = []
if len(parent_list) == 0:
for i in child_list:
if i.name.startswith('!'):
continue
else:
effective_list.append(deepcopy(i))
inherit_field_list = ["device", "part_uuid", "size",
"mountpoint", "fstype", "mount_options",
"fs_uuid", "fs_label"]
parent_partitions = []
for i in parent_list:
parent_name = i.name
parent_partitions.append(parent_name)
add = True
for j in child_list:
if j.name == ("!" + parent_name):
add = False
break
elif j.name == parent_name:
p = HostPartition(j.api_version)
p.name = j.name
for f in inherit_field_list:
setattr(p, Utils.apply_field_inheritance(getattr(j, f),
getattr(i, f))
)
add = False
effective_list.append(p)
if add:
effective_list.append(deepcopy(i))
for j in child_list:
if j.name not in parent_list:
effective_list.append(deepcopy(j))
return effective_list
# A BaremetalNode is really nothing more than a physical
# instantiation of a HostProfile, so they both represent
# the same set of CIs
class BaremetalNode(HostProfile):
def __init__(self, **kwargs):
super(BaremetalNode, self).__init__(**kwargs)
def apply_host_profile(self, site):
return self.apply_inheritance(site)
# Translate device alises to physical selectors and copy
# other hardware attributes into this object
def apply_hardware_profile(self, site):
self_copy = deepcopy(self)
if self.hardware_profile is None:
raise ValueError("Hardware profile not set")
hw_profile = site.get_hardware_profile(self.hardware_profile)
for i in self_copy.interfaces:
for s in i.hardware_slaves:
selector = hw_profile.resolve_alias("pci", s)
if selector is None:
i.add_selector("name", address=p.device)
else:
i.add_selector("address", address=selector['address'],
dev_type=selector['device_type'])
for p in self_copy.partitions:
selector = hw_profile.resolve_alias("scsi", p.device)
if selector is None:
p.set_selector("name", address=p.device)
else:
p.set_selector("address", address=selector['address'],
dev_type=selector['device_type'])
hardware = {"vendor": getattr(hw_profile, 'vendor', None),
"generation": getattr(hw_profile, 'generation', None),
"hw_version": getattr(hw_profile, 'hw_version', None),
"bios_version": getattr(hw_profile, 'bios_version', None),
"boot_mode": getattr(hw_profile, 'boot_mode', None),
"bootstrap_protocol": getattr(hw_profile,
'bootstrap_protocol',
None),
"pxe_interface": getattr(hw_profile, 'pxe_interface', None)
}
self_copy.hardware = hardware
return self_copy
def get_interface(self, iface_name):
for i in self.interfaces:
if i.device_name == iface_name:
return i
return None
# Utility class for calculating inheritance
class Utils(object):

View File

@ -0,0 +1,378 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Models for helm_drydock
#
import logging
from copy import deepcopy
from helm_drydock.orchestrator.enum import SiteStatus
from helm_drydock.orchestrator.enum import NodeStatus
from helm_drydock.model.network import Network
from helm_drydock.model.network import NetworkLink
from helm_drydock.model import Utils
class HostProfile(object):
def __init__(self, **kwargs):
self.log = logging.Logger('model')
self.api_version = kwargs.get('apiVersion', '')
if self.api_version == "v1.0":
metadata = kwargs.get('metadata', {})
spec = kwargs.get('spec', {})
self.name = metadata.get('name', '')
self.site = metadata.get('region', '')
self.parent_profile = spec.get('host_profile', None)
self.hardware_profile = spec.get('hardware_profile', None)
oob = spec.get('oob', {})
self.oob_type = oob.get('type', None)
self.oob_network = oob.get('network', None)
self.oob_account = oob.get('account', None)
self.oob_credential = oob.get('credential', None)
storage = spec.get('storage', {})
self.storage_layout = storage.get('layout', 'lvm')
bootdisk = storage.get('bootdisk', {})
self.bootdisk_device = bootdisk.get('device', None)
self.bootdisk_root_size = bootdisk.get('root_size', None)
self.bootdisk_boot_size = bootdisk.get('boot_size', None)
partitions = storage.get('partitions', [])
self.partitions = []
for p in partitions:
self.partitions.append(HostPartition(self.api_version, **p))
interfaces = spec.get('interfaces', [])
self.interfaces = []
for i in interfaces:
self.interfaces.append(HostInterface(self.api_version, **i))
node_metadata = spec.get('metadata', {})
metadata_tags = node_metadata.get('tags', [])
self.tags = []
for t in metadata_tags:
self.tags.append(t)
owner_data = node_metadata.get('owner_data', {})
self.owner_data = {}
for k, v in owner_data.items():
self.owner_data[k] = v
self.rack = node_metadata.get('rack', None)
else:
self.log.error("Unknown API version %s of %s" %
(self.api_version, self.__class__))
raise ValueError('Unknown API version of object')
def get_rack(self):
return self.rack
def get_name(self):
return self.name
def has_tag(self, tag):
if tag in self.tags:
return True
return False
def apply_inheritance(self, site):
# We return a deep copy of the profile so as not to corrupt
# the original model
self_copy = deepcopy(self)
if self.parent_profile is None:
return self_copy
parent = site.get_host_profile(self.parent_profile)
if parent is None:
raise NameError("Cannot find parent profile %s for %s"
% (self.parent_profile, self.name))
parent = parent.apply_inheritance(site)
# First compute inheritance for simple fields
inheritable_field_list = [
"hardware_profile", "oob_type", "oob_network",
"oob_credential", "oob_account", "storage_layout",
"bootdisk_device", "bootdisk_root_size", "bootdisk_boot_size",
"rack"]
for f in inheritable_field_list:
setattr(self_copy, f,
Utils.apply_field_inheritance(getattr(self, f, None),
getattr(parent, f, None)))
# Now compute inheritance for complex types
self_copy.tags = Utils.merge_lists(self.tags, parent.tags)
self_copy.owner_data = Utils.merge_dicts(
self.owner_data, parent.owner_data)
self_copy.interfaces = HostInterface.merge_lists(
self.interfaces, parent.interfaces)
self_copy.partitions = HostPartition.merge_lists(
self.partitions, parent.partitions)
return self_copy
class HostInterface(object):
def __init__(self, api_version, **kwargs):
self.log = logging.Logger('model')
self.api_version = api_version
if self.api_version == "v1.0":
self.device_name = kwargs.get('device_name', None)
self.network_link = kwargs.get('device_link', None)
self.hardware_slaves = []
slaves = kwargs.get('slaves', [])
for s in slaves:
self.hardware_slaves.append(s)
self.networks = []
networks = kwargs.get('networks', [])
for n in networks:
self.networks.append(n)
else:
self.log.error("Unknown API version %s of %s" %
(self.api_version, self.__class__))
raise ValueError('Unknown API version of object')
# The device attribute may be hardware alias that translates to a
# physical device address. If the device attribute does not match an
# alias, we assume it directly identifies a OS device name. When the
# apply_hardware_profile method is called on the parent Node of this
# device, the selector will be decided and applied
def add_selector(self, sel_type, address='', dev_type=''):
if getattr(self, 'selectors', None) is None:
self.selectors = []
new_selector = {}
new_selector['selector_type'] = sel_type
new_selector['address'] = address
new_selector['device_type'] = dev_type
self.selectors.append(new_selector)
def get_slave_selectors(self):
return self.selectors
# Return number of slaves for this interface
def get_slave_count(self):
return len(self.hardware_slaves)
def apply_link_config(self, net_link):
if (net_link is not None and
isinstance(net_link, NetworkLink) and
net_link.name == self.network_link):
self.attached_link = deepcopy(net_link)
return True
return False
def apply_network_config(self, network):
if network in self.networks:
if getattr(self, 'attached_networks', None) is None:
self.attached_networks = []
self.attached_networks.append(deepcopy(network))
return True
else:
return False
def set_network_address(self, network_name, address):
if getattr(self, 'attached_networks', None) is None:
return False
for n in self.attached_neteworks:
if n.name == network_name:
n.assigned_address = address
def get_network_configs(self):
return self.attached_networks
"""
Merge two lists of HostInterface models with child_list taking
priority when conflicts. If a member of child_list has a device_name
beginning with '!' it indicates that HostInterface should be
removed from the merged list
"""
@staticmethod
def merge_lists(child_list, parent_list):
if len(child_list) == 0:
return deepcopy(parent_list)
effective_list = []
if len(parent_list) == 0:
for i in child_list:
if i.device_name.startswith('!'):
continue
else:
effective_list.append(deepcopy(i))
return effective_list
parent_interfaces = []
for i in parent_list:
parent_name = i.device_name
parent_interfaces.append(parent_name)
add = True
for j in child_list:
if j.device_name == ("!" + parent_name):
add = False
break
elif j.device_name == parent_name:
m = HostInterface(j.api_version)
m.device_name = j.device_name
m.network_link = \
Utils.apply_field_inheritance(j.network_link,
i.network_link)
s = filter(lambda x: ("!" + x) not in j.hardware_slaves,
i.hardware_slaves)
s = list(s)
s.extend(filter(lambda x: not x.startswith("!"),
j.hardware_slaves))
m.hardware_slaves = s
n = filter(lambda x: ("!" + x) not in j.networks,
i.networks)
n = list(n)
n.extend(filter(lambda x: not x.startswith("!"),
j.networks))
m.networks = n
effective_list.append(m)
add = False
break
if add:
effective_list.append(deepcopy(i))
for j in child_list:
if (j.device_name not in parent_interfaces
and not j.device_name.startswith("!")):
effective_list.append(deepcopy(j))
return effective_list
class HostPartition(object):
def __init__(self, api_version, **kwargs):
self.api_version = api_version
if self.api_version == "v1.0":
self.name = kwargs.get('name', None)
self.device = kwargs.get('device', None)
self.part_uuid = kwargs.get('part_uuid', None)
self.size = kwargs.get('size', None)
self.mountpoint = kwargs.get('mountpoint', None)
self.fstype = kwargs.get('fstype', 'ext4')
self.mount_options = kwargs.get('mount_options', 'defaults')
self.fs_uuid = kwargs.get('fs_uuid', None)
self.fs_label = kwargs.get('fs_label', None)
else:
raise ValueError('Unknown API version of object')
# The device attribute may be hardware alias that translates to a
# physical device address. If the device attribute does not match an
# alias, we assume it directly identifies a OS device name. When the
# apply_hardware_profile method is called on the parent Node of this
# device, the selector will be decided and applied
def set_selector(self, sel_type, address='', dev_type=''):
selector = {}
selector['type'] = sel_type
selector['address'] = address
selector['device_type'] = dev_type
self.selector = selector
def get_selector(self):
return self.selector
"""
Merge two lists of HostPartition models with child_list taking
priority when conflicts. If a member of child_list has a name
beginning with '!' it indicates that HostPartition should be
removed from the merged list
"""
@staticmethod
def merge_lists(child_list, parent_list):
if len(child_list) == 0:
return deepcopy(parent_list)
effective_list = []
if len(parent_list) == 0:
for i in child_list:
if i.name.startswith('!'):
continue
else:
effective_list.append(deepcopy(i))
inherit_field_list = ["device", "part_uuid", "size",
"mountpoint", "fstype", "mount_options",
"fs_uuid", "fs_label"]
parent_partitions = []
for i in parent_list:
parent_name = i.name
parent_partitions.append(parent_name)
add = True
for j in child_list:
if j.name == ("!" + parent_name):
add = False
break
elif j.name == parent_name:
p = HostPartition(j.api_version)
p.name = j.name
for f in inherit_field_list:
setattr(p, Utils.apply_field_inheritance(getattr(j, f),
getattr(i, f))
)
add = False
effective_list.append(p)
if add:
effective_list.append(deepcopy(i))
for j in child_list:
if j.name not in parent_list:
effective_list.append(deepcopy(j))
return effective_list

View File

@ -0,0 +1,95 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Models for helm_drydock
#
import logging
from copy import deepcopy
from helm_drydock.orchestrator.enum import SiteStatus
from helm_drydock.orchestrator.enum import NodeStatus
class HardwareProfile(object):
def __init__(self, **kwargs):
self.log = logging.Logger('model')
self.api_version = kwargs.get('apiVersion', '')
if self.api_version == "v1.0":
metadata = kwargs.get('metadata', {})
spec = kwargs.get('spec', {})
# Need to add validation logic, we'll assume the input is
# valid for now
self.name = metadata.get('name', '')
self.site = metadata.get('region', '')
self.vendor = spec.get('vendor', None)
self.generation = spec.get('generation', None)
self.hw_version = spec.get('hw_version', None)
self.bios_version = spec.get('bios_version', None)
self.boot_mode = spec.get('boot_mode', None)
self.bootstrap_protocol = spec.get('bootstrap_protocol', None)
self.pxe_interface = spec.get('pxe_interface', None)
self.devices = []
device_aliases = spec.get('device_aliases', {})
pci_devices = device_aliases.get('pci', [])
scsi_devices = device_aliases.get('scsi', [])
for d in pci_devices:
d['bus_type'] = 'pci'
self.devices.append(
HardwareDeviceAlias(self.api_version, **d))
for d in scsi_devices:
d['bus_type'] = 'scsi'
self.devices.append(
HardwareDeviceAlias(self.api_version, **d))
else:
self.log.error("Unknown API version %s of %s" %
(self.api_version, self.__class__))
raise ValueError('Unknown API version of object')
return
def resolve_alias(self, alias_type, alias):
selector = {}
for d in self.devices:
if d.alias == alias and d.bus_type == alias_type:
selector['address'] = d.address
selector['device_type'] = d.type
return selector
return None
class HardwareDeviceAlias(object):
def __init__(self, api_version, **kwargs):
self.log = logging.Logger('model')
self.api_version = api_version
if self.api_version == "v1.0":
self.bus_type = kwargs.get('bus_type', None)
self.address = kwargs.get('address', None)
self.alias = kwargs.get('alias', None)
self.type = kwargs.get('type', None)
else:
self.log.error("Unknown API version %s of %s" %
(self.api_version, self.__class__))
raise ValueError('Unknown API version of object')

View File

@ -0,0 +1,133 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Models for helm_drydock
#
import logging
from copy import deepcopy
from helm_drydock.orchestrator.enum import SiteStatus
from helm_drydock.orchestrator.enum import NodeStatus
class NetworkLink(object):
def __init__(self, **kwargs):
self.log = logging.Logger('model')
self.api_version = kwargs.get('apiVersion', '')
if self.api_version == "v1.0":
metadata = kwargs.get('metadata', {})
spec = kwargs.get('spec', {})
self.name = metadata.get('name', '')
self.site = metadata.get('region', '')
bonding = spec.get('bonding', {})
self.bonding_mode = bonding.get('mode', 'none')
# How should we define defaults for CIs not in the input?
if self.bonding_mode == '802.3ad':
self.bonding_xmit_hash = bonding.get('hash', 'layer3+4')
self.bonding_peer_rate = bonding.get('peer_rate', 'fast')
self.bonding_mon_rate = bonding.get('mon_rate', '100')
self.bonding_up_delay = bonding.get('up_delay', '200')
self.bonding_down_delay = bonding.get('down_delay', '200')
self.mtu = spec.get('mtu', 1500)
self.linkspeed = spec.get('linkspeed', 'auto')
trunking = spec.get('trunking', {})
self.trunk_mode = trunking.get('mode', 'none')
self.native_network = spec.get('default_network', '')
else:
self.log.error("Unknown API version %s of %s" %
(self.api_version, self.__class__))
raise ValueError('Unknown API version of object')
class Network(object):
def __init__(self, **kwargs):
self.log = logging.Logger('model')
self.api_version = kwargs.get('apiVersion', '')
if self.api_version == "v1.0":
metadata = kwargs.get('metadata', {})
spec = kwargs.get('spec', {})
self.name = metadata.get('name', '')
self.site = metadata.get('region', '')
self.cidr = spec.get('cidr', None)
self.allocation_strategy = spec.get('allocation', 'static')
self.vlan_id = spec.get('vlan_id', 1)
self.mtu = spec.get('mtu', 0)
dns = spec.get('dns', {})
self.dns_domain = dns.get('domain', 'local')
self.dns_servers = dns.get('servers', None)
ranges = spec.get('ranges', [])
self.ranges = []
for r in ranges:
self.ranges.append(NetworkAddressRange(self.api_version, **r))
routes = spec.get('routes', [])
self.routes = []
for r in routes:
self.routes.append(NetworkRoute(self.api_version, **r))
else:
self.log.error("Unknown API version %s of %s" %
(self.api_version, self.__class__))
raise ValueError('Unknown API version of object')
class NetworkAddressRange(object):
def __init__(self, api_version, **kwargs):
self.log = logging.Logger('model')
self.api_version = api_version
if self.api_version == "v1.0":
self.type = kwargs.get('type', None)
self.start = kwargs.get('start', None)
self.end = kwargs.get('end', None)
else:
self.log.error("Unknown API version %s of %s" %
(self.api_version, self.__class__))
raise ValueError('Unknown API version of object')
class NetworkRoute(object):
def __init__(self, api_version, **kwargs):
self.log = logging.Logger('model')
self.api_version = api_version
if self.api_version == "v1.0":
self.type = kwargs.get('subnet', None)
self.start = kwargs.get('gateway', None)
self.end = kwargs.get('metric', 100)
else:
self.log.error("Unknown API version %s of %s" %
(self.api_version, self.__class__))
raise ValueError('Unknown API version of object')

154
helm_drydock/model/node.py Normal file
View File

@ -0,0 +1,154 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Models for helm_drydock
#
import logging
from copy import deepcopy
from helm_drydock.orchestrator.enum import SiteStatus
from helm_drydock.orchestrator.enum import NodeStatus
from helm_drydock.model.hostprofile import HostProfile
from helm_drydock.model import Utils
class BaremetalNode(HostProfile):
# A BaremetalNode is really nothing more than a physical
# instantiation of a HostProfile, so they both represent
# the same set of CIs
def __init__(self, **kwargs):
super(BaremetalNode, self).__init__(**kwargs)
if self.api_version == "v1.0":
self.addressing = []
spec = kwargs.get('spec', {})
addresses = spec.get('addressing', [])
if len(addresses) == 0:
raise ValueError('BaremetalNode needs at least' \
' 1 assigned address')
for a in addresses:
assignment = {}
address = a.get('address', '')
if address == 'dhcp':
assignment['type'] = 'dhcp'
assignment['address'] = None
assignment['network'] = a.get('network')
self.addressing.append(assignment)
elif address != '':
assignment['type'] = 'static'
assignment['address'] = a.get('address')
assignment['network'] = a.get('network')
self.addressing.append(assignment)
else:
self.log.error("Invalid address assignment %s on Node %s"
% (address, self.name))
self.build = kwargs.get('build', {})
def start_build(self):
if self.build.get('status','') == '':
self.build['status'] = NodeStatus.Unknown
def apply_host_profile(self, site):
return self.apply_inheritance(site)
# Translate device alises to physical selectors and copy
# other hardware attributes into this object
def apply_hardware_profile(self, site):
self_copy = deepcopy(self)
if self.hardware_profile is None:
raise ValueError("Hardware profile not set")
hw_profile = site.get_hardware_profile(self.hardware_profile)
for i in self_copy.interfaces:
for s in i.hardware_slaves:
selector = hw_profile.resolve_alias("pci", s)
if selector is None:
i.add_selector("name", address=p.device)
else:
i.add_selector("address", address=selector['address'],
dev_type=selector['device_type'])
for p in self_copy.partitions:
selector = hw_profile.resolve_alias("scsi", p.device)
if selector is None:
p.set_selector("name", address=p.device)
else:
p.set_selector("address", address=selector['address'],
dev_type=selector['device_type'])
hardware = {"vendor": getattr(hw_profile, 'vendor', None),
"generation": getattr(hw_profile, 'generation', None),
"hw_version": getattr(hw_profile, 'hw_version', None),
"bios_version": getattr(hw_profile, 'bios_version', None),
"boot_mode": getattr(hw_profile, 'boot_mode', None),
"bootstrap_protocol": getattr(hw_profile,
'bootstrap_protocol',
None),
"pxe_interface": getattr(hw_profile, 'pxe_interface', None)
}
self_copy.hardware = hardware
return self_copy
def apply_network_connections(self, site):
self_copy = deepcopy(self)
for n in site.network_links:
for i in self_copy.interfaces:
i.apply_link_config(n)
for n in site.networks:
for i in self_copy.interfaces:
i.apply_network_config(n)
for a in self_copy.addressing:
for i in self_copy.interfaces:
i.set_network_address(a.get('network'), a.get('address'))
return self_copy
def get_interface(self, iface_name):
for i in self.interfaces:
if i.device_name == iface_name:
return i
return None
def get_status(self):
return self.build['status']
def set_status(self, status):
if isinstance(status, NodeStatus):
self.build['status'] = status
def get_last_build_action(self):
return self.build.get('last_action', None)
def set_last_build_action(self, action, result, detail=None):
last_action = self.build.get('last_action', None)
if last_action is None:
self.build['last_action'] = {}
last_action = self.build['last_action']
last_action['action'] = action
last_action['result'] = result
if detail is not None:
last_action['detail'] = detail

122
helm_drydock/model/site.py Normal file
View File

@ -0,0 +1,122 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Models for helm_drydock
#
import logging
from copy import deepcopy
from helm_drydock.orchestrator.enum import SiteStatus
from helm_drydock.orchestrator.enum import NodeStatus
class Site(object):
def __init__(self, **kwargs):
self.log = logging.Logger('model')
if kwargs is None:
raise ValueError("Empty arguments")
self.api_version = kwargs.get('apiVersion', '')
self.build = kwargs.get('build', {})
if self.api_version == "v1.0":
metadata = kwargs.get('metadata', {})
# Need to add validation logic, we'll assume the input is
# valid for now
self.name = metadata.get('name', '')
spec = kwargs.get('spec', {})
self.tag_definitions = []
tag_defs = spec.get('tag_definitions', [])
for t in tag_defs:
self.tag_definitions.append(
NodeTagDefinition(self.api_version, **t))
self.networks = []
self.network_links = []
self.host_profiles = []
self.hardware_profiles = []
self.baremetal_nodes = []
else:
self.log.error("Unknown API version %s of %s" %
(self.api_version, self.__class__))
raise ValueError('Unknown API version of object')
def start_build(self):
if self.build.get('status', '') == '':
self.build['status'] = SiteStatus.Unknown
def get_network(self, network_name):
for n in self.networks:
if n.name == network_name:
return n
return None
def get_network_link(self, link_name):
for l in self.network_links:
if l.name == link_name:
return l
return None
def get_host_profile(self, profile_name):
for p in self.host_profiles:
if p.name == profile_name:
return p
return None
def get_hardware_profile(self, profile_name):
for p in self.hardware_profiles:
if p.name == profile_name:
return p
return None
def get_baremetal_node(self, node_name):
for n in self.baremetal_nodes:
if n.name == node_name:
return n
return None
def set_status(self, status):
if isinstance(status, SiteStatus):
self.build['status'] = status
class NodeTagDefinition(object):
def __init__(self, api_version, **kwargs):
self.api_version = api_version
if self.api_version == "v1.0":
self.tag = kwargs.get('tag', '')
self.definition_type = kwargs.get('definition_type', '')
self.definition = kwargs.get('definition', '')
if self.definition_type not in ['lshw_xpath']:
raise ValueError('Unknown definition type in ' \
'NodeTagDefinition: %s' % (self.definition_type))
else:
self.log.error("Unknown API version %s of %s" %
(self.api_version, self.__class__))
raise ValueError('Unknown API version of object')

View File

@ -10,4 +10,62 @@
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# limitations under the License.
from enum import Enum, unique
import uuid
class Orchestrator(object):
# enabled_drivers is a map which provider drivers
# should be enabled for use by this orchestrator
def __init__(self, enabled_drivers=None, design_state=None):
self.enabled_drivers = {}
self.enabled_drivers['oob'] = enabled_drivers.get('oob', None)
self.enabled_drivers['server'] = enabled_drivers.get('server', None)
self.enabled_drivers['network'] = enabled_drivers.get('network', None)
self.design_state = design_state
"""
execute_task
This is the core of the orchestrator. The task will describe the action
to take and the context/scope of the command. We will then source
the current designed state and current built state from the statemgmt
module. Based on those 3 inputs, we'll decide what is needed next.
"""
def execute_task(self, task):
if design_state is None:
raise Exception("Cannot execute task without initialized state manager")
class OrchestrationTask(object):
def __init__(self, action, **kwargs):
self.taskid = uuid.uuid4()
self.action = action
parent_task = kwargs.get('parent_task','')
# Validate parameters based on action
self.site = kwargs.get('site', '')
if self.site == '':
raise ValueError("Task requires 'site' parameter")
if action in [Action.VerifyNode, Action.PrepareNode,
Action.DeployNode, Action.DestroyNode]:
self.node_filter = kwargs.get('node_filter', None)
def child_task(self, action, **kwargs):
child_task = OrchestrationTask(action, parent_task=self.taskid, site=self.site, **kwargs)
return child_task

View File

@ -16,6 +16,8 @@ import logging
from copy import deepcopy
from helm_drydock.error import DesignError
class DesignStateClient(object):
def __init__(self):
@ -31,34 +33,47 @@ class DesignStateClient(object):
return a Site model populated with all components from the design state
"""
def load_design_data(self, site_name, design_state=None):
site = design_state.get_site(site_name)
def load_design_data(self, site_name, design_state=None, change_id=None):
if design_state is None:
raise ValueError("Design state is None")
networks = design_state.get_networks()
design_data = None
if change_id is None:
try:
design_data = design_state.get_design_base()
except DesignError(e):
raise e
else:
design_data = design_state.get_design_change(change_id)
site = design_data.get_site(site_name)
networks = design_data.get_networks()
for n in networks:
if n.site == site_name:
site.networks.append(n)
network_links = design_state.get_network_links()
network_links = design_data.get_network_links()
for l in network_links:
if l.site == site_name:
site.network_links.append(l)
host_profiles = design_state.get_host_profiles()
host_profiles = design_data.get_host_profiles()
for p in host_profiles:
if p.site == site_name:
site.host_profiles.append(p)
hardware_profiles = design_state.get_hardware_profiles()
hardware_profiles = design_data.get_hardware_profiles()
for p in hardware_profiles:
if p.site == site_name:
site.hardware_profiles.append(p)
baremetal_nodes = design_state.get_baremetal_nodes()
baremetal_nodes = design_data.get_baremetal_nodes()
for n in baremetal_nodes:
if n.site == site_name:
@ -66,12 +81,6 @@ class DesignStateClient(object):
return site
"""
compute_model_inheritance - given a fully populated Site model, compute the effecitve
design by applying inheritance and references
return a Site model reflecting the effective design for the site
"""
def compute_model_inheritance(self, site_root):
# For now the only thing that really incorporates inheritance is
@ -86,8 +95,15 @@ class DesignStateClient(object):
for n in site_copy.baremetal_nodes:
resolved = n.apply_host_profile(site_copy)
resolved = resolved.apply_hardware_profile(site_copy)
resolved = resolved.apply_network_connections(site_copy)
effective_nodes.append(resolved)
site_copy.baremetal_nodes = effective_nodes
return site_copy
"""
compute_model_inheritance - given a fully populated Site model,
compute the effecitve design by applying inheritance and references
return a Site model reflecting the effective design for the site
"""

View File

@ -0,0 +1,64 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from enum import Enum, unique
@unique
class Action(Enum):
Noop = 'noop'
ValidateDesign = 'validate_design'
VerifySite = 'verify_site'
PrepareSite = 'prepare_site'
VerifyNode = 'verify_node'
PrepareNode = 'prepare_node'
DeployNode = 'deploy_node'
DestroyNode = 'destroy_node'
@unique
class ActionResult(Enum):
Success = 'success'
PartialSuccess = 'partial_success'
Failure = 'failure'
DependentFailure = 'dependent_failure'
@unique
class SiteStatus(Enum):
Unknown = 'unknown'
DesignStarted = 'design_started'
DesignAvailable = 'design_available'
DesignValidated = 'design_validated'
Deploying = 'deploying'
Deployed = 'deployed'
DesignUpdated = 'design_updated'
@unique
class NodeStatus(Enum):
Unknown = 'unknown'
Designed = 'designed'
Present = 'present' # IPMI access verified
BasicVerifying = 'basic_verifying' # Base node verification in process
FailedBasicVerify = 'failed_basic_verify' # Base node verification failed
BasicVerified = 'basic_verified' # Base node verification successful
Preparing = 'preparing' # Node preparation in progress
FailedPrepare = 'failed_prepare' # Node preparation failed
Prepared = 'prepared' # Node preparation complete
FullyVerifying = 'fully_verifying' # Node full verification in progress
FailedFullVerify = 'failed_full_verify' # Node full verification failed
FullyVerified = 'fully_verified' # Deeper verification successful
Deploying = 'deploy' # Node deployment in progress
FailedDeploy = 'failed_deploy' # Node deployment failed
Deployed = 'deployed' # Node deployed successfully
Bootstrapping = 'bootstrapping' # Node bootstrapping
FailedBootstrap = 'failed_bootstrap' # Node bootstrapping failed
Bootstrapped = 'bootstrapped' # Node fully bootstrapped
Complete = 'complete' # Node is complete

View File

@ -0,0 +1,83 @@
# Orchestrator #
The orchestrator is the core of drydock and will manage
the ordering of driver actions to implement the main Drydock
actions. Each of these actions will be started by the
external cLCP orchestrator with different parameters to
control the scope of the action.
Orchestrator should persist the state of each task
such that on failure the task can retried and only the
steps needed will be executed.
Bullet points listed below are not exhaustive and will
change as we move through testing
## ValidateDesign ##
Load design data from the statemgmt persistent store and
validate that the current state of design data represents
a valid site design. No claim is made that the design data
is compatible with the physical state of the site.
## VerifySite ##
Verify site-wide resources are in a useful state
* Driver downstream resources are reachable (e.g. MaaS)
* OS images needed for bootstrapping are available
* Promenade or other next-step services are up and available
* Verify credentials are available
## PrepareSite ##
Begin preparing site-wide resources for bootstrapping. This
action will lock site design data for changes.
* Configure bootstrapper with site network configs
* Shuffle images so they are correctly configured for bootstrapping
## VerifyNode ##
Verification of per-node configurations within the context
of the current node status
* Status: Present
* Basic hardware verification as available via OOB driver
- BIOS firmware
- PCI layout
- Drives
- Hardware alarms
* IPMI connectivity
* Status: Prepared
- Full hardware manifest
- Possibly network connectivity
- Firmware versions
## PrepareNode ##
Prepare a node for bootstrapping
* Configure network port for PXE
* Configure a node for PXE boot
* Power-cycle the node
* Setup commissioning configuration
- Hardware drivers
- Hardware configuration (e.g. RAID)
* Configure node networking
* Configure node storage
## DeployNode ##
Begin bootstrapping the node and monitor
success
* Initialize the Introspection service for the node
* Bootstrap the node (i.e. Write persistent OS install)
* Ensure network port is returned to production configuration
* Reboot node from local disk
* Monitor platform bootstrapping
## DestroyNode ##
Destroy current node configuration and rebootstrap from scratch

View File

@ -11,23 +11,132 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from copy import deepcopy
from datetime import datetime
from datetime import timezone
import helm_drydock.model as model
import uuid
import helm_drydock.model.node as node
import helm_drydock.model.hostprofile as hostprofile
import helm_drydock.model.network as network
import helm_drydock.model.site as site
import helm_drydock.model.hwprofile as hwprofile
from helm_drydock.error import DesignError
class DesignState(object):
def __init__(self):
self.design_base = None
self.design_changes = []
self.builds = []
return
# TODO Need to lock a design base or change once implementation
# has started
def get_design_base(self):
if self.design_base is None:
raise DesignError("No design base submitted")
return deepcopy(self.design_base)
def post_design_base(self, site_design):
if site_design is not None and isinstance(site_design, SiteDesign):
self.design_base = deepcopy(site_design)
return True
def put_design_base(self, site_design):
# TODO Support merging
if site_design is not None and isinstance(site_design, SiteDesign):
self.design_base = deepcopy(site_design)
return True
def get_design_change(self, changeid):
match = [x for x in self.design_changes if x.changeid == changeid]
if len(match) == 0:
raise DesignError("No design change %s found." % (changeid))
else:
return deepcopy(match[0])
def post_design_change(self, site_design):
if site_design is not None and isinstance(site_design, SiteDesign):
exists = [(x) for x
in self.design_changes
if x.changeid == site_design.changeid]
if len(exists) > 0:
raise DesignError("Existing change %s found" %
(site_design.changeid))
self.design_changes.append(deepcopy(site_design))
return True
else:
raise DesignError("Design change must be a SiteDesign instance")
def put_design_change(self, site_design):
# TODO Support merging
if site_design is not None and isinstance(site_design, SiteDesign):
design_copy = deepcopy(site_design)
self.design_changes = [design_copy
if x.changeid == design_copy.changeid
else x
for x
in self.design_changes]
return True
else:
raise DesignError("Design change must be a SiteDesign instance")
def get_current_build(self):
latest_stamp = 0
current_build = None
for b in self.builds:
if b.build_id > latest_stamp:
latest_stamp = b.build_id
current_build = b
return deepcopy(current_build)
def get_build(self, build_id):
for b in self.builds:
if b.build_id == build_id:
return b
return None
def post_build(self, site_build):
if site_build is not None and isinstance(site_build, SiteBuild):
exists = [b for b in self.builds
if b.build_id == site_build.build_id]
if len(exists) > 0:
raise DesignError("Already a site build with ID %s" %
(str(site_build.build_id)))
else:
self.builds.append(deepcopy(site_build))
return True
class SiteDesign(object):
def __init__(self, ischange=False):
if ischange:
self.changeid = uuid.uuid4()
else:
self.changeid = 0
self.sites = []
self.networks = []
self.network_links = []
self.host_profiles = []
self.hardware_profiles = []
self.baremetal_nodes = []
return
def add_site(self, new_site):
if new_site is None or not isinstance(new_site, model.Site):
raise Exception("Invalid Site model")
if new_site is None or not isinstance(new_site, site.Site):
raise DesignError("Invalid Site model")
self.sites.append(new_site)
@ -39,11 +148,11 @@ class DesignState(object):
if s.name == site_name:
return s
raise NameError("Site %s not found in design state" % site_name)
raise DesignError("Site %s not found in design state" % site_name)
def add_network(self, new_network):
if new_network is None or not isinstance(new_network, model.Network):
raise Exception("Invalid Network model")
if new_network is None or not isinstance(new_network, network.Network):
raise DesignError("Invalid Network model")
self.networks.append(new_network)
@ -55,11 +164,13 @@ class DesignState(object):
if n.name == network_name:
return n
raise NameError("Network %s not found in design state" % network_name)
raise DesignError("Network %s not found in design state"
% network_name)
def add_network_link(self, new_network_link):
if new_network_link is None or not isinstance(new_network_link, model.NetworkLink):
raise Exception("Invalid NetworkLink model")
if new_network_link is None or not isinstance(new_network_link,
network.NetworkLink):
raise DesignError("Invalid NetworkLink model")
self.network_links.append(new_network_link)
@ -71,11 +182,13 @@ class DesignState(object):
if l.name == link_name:
return l
raise NameError("NetworkLink %s not found in design state" % link_name)
raise DesignError("NetworkLink %s not found in design state"
% link_name)
def add_host_profile(self, new_host_profile):
if new_host_profile is None or not isinstance(new_host_profile, model.HostProfile):
raise Exception("Invalid HostProfile model")
if new_host_profile is None or not isinstance(new_host_profile,
hostprofile.HostProfile):
raise DesignError("Invalid HostProfile model")
self.host_profiles.append(new_host_profile)
@ -87,11 +200,13 @@ class DesignState(object):
if p.name == profile_name:
return p
raise NameError("HostProfile %s not found in design state" % profile_name)
raise DesignError("HostProfile %s not found in design state"
% profile_name)
def add_hardware_profile(self, new_hardware_profile):
if new_hardware_profile is None or not isinstance(new_hardware_profile, model.HardwareProfile):
raise Exception("Invalid HardwareProfile model")
if (new_hardware_profile is None or
not isinstance(new_hardware_profile, hwprofile.HardwareProfile)):
raise DesignError("Invalid HardwareProfile model")
self.hardware_profiles.append(new_hardware_profile)
@ -103,11 +218,13 @@ class DesignState(object):
if p.name == profile_name:
return p
raise NameError("HardwareProfile %s not found in design state" % profile_name)
raise DesignError("HardwareProfile %s not found in design state"
% profile_name)
def add_baremetal_node(self, new_baremetal_node):
if new_baremetal_node is None or not isinstance(new_baremetal_node, model.BaremetalNode):
raise Exception("Invalid BaremetalNode model")
if (new_baremetal_node is None or
not isinstance(new_baremetal_node, node.BaremetalNode)):
raise DesignError("Invalid BaremetalNode model")
self.baremetal_nodes.append(new_baremetal_node)
@ -119,4 +236,59 @@ class DesignState(object):
if n.name == node_name:
return n
raise NameError("BaremetalNode %s not found in design state" % node_name)
raise DesignError("BaremetalNode %s not found in design state"
% node_name)
class SiteBuild(SiteDesign):
def __init__(self, build_id=None):
super(SiteBuild, self).__init__()
if build_id is None:
self.build_id = datetime.datetime.now(timezone.utc).timestamp()
else:
self.build_id = build_id
def get_filtered_nodes(self, node_filter):
effective_nodes = self.get_baremetal_nodes()
# filter by rack
rack_filter = node_filter.get('rackname', None)
if rack_filter is not None:
rack_list = rack_filter.split(',')
effective_nodes = [x
for x in effective_nodes
if x.get_rack() in rack_list]
# filter by name
name_filter = node_filter.get('nodename', None)
if name_filter is not None:
name_list = name_filter.split(',')
effective_nodes = [x
for x in effective_nodes
if x.get_name() in name_list]
# filter by tag
tag_filter = node_filter.get('tags', None)
if tag_filter is not None:
tag_list = tag_filter.split(',')
effective_nodes = [x
for x in effective_nodes
for t in tag_list
if x.has_tag(t)]
return effective_nodes
"""
Support filtering on rack name, node name or node tag
for now. Each filter can be a comma-delimited list of
values. The final result is an intersection of all the
filters
"""
def set_nodes_status(self, node_filter, status):
target_nodes = self.get_filtered_nodes(node_filter)
for n in target_nodes:
n.set_status(status)

View File

@ -0,0 +1,41 @@
# Statemgmt #
Statemgmt is the interface to the persistence store
for holding site design data as well as build status data
/drydock - Base namespace for drydock data
## As Designed ##
Serialization of Drydock internal model as ingested. Not externally writable.
/drydock/design
/drydock/design/base - The base site design used for the first deployment
/drydock/design/[changeID] - A change layer on top of the base site design. Chrono ordered
## As Built ##
Serialization of Drydock internal model as rendered to effective implementation including build status. Not externally writable.
/drydock/build
/drydock/build/[datestamp] - A point-in-time view of what was deployed with deployment results
## Node data ##
Per-node data that can drive introspection as well as accept updates from nodes
/drydock/nodedata
/drydock/nodedata/[nodename] - Per-node data, can be seeded with Ingested metadata but can be updated during by orchestrator or node
## Service data ##
Per-service data (may not be needed)
/drydock/servicedata
/drydock/servicedata/[servicename]
## Global data ##
Generic k:v store that can be produced/consumed by non-Drydock services via the Drydock API
/drydock/globaldata

View File

@ -1,11 +0,0 @@
[tox]
envlist = py35
[testenv]
deps=
-rrequirements.txt
setenv=
PYTHONWARNING=all
[flake8]
ignore=E302,H306

View File

@ -1,7 +1,5 @@
PyYAML
oauth
requests-oauthlib
pyipmi
netaddr
pecan
python-libmaas
PyYAML==3.12
oauth==1.0.1
requests-oauthlib==0.8.0
netaddr==0.7.19
python-libmaas==0.4.1

View File

@ -50,7 +50,7 @@ setup(name='helm_drydock',
'PyYAML',
'oauth',
'requests-oauthlib',
'pyipmi',
'pyghmi',
'netaddr',
'pecan',
'webob'

View File

@ -13,7 +13,7 @@
# limitations under the License.
from helm_drydock.ingester import Ingester
from helm_drydock.statemgmt import DesignState
from helm_drydock.statemgmt import DesignState, SiteDesign
from helm_drydock.orchestrator.designdata import DesignStateClient
from copy import deepcopy
@ -34,6 +34,11 @@ class TestClass(object):
client = DesignStateClient()
design_data = client.load_design_data("sitename", design_state=loaded_design)
assert len(design_data.baremetal_nodes) == 2
print(yaml.dump(design_data, default_flow_style=False))
design_data = client.compute_model_inheritance(design_data)
node = design_data.get_baremetal_node("controller01")
@ -54,13 +59,15 @@ class TestClass(object):
def loaded_design(self, input_files):
input_file = input_files.join("fullsite.yaml")
module_design_state = DesignState()
design_state = DesignState()
design_data = SiteDesign()
design_state.post_design_base(design_data)
ingester = Ingester()
ingester.enable_plugins([helm_drydock.ingester.plugins.yaml.YamlIngester])
ingester.ingest_data(plugin_name='yaml', design_state=module_design_state, filenames=[str(input_file)])
ingester.ingest_data(plugin_name='yaml', design_state=design_state, filenames=[str(input_file)])
return module_design_state
return design_state

View File

@ -13,7 +13,7 @@
# limitations under the License.
from helm_drydock.ingester import Ingester
from helm_drydock.statemgmt import DesignState
from helm_drydock.statemgmt import DesignState, SiteDesign
import pytest
import shutil
@ -27,14 +27,19 @@ class TestClass(object):
def test_ingest_full_site(self, input_files):
input_file = input_files.join("fullsite.yaml")
design_state = DesignState()
design_data = SiteDesign()
design_state.post_design_base(design_data)
ingester = Ingester()
ingester.enable_plugins([helm_drydock.ingester.plugins.yaml.YamlIngester])
ingester.ingest_data(plugin_name='yaml', design_state=design_state, filenames=[str(input_file)])
assert len(design_state.get_host_profiles()) == 3
assert len(design_state.get_baremetal_nodes()) == 2
design_data = design_state.get_design_base()
assert len(design_data.get_host_profiles()) == 3
assert len(design_data.get_baremetal_nodes()) == 2
def test_ingest_federated_design(self, input_files):
profiles_file = input_files.join("fullsite_profiles.yaml")
@ -42,13 +47,17 @@ class TestClass(object):
nodes_file = input_files.join("fullsite_nodes.yaml")
design_state = DesignState()
design_data = SiteDesign()
design_state.post_design_base(design_data)
ingester = Ingester()
ingester.enable_plugins([helm_drydock.ingester.plugins.yaml.YamlIngester])
ingester.ingest_data(plugin_name='yaml', design_state=design_state,
filenames=[str(profiles_file), str(networks_file), str(nodes_file)])
assert len(design_state.host_profiles) == 3
design_data = design_state.get_design_base()
assert len(design_data.host_profiles) == 3
@pytest.fixture(scope='module')
def input_files(self, tmpdir_factory, request):

View File

@ -14,7 +14,7 @@
import pytest
import yaml
from helm_drydock.model import HardwareProfile
from helm_drydock.model.hwprofile import HardwareProfile
class TestClass(object):

View File

@ -25,8 +25,7 @@ metadata:
date: 17-FEB-2017
description: Sample site design
author: sh8121@att.com
spec:
# Not sure if we have site wide data that doesn't fall into another 'Kind'
# Not sure if we have site wide data that doesn't fall into another 'Kind'
---
apiVersion: 'v1.0'
kind: NetworkLink
@ -454,4 +453,4 @@ spec:
scsi:
- address: scsi@2:0.0.0
alias: primary_boot
type: 'VBOX HARDDISK'
type: 'VBOX HARDDISK'

View File

@ -225,4 +225,4 @@ spec:
metric: 9
dns:
domain: sitename.example.com
servers: 8.8.8.8
servers: 8.8.8.8

View File

@ -25,8 +25,7 @@ metadata:
date: 17-FEB-2017
description: Sample site design
author: sh8121@att.com
spec:
# Not sure if we have site wide data that doesn't fall into another 'Kind'
# Not sure if we have site wide data that doesn't fall into another 'Kind'
---
apiVersion: 'v1.0'
kind: HostProfile
@ -194,4 +193,4 @@ spec:
scsi:
- address: scsi@2:0.0.0
alias: primary_boot
type: 'VBOX HARDDISK'
type: 'VBOX HARDDISK'