airship
/
drydock
Code Issues Proposed changes

Merge branch 'start_coding'

This commit is contained in:
Scott Hussey 2017-02-28 15:18:52 -06:00
parent 014990b440 f1e2a3f343
commit 3a331f6c51
30 changed files with 2239 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
README.md
View File

@ -1,33 +1,33 @@
# drydock
# helm_drydock
A python REST orchestrator to translate a YAML host topology to a provisioned set of hosts and provide a set of cloud-init post-provisioning instructions.
## Modular service
### Design Consumer ###
aka smelter
aka ingester
Pluggable service to ingest a inventory/design specification, convert it to a standard
internal representaion, and persist it to the Design State API. Initial implementation
is the consumer of AIC YAML schema.
is the consumer of YAML schema.
### Design State API ###
aka tarot
aka statemgmt
API for querying and updating the current design specification and persisted orchestration status.
CRUD support of CIs that are not bootstrap-related, but can be used by other automation.
### Control API ###
aka cockpit
aka control
User-approachable API for initiating orchestration actions or accessing other internal
APIs
### Infrastructure Orchestrator ###
aka alchemist
aka orchestrator
Handle validation of complete design, ordering and managing downstream API calls for hardware
provisioning/bootstrapping
@ -38,13 +38,9 @@ aka maasdriver
Pluggable provisioner for server bootstrapping. Initial implementation is MaaS client.
### Network Driver ###
Pluggable provisioner for network provisioning. Initial implementation is Noop.
### Introspection API ###
aka jabberwocky
aka introspection
API for bootstrapping nodes to load self data. Possibly pluggable as this is basically an
authenticated bridge to the Design State API

3
examples/bootstrap_seed.yaml
View File

@ -397,7 +397,8 @@ spec:
- network: public
address: 172.16.3.20
metadata:
roles: os_ctl
tags:
- os_ctl
rack: rack01
---
apiVersion: 'v1.0'

13
helm_drydock/__init__.py Normal file
View File

@ -0,0 +1,13 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

41
helm_drydock/config.py Normal file
View File

@ -0,0 +1,41 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#
# Read application configuration
#
# configuration map with defaults
class DrydockConfig(object):
def __init__(self):
self.server_driver_config = {
selected_driver = helm_drydock.drivers.server.maasdriver,
params = {
maas_api_key = ""
maas_api_url = ""
}
}
self.selected_network_driver = helm_drydock.drivers.network.noopdriver
self.control_config = {}
self.ingester_config = {
plugins = [helm_drydock.ingester.plugins.aicyaml.AicYamlIngester]
}
self.introspection_config = {}
self.orchestrator_config = {}
self.statemgmt_config = {
backend_driver = helm_drydock.drivers.statemgmt.etcd,
}

13
helm_drydock/control/__init__.py Normal file
View File

@ -0,0 +1,13 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

35
helm_drydock/control/root.py Normal file
View File

@ -0,0 +1,35 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from pecan import expose
from webob.exc import status_map
class RootController(object):
@expose(generic=True, template='index.html')
def index(self):
return dict()
@index.when(method='POST')
def index_post(self, q):
redirect('https://pecan.readthedocs.io/en/latest/search.html?q=%s' % q)
@expose('error.html')
def error(self, status):
try:
status = int(status)
except ValueError:
status = 0
message = getattr(status_map.get(status), 'explanation', '')
return dict(status=status, message=message)

19
helm_drydock/drivers/__init__.py Normal file
View File

@ -0,0 +1,19 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
class ProviderDriver(object):
__init__(self):
pass

18
helm_drydock/drivers/server/__init__.py Normal file
View File

@ -0,0 +1,18 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import helm_drydock.drivers.ProviderDriver
class ServerDriver(ProviderDriver):

17
helm_drydock/drivers/server/maasdriver/__init__.py Normal file
View File

@ -0,0 +1,17 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import helm_drydock.drivers.server.ServerDriver
class MaasServerDriver(object):

90
helm_drydock/ingester/__init__.py Normal file
View File

@ -0,0 +1,90 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# ingester - Ingest host topologies to define site design and
# persist design to helm-drydock's statemgmt service
import logging
import yaml
import helm_drydock.model as model
from helm_drydock.statemgmt import DesignState
class Ingester(object):
def __init__(self):
logging.basicConfig(format="%(asctime)-15s [%(levelname)] %(module)s %(process)d %(message)s")
self.log = logging.Logger("ingester")
self.registered_plugins = {}
"""
enable_plugins
params: plugins - A list of class objects denoting the ingester plugins to be enabled
Enable plugins that can be used for ingest_data calls. Each plugin should use
helm_drydock.ingester.plugins.IngesterPlugin as its base class. As long as one
enabled plugin successfully initializes, the call is considered successful. Otherwise
it will throw an exception
"""
def enable_plugins(self, plugins=[]):
if len(plugins) == 0:
self.log.error("Cannot have an empty plugin list.")
for plugin in plugins:
try:
new_plugin = plugin()
plugin_name = new_plugin.get_name()
self.registered_plugins[plugin_name] = new_plugin
except:
self.log.error("Could not enable plugin %s" % (plugin.__name__))
if len(self.registered_plugins) == 0:
self.log.error("Could not enable at least one plugin")
raise Exception("Could not enable at least one plugin")
"""
ingest_data
params: plugin_name - Which plugin should be used for ingestion
params: params - A map of parameters that will be passed to the plugin's ingest_data method
Execute a data ingestion using the named plugin (assuming it is enabled)
"""
def ingest_data(self, plugin_name='', design_state=None, **kwargs):
if design_state is None:
self.log.error("ingest_data called without valid DesignState handler")
raise Exception("Invalid design_state handler")
if plugin_name in self.registered_plugins:
design_data = self.registered_plugins[plugin_name].ingest_data(**kwargs)
# Need to persist data here, but we don't yet have the statemgmt service working
for m in design_data:
if type(m) is model.Site:
design_state.add_site(m)
elif type(m) is model.Network:
design_state.add_network(m)
elif type(m) is model.NetworkLink:
design_state.add_network_link(m)
elif type(m) is model.HostProfile:
design_state.add_host_profile(m)
elif type(m) is model.HardwareProfile:
design_state.add_hardware_profile(m)
elif type(m) is model.BaremetalNode:
design_state.add_baremetal_node(m)
else:
self.log.error("Could not find plugin %s to ingest data." % (plugin_name))
raise LookupError("Could not find plugin %s" % plugin_name)

30
helm_drydock/ingester/plugins/__init__.py Normal file
View File

@ -0,0 +1,30 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Plugins to parse incoming topology and translate it to helm-drydock's
# model representation
import logging
class IngesterPlugin(object):
def __init__(self):
self.log = logging.Logger('ingester')
return
def get_data(self):
return "ingester_skeleton"
def ingest_data(self, **kwargs):
return {}

99
helm_drydock/ingester/plugins/aicyaml.py Normal file
View File

@ -0,0 +1,99 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# AIC YAML Ingester - This data ingester will consume a AIC YAML design
# file
#
import yaml
import logging
import helm_drydock.model as model
from helm_drydock.ingester.plugins import IngesterPlugin
class AicYamlIngester(IngesterPlugin):
kind_map = {
"Region": model.Site,
"NetworkLink": model.NetworkLink,
"HardwareProfile": model.HardwareProfile,
"Network": model.Network,
"HostProfile": model.HostProfile,
"BaremetalNode": model.BaremetalNode,
}
def __init__(self):
super(AicYamlIngester, self).__init__()
def get_name(self):
return "aic_yaml"
"""
AIC YAML ingester params
filenames - Array of absolute path to the YAML files to ingest
returns an array of objects from helm_drydock.model
"""
def ingest_data(self, **kwargs):
if 'filenames' in kwargs:
# TODO validate filenames is array
for f in kwargs.get('filenames'):
try:
file = open(f,'rt')
contents = file.read()
file.close()
except OSError as err:
self.log.error(
"Error opening input file %s for ingestion: %s"
% (filename, err))
continue
try:
parsed_data = yaml.load_all(contents)
except yaml.YAMLError as err:
self.log.error("Error parsing YAML in %s: %s" % (f, err))
continue
models = []
for d in parsed_data:
kind = d.get('kind', '')
if kind != '':
if kind in AicYamlIngester.kind_map:
try:
model = AicYamlIngester.kind_map[kind](**d)
models.append(model)
except Exception as err:
self.log.error("Error building model %s: %s"
% (kind, str(err)))
else:
self.log.error(
"Error processing document, unknown kind %s"
% (kind))
continue
else:
self.log.error(
"Error processing document in %s, no kind field"
% (f))
continue
return models
else:
raise ValueError('Missing parameter "filename"')
return processed_data

671
helm_drydock/model/__init__.py Normal file
View File

@ -0,0 +1,671 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Models for helm_drydock
#
from copy import deepcopy
class HardwareProfile(object):
def __init__(self, **kwargs):
self.api_version = kwargs.get('apiVersion', '')
if self.api_version == "v1.0":
metadata = kwargs.get('metadata', {})
spec = kwargs.get('spec', {})
# Need to add validation logic, we'll assume the input is
# valid for now
self.name = metadata.get('name', '')
self.site = metadata.get('region', '')
self.vendor = spec.get('vendor', None)
self.generation = spec.get('generation', None)
self.hw_version = spec.get('hw_version', None)
self.bios_version = spec.get('bios_version', None)
self.boot_mode = spec.get('boot_mode', None)
self.bootstrap_protocol = spec.get('bootstrap_protocol', None)
self.pxe_interface = spec.get('pxe_interface', None)
self.devices = []
device_aliases = spec.get('device_aliases', {})
pci_devices = device_aliases.get('pci', [])
scsi_devices = device_aliases.get('scsi', [])
for d in pci_devices:
d['bus_type'] = 'pci'
self.devices.append(
HardwareDeviceAlias(self.api_version, **d))
for d in scsi_devices:
d['bus_type'] = 'scsi'
self.devices.append(
HardwareDeviceAlias(self.api_version, **d))
else:
raise ValueError('Unknown API version of object')
return
def resolve_alias(self, alias_type, alias):
for d in self.devices:
if d.alias == alias and d.bus_type == alias_type:
return deepcopy(d)
return None
class HardwareDeviceAlias(object):
def __init__(self, api_version, **kwargs):
self.api_version = api_version
if self.api_version == "v1.0":
self.bus_type = kwargs.get('bus_type', None)
self.address = kwargs.get('address', None)
self.alias = kwargs.get('alias', None)
self.type = kwargs.get('type', None)
else:
raise ValueError('Unknown API version of object')
class Site(object):
def __init__(self, **kwargs):
self.api_version = kwargs.get('apiVersion', '')
if self.api_version == "v1.0":
metadata = kwargs.get('metadata', {})
# Need to add validation logic, we'll assume the input is
# valid for now
self.name = metadata.get('name', '')
self.networks = []
self.network_links = []
self.host_profiles = []
self.hardware_profiles = []
self.baremetal_nodes = []
else:
raise ValueError('Unknown API version of object')
def get_network(self, network_name):
for n in self.networks:
if n.name == network_name:
return n
return None
def get_network_link(self, link_name):
for l in self.network_links:
if l.name == link_name:
return l
return None
def get_host_profile(self, profile_name):
for p in self.host_profiles:
if p.name == profile_name:
return p
return None
def get_hardware_profile(self, profile_name):
for p in self.hardware_profiles:
if p.name == profile_name:
return p
return None
def get_baremetal_node(self, node_name):
for n in self.baremetal_nodes:
if n.name == node_name:
return n
return None
class NetworkLink(object):
def __init__(self, **kwargs):
self.api_version = kwargs.get('apiVersion', '')
if self.api_version == "v1.0":
metadata = kwargs.get('metadata', {})
spec = kwargs.get('spec', {})
self.name = metadata.get('name', '')
self.site = metadata.get('region', '')
bonding = spec.get('bonding', {})
self.bonding_mode = bonding.get('mode', 'none')
# TODO How should we define defaults for CIs not in the input?
if self.bonding_mode == '802.3ad':
self.bonding_xmit_hash = bonding.get('hash', 'layer3+4')
self.bonding_peer_rate = bonding.get('peer_rate', 'fast')
self.bonding_mon_rate = bonding.get('mon_rate', '')
self.bonding_up_delay = bonding.get('up_delay', '')
self.bonding_down_delay = bonding.get('down_delay', '')
self.mtu = spec.get('mtu', 1500)
self.linkspeed = spec.get('linkspeed', 'auto')
trunking = spec.get('trunking', {})
self.trunk_mode = trunking.get('mode', 'none')
self.native_network = spec.get('default_network', '')
else:
raise ValueError('Unknown API version of object')
class Network(object):
def __init__(self, **kwargs):
self.api_version = kwargs.get('apiVersion', '')
if self.api_version == "v1.0":
metadata = kwargs.get('metadata', {})
spec = kwargs.get('spec', {})
self.name = metadata.get('name', '')
self.site = metadata.get('region', '')
self.cidr = spec.get('cidr', None)
self.allocation_strategy = spec.get('allocation', 'static')
self.vlan_id = spec.get('vlan_id', 1)
self.mtu = spec.get('mtu', 0)
dns = spec.get('dns', {})
self.dns_domain = dns.get('domain', 'local')
self.dns_servers = dns.get('servers', None)
ranges = spec.get('ranges', [])
self.ranges = []
for r in ranges:
self.ranges.append(NetworkAddressRange(self.api_version, **r))
routes = spec.get('routes', [])
self.routes = []
for r in routes:
self.routes.append(NetworkRoute(self.api_version, **r))
else:
raise ValueError('Unknown API version of object')
class NetworkAddressRange(object):
def __init__(self, api_version, **kwargs):
self.api_version = api_version
if self.api_version == "v1.0":
self.type = kwargs.get('type', None)
self.start = kwargs.get('start', None)
self.end = kwargs.get('end', None)
else:
raise ValueError('Unknown API version of object')
class NetworkRoute(object):
def __init__(self, api_version, **kwargs):
self.api_version = api_version
if self.api_version == "v1.0":
self.type = kwargs.get('subnet', None)
self.start = kwargs.get('gateway', None)
self.end = kwargs.get('metric', 100)
else:
raise ValueError('Unknown API version of object')
class HostProfile(object):
def __init__(self, **kwargs):
self.api_version = kwargs.get('apiVersion', '')
if self.api_version == "v1.0":
metadata = kwargs.get('metadata', {})
spec = kwargs.get('spec', {})
self.name = metadata.get('name', '')
self.site = metadata.get('region', '')
self.parent_profile = spec.get('host_profile', None)
self.hardware_profile = spec.get('hardware_profile', None)
oob = spec.get('oob', {})
self.oob_type = oob.get('type', None)
self.oob_network = oob.get('network', None)
self.oob_account = oob.get('account', None)
self.oob_credential = oob.get('credential', None)
storage = spec.get('storage', {})
self.storage_layout = storage.get('layout', 'lvm')
bootdisk = storage.get('bootdisk', {})
self.bootdisk_device = bootdisk.get('device', None)
self.bootdisk_root_size = bootdisk.get('root_size', None)
self.bootdisk_boot_size = bootdisk.get('boot_size', None)
partitions = storage.get('partitions', [])
self.partitions = []
for p in partitions:
self.partitions.append(HostPartition(self.api_version, **p))
interfaces = spec.get('interfaces', [])
self.interfaces = []
for i in interfaces:
self.interfaces.append(HostInterface(self.api_version, **i))
metadata = spec.get('metadata', {})
metadata_tags = metadata.get('tags', [])
self.tags = []
for t in metadata_tags:
self.tags.append(t)
owner_data = metadata.get('owner_data', {})
self.owner_data = {}
for k, v in owner_data.items():
self.owner_data[k] = v
self.rack = metadata.get('rack', None)
else:
raise ValueError('Unknown API version of object')
def apply_inheritance(self, site):
# We return a deep copy of the profile so as not to corrupt
# the original model
self_copy = deepcopy(self)
if self.parent_profile is None:
return self_copy
parent = site.get_host_profile(self.parent_profile)
if parent is None:
return self_copy
parent = parent.apply_inheritance(site)
# First compute inheritance for simple fields
inheritable_field_list = [
"hardware_profile", "oob_type", "oob_network",
"oob_credential", "oob_account", "storage_layout",
"bootdisk_device", "bootdisk_root_size", "bootdisk_boot_size",
"rack"]
for f in inheritable_field_list:
setattr(self_copy, f,
Utils.apply_field_inheritance(getattr(self, f, None),
getattr(parent, f, None)))
# Now compute inheritance for complex types
self_copy.tags = Utils.merge_lists(self.tags, parent.tags)
self_copy.owner_data = Utils.merge_dicts(
self.owner_data, parent.owner_data)
self_copy.interfaces = HostInterface.merge_lists(
self.interfaces, parent.interfaces)
self_copy.partitions = HostPartition.merge_lists(
self.partitions, parent.partitions)
return self_copy
class HostInterface(object):
def __init__(self, api_version, **kwargs):
self.api_version = api_version
if self.api_version == "v1.0":
self.device_name = kwargs.get('device_name', None)
self.network_link = kwargs.get('device_link', None)
self.hardware_slaves = []
slaves = kwargs.get('slaves', [])
for s in slaves:
self.hardware_slaves.append(s)
self.networks = []
networks = kwargs.get('networks', [])
for n in networks:
self.networks.append(n)
else:
raise ValueError('Unknown API version of object')
# The device attribute may be hardware alias that translates to a
# physical device address. If the device attribute does not match an
# alias, we assume it directly identifies a OS device name. When the
# apply_hardware_profile method is called on the parent Node of this
# device, the selector will be decided and applied
def add_selector(self, sel_type, selector):
if getattr(self, 'selectors', None) is None:
self.selectors = []
new_selector = {}
new_selector['selector_type'] = sel_type
new_selector['selector'] = selector
self.selectors.append(new_selector)
"""
Merge two lists of HostInterface models with child_list taking
priority when conflicts. If a member of child_list has a device_name
beginning with '!' it indicates that HostInterface should be
removed from the merged list
"""
@staticmethod
def merge_lists(child_list, parent_list):
if len(child_list) == 0:
return deepcopy(parent_list)
effective_list = []
if len(parent_list) == 0:
for i in child_list:
if i.device_name.startswith('!'):
continue
else:
effective_list.append(deepcopy(i))
parent_interfaces = []
for i in parent_list:
parent_name = i.device_name
parent_interfaces.append(parent_name)
for j in child_list:
if j.device_name == ("!" + parent_name):
break
elif j.device_name == parent_name:
m = HostInterface(j.api_version)
m.device_name = j.device_name
m.network_link = \
Utils.apply_field_inheritance(j.network_link,
i.network_link)
s = filter(lambda x: ("!" + x) not in j.hardware_slaves,
i.hardware_slaves)
s = list(s)
s.extend(filter(lambda x: not x.startswith("!"),
j.hardware_slaves))
m.hardware_slaves = s
n = filter(lambda x: ("!" + x) not in j.networks,
i.networks)
n = list(n)
n.extend(filter(lambda x: not x.startswith("!"),
j.networks))
m.networks = n
effective_list.append(m)
for j in child_list:
if j.device_name not in parent_list:
effective_list.append(deepcopy(j))
return effective_list
class HostPartition(object):
def __init__(self, api_version, **kwargs):
self.api_version = api_version
if self.api_version == "v1.0":
self.name = kwargs.get('name', None)
self.device = kwargs.get('device', None)
self.part_uuid = kwargs.get('part_uuid', None)
self.size = kwargs.get('size', None)
self.mountpoint = kwargs.get('mountpoint', None)
self.fstype = kwargs.get('fstype', 'ext4')
self.mount_options = kwargs.get('mount_options', 'defaults')
self.fs_uuid = kwargs.get('fs_uuid', None)
self.fs_label = kwargs.get('fs_label', None)
else:
raise ValueError('Unknown API version of object')
# The device attribute may be hardware alias that translates to a
# physical device address. If the device attribute does not match an
# alias, we assume it directly identifies a OS device name. When the
# apply_hardware_profile method is called on the parent Node of this
# device, the selector will be decided and applied
def set_selector(self, sel_type, selector):
self.selector_type = sel_type
self.selector = selector
"""
Merge two lists of HostPartition models with child_list taking
priority when conflicts. If a member of child_list has a name
beginning with '!' it indicates that HostPartition should be
removed from the merged list
"""
@staticmethod
def merge_lists(child_list, parent_list):
if len(child_list) == 0:
return deepcopy(parent_list)
effective_list = []
if len(parent_list) == 0:
for i in child_list:
if i.name.startswith('!'):
continue
else:
effective_list.append(deepcopy(i))
inherit_field_list = ["device", "part_uuid", "size",
"mountpoint", "fstype", "mount_options",
"fs_uuid", "fs_label"]
parent_partitions = []
for i in parent_list:
parent_name = i.name
parent_partitions.append(parent_name)
for j in child_list:
if j.name == ("!" + parent_name):
break
elif j.name == parent_name:
p = HostPartition(j.api_version)
p.name = j.name
for f in inherit_field_list:
setattr(p, Utils.apply_field_inheritance(getattr(j, f),
getattr(i, f))
)
effective_list.append(p)
for j in child_list:
if j.name not in parent_list:
effective_list.append(deepcopy(j))
return effective_list
# A BaremetalNode is really nothing more than a physical
# instantiation of a HostProfile, so they both represent
# the same set of CIs
class BaremetalNode(HostProfile):
def __init__(self, **kwargs):
super(BaremetalNode, self).__init__(**kwargs)
def apply_host_profile(self, site):
return self.apply_inheritance(site)
# Translate device alises to physical selectors and copy
# other hardware attributes into this object
def apply_hardware_profile(self, site):
self_copy = deepcopy(self)
if self.hardware_profile is None:
raise ValueError("Hardware profile not set")
hw_profile = site.get_hardware_profile(self.hardware_profile)
for i in self_copy.interfaces:
for s in i.hardware_slaves:
selector = hw_profile.resolve_alias("pci", s)
if selector is None:
i.add_selector("name", s)
else:
i.add_selector("address", selector)
for p in self_copy.partitions:
selector = hw_profile.resolve_alias("scsi", p.device)
if selector is None:
p.set_selector("name", p.device)
else:
p.set_selector("address", selector)
hardware = {"vendor": getattr(hw_profile, 'vendor', None),
"generation": getattr(hw_profile, 'generation', None),
"hw_version": getattr(hw_profile, 'hw_version', None),
"bios_version": getattr(hw_profile, 'bios_version', None),
"boot_mode": getattr(hw_profile, 'boot_mode', None),
"bootstrap_protocol": getattr(hw_profile,
'bootstrap_protocol',
None),
"pxe_interface": getattr(hw_profile, 'pxe_interface', None)
}
self_copy.hardware = hardware
return self_copy
# Utility class for calculating inheritance
class Utils(object):
"""
apply_field_inheritance - apply inheritance rules to a single field value
param child_field - value of the child field, or the field receiving
the inheritance
param parent_field - value of the parent field, or the field supplying
the inheritance
return the correct value for child_field based on the inheritance algorithm
Inheritance algorithm
1. If child_field is not None, '!' for string vals or -1 for numeric
vals retain the value
of child_field
2. If child_field is '!' return None to unset the field value
3. If child_field is -1 return None to unset the field value
4. If child_field is None return parent_field
"""
@staticmethod
def apply_field_inheritance(child_field, parent_field):
if child_field is not None:
if child_field != '!' and child_field != -1:
return child_field
else:
return None
else:
return parent_field
"""
merge_lists - apply inheritance rules to a list of simple values
param child_list - list of values from the child
param parent_list - list of values from the parent
return a merged list with child values taking prority
1. All members in the child list not starting with '!'
2. If a member in the parent list has a corresponding member in the
chid list prefixed with '!' it is removed
3. All remaining members of the parent list
"""
@staticmethod
def merge_lists(child_list, parent_list):
if type(child_list) is not list or type(parent_list) is not list:
raise ValueError("One parameter is not a list")
effective_list = []
# Probably should handle non-string values
effective_list.extend(
filter(lambda x: not x.startswith("!"), child_list))
effective_list.extend(
filter(lambda x: ("!" + x) not in child_list,
filter(lambda x: x not in effective_list, parent_list)))
return effective_list
"""
merge_dicts - apply inheritance rules to a dict
param child_dict - dict of k:v from child
param parent_dict - dict of k:v from the parent
return a merged dict with child values taking prority
1. All members in the child dict with a key not starting with '!'
2. If a member in the parent dict has a corresponding member in the
chid dict where the key is prefixed with '!' it is removed
3. All remaining members of the parent dict
"""
@staticmethod
def merge_dicts(child_dict, parent_dict):
if type(child_dict) is not dict or type(parent_dict) is not dict:
raise ValueError("One parameter is not a dict")
effective_dict = {}
# Probably should handle non-string keys
use_keys = filter(lambda x: ("!" + x) not in child_dict.keys(),
parent_dict)
for k in use_keys:
effective_dict[k] = deepcopy(parent_dict[k])
use_keys = filter(lambda x: not x.startswith("!"), child_dict)
for k in use_keys:
effective_dict[k] = deepcopy(child_dict[k])
return effective_dict

13
helm_drydock/orchestrator/__init__.py Normal file
View File

@ -0,0 +1,13 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

100
helm_drydock/orchestrator/designdata.py Normal file
View File

@ -0,0 +1,100 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
from copy import deepcopy
class DesignStateClient(object):
def __init__(self):
self.log = logging.Logger('orchestrator')
"""
load_design_data - Pull all the defined models in statemgmt and assemble
them into a representation of the site. Does not compute inheritance.
Throws an exception if multiple Site models are found.
param design_state - Instance of statemgmt.DesignState to load data from
return a Site model populated with all components from the design state
"""
def load_design_data(self, design_state=None):
if len(design_state.get_sites()) != 1:
self.log.error("Invalid design state, should only have 1 Site model")
raise Exception("Invalid design state, should only have 1 Site model")
site = design_state.get_sites()[0]
site_name = site.name
networks = design_state.get_networks()
for n in networks:
if n.site == site_name:
site.networks.append(n)
network_links = design_state.get_network_links()
for l in network_links:
if l.site == site_name:
site.network_links.append(l)
host_profiles = design_state.get_host_profiles()
for p in host_profiles:
if p.site == site_name:
site.host_profiles.append(p)
hardware_profiles = design_state.get_hardware_profiles()
for p in hardware_profiles:
if p.site == site_name:
site.hardware_profiles.append(p)
baremetal_nodes = design_state.get_baremetal_nodes()
for n in baremetal_nodes:
if n.site == site_name:
site.baremetal_nodes.append(n)
return site
"""
compute_model_inheritance - given a fully populated Site model, compute the effecitve
design by applying inheritance and references
return a Site model reflecting the effective design for the site
"""
def compute_model_inheritance(self, site_root):
# For now the only thing that really incorporates inheritance is
# host profiles and baremetal nodes. So we'll just resolve it for
# the baremetal nodes which recursively resolves it for host profiles
# assigned to those nodes
site_copy = deepcopy(site_root)
effective_nodes = []
for n in site_copy.baremetal_nodes:
resolved = n.apply_host_profile(site_copy)
resolved = resolved.apply_hardware_profile(site_copy)
effective_nodes.append(resolved)
site_copy.baremetal_nodes = effective_nodes
return site_copy

122
helm_drydock/statemgmt/__init__.py Normal file
View File

@ -0,0 +1,122 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import helm_drydock.model as model
class DesignState(object):
def __init__(self):
self.sites = []
self.networks = []
self.network_links = []
self.host_profiles = []
self.hardware_profiles = []
self.baremetal_nodes = []
return
def add_site(self, new_site):
if new_site is None or not isinstance(new_site, model.Site):
raise Exception("Invalid Site model")
self.sites.append(new_site)
def get_sites(self):
return self.sites
def get_site(self, site_name):
for s in self.sites:
if s.name == site_name:
return s
raise NameError("Site %s not found in design state" % site_name)
def add_network(self, new_network):
if new_network is None or not isinstance(new_network, model.Network):
raise Exception("Invalid Network model")
self.networks.append(new_network)
def get_networks(self):
return self.networks
def get_network(self, network_name):
for n in self.networks:
if n.name == network_name:
return n
raise NameError("Network %s not found in design state" % network_name)
def add_network_link(self, new_network_link):
if new_network_link is None or not isinstance(new_network_link, model.NetworkLink):
raise Exception("Invalid NetworkLink model")
self.network_links.append(new_network_link)
def get_network_links(self):
return self.network_links
def get_network_link(self, link_name):
for l in self.network_links:
if l.name == link_name:
return l
raise NameError("NetworkLink %s not found in design state" % link_name)
def add_host_profile(self, new_host_profile):
if new_host_profile is None or not isinstance(new_host_profile, model.HostProfile):
raise Exception("Invalid HostProfile model")
self.host_profiles.append(new_host_profile)
def get_host_profiles(self):
return self.host_profiles
def get_host_profile(self, profile_name):
for p in self.host_profiles:
if p.name == profile_name:
return p
raise NameError("HostProfile %s not found in design state" % profile_name)
def add_hardware_profile(self, new_hardware_profile):
if new_hardware_profile is None or not isinstance(new_hardware_profile, model.HardwareProfile):
raise Exception("Invalid HardwareProfile model")
self.hardware_profiles.append(new_hardware_profile)
def get_hardware_profiles(self):
return self.hardware_profiles
def get_hardware_profile(self, profile_name):
for p in self.hardware_profiles:
if p.name == profile_name:
return p
raise NameError("HardwareProfile %s not found in design state" % profile_name)
def add_baremetal_node(self, new_baremetal_node):
if new_baremetal_node is None or not isinstance(new_baremetal_node, model.BaremetalNode):
raise Exception("Invalid BaremetalNode model")
self.baremetal_nodes.append(new_baremetal_node)
def get_baremetal_nodes(self):
return self.baremetal_nodes
def get_baremetal_node(self, node_name):
for n in self.baremetal_nodes:
if n.name == node_name:
return n
raise NameError("BaremetalNode %s not found in design state" % node_name)

11
helm_drydock/tox.ini Normal file
View File

@ -0,0 +1,11 @@
[tox]
envlist = py35
[testenv]
deps=
-rrequirements.txt
setenv=
PYTHONWARNING=all
[flake8]
ignore=E302,H306

7
requirements.txt Normal file
View File

@ -0,0 +1,7 @@
PyYAML
oauth
requests-oauthlib
pyipmi
netaddr
pecan
python-libmaas

8
setup.py
View File

@ -43,16 +43,20 @@ setup(name='helm_drydock',
'helm_drydock.model',
'helm_drydock.ingester',
'helm_drydock.ingester.plugins',
'helm_drydock.statemgmt'],
'helm_drydock.statemgmt',
'helm_drydock.orchestrator',
'helm_drydock.control'],
install_requires=[
'PyYAML',
'oauth',
'requests-oauthlib',
'pyipmi',
'netaddr',
'pecan'
'pecan',
'webob'
],
dependency_link=[
'git+https://github.com/maas/python-libmaas.git'
]
)

2
testrequirements.txt Normal file
View File

@ -0,0 +1,2 @@
pytest
tox

457
tests/aicyaml_samples/fullsite.yaml Normal file
View File

@ -0,0 +1,457 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
####################
#
# bootstrap_seed.yaml - Site server design definition for physical layer
#
####################
# version the schema in this file so consumers can rationally parse it
---
apiVersion: 'v1.0'
kind: Region
metadata:
name: sitename
date: 17-FEB-2017
description: Sample site design
author: sh8121@att.com
spec:
# Not sure if we have site wide data that doesn't fall into another 'Kind'
---
apiVersion: 'v1.0'
kind: NetworkLink
metadata:
name: oob
region: sitename
date: 17-FEB-2017
author: sh8121@att.com
description: Describe layer 1 attributes. Primary key is 'name'. These settings will generally be things the switch and server have to agree on
spec:
bonding:
mode: none
mtu: 1500
linkspeed: 100full
trunking:
mode: none
default_network: oob
---
# pxe is a bit of 'magic' indicating the link config used when PXE booting
# a node. All other links indicate network configs applied when the node
# is deployed.
apiVersion: 'v1.0'
kind: NetworkLink
metadata:
name: pxe
region: sitename
date: 17-FEB-2017
author: sh8121@att.com
description: Describe layer 1 attributes. Primary key is 'name'. These settings will generally be things the switch and server have to agree on
spec:
bonding:
mode: none
mtu: 1500
linkspeed: auto
# Is this link supporting multiple layer 2 networks?
# none is a port-based VLAN identified by default_network
# tagged is is using 802.1q VLAN tagging. Untagged packets will default to default_netwokr
trunking:
mode: none
# use name, will translate to VLAN ID
default_network: pxe
---
apiVersion: 'v1.0'
kind: NetworkLink
metadata:
name: gp
region: sitename
date: 17-FEB-2017
author: sh8121@att.com
description: Describe layer 1 attributes. These CIs will generally be things the switch and server have to agree on
# pxe is a bit of 'magic' indicating the link config used when PXE booting
# a node. All other links indicate network configs applied when the node
# is deployed.
spec:
# If this link is a bond of physical links, how is it configured
# 802.3ad
# active-backup
# balance-rr
# Can add support for others down the road
bonding:
mode: 802.3ad
# For LACP (802.3ad) xmit hashing policy: layer2, layer2+3, layer3+4, encap3+4
hash: layer3+4
# 802.3ad specific options
peer_rate: slow
mon_rate: default
up_delay: default
down_delay: default
mtu: 9000
linkspeed: auto
# Is this link supporting multiple layer 2 networks?
trunking:
mode: tagged
default_network: mgmt
---
apiVersion: 'v1.0'
kind: Network
metadata:
name: oob
region: sitename
date: 17-FEB-2017
author: sh8121@att.com
description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
spec:
allocation: static
cidr: 172.16.100.0/24
ranges:
- type: static
start: 172.16.100.15
end: 172.16.100.254
dns:
domain: ilo.sitename.att.com
servers: 172.16.100.10
---
apiVersion: 'v1.0'
kind: Network
metadata:
name: pxe
region: sitename
date: 17-FEB-2017
author: sh8121@att.com
description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
spec:
# Layer 2 VLAN segment id, could support other segmentations. Optional
vlan_id: '99'
# How are addresses assigned?
allocation: dhcp
# MTU for this VLAN interface, if not specified it will be inherited from the link
mtu: 1500
# Network address
cidr: 172.16.0.0/24
# Desribe IP address ranges
ranges:
- type: dhcp
start: 172.16.0.5
end: 172.16.0.254
# DNS settings for this network
dns:
# Domain addresses on this network will be registered under
domain: admin.sitename.att.com
# DNS servers that a server using this network as its default gateway should use
servers: 172.16.0.10
---
apiVersion: 'v1.0'
kind: Network
metadata:
name: mgmt
region: sitename
date: 17-FEB-2017
author: sh8121@att.com
description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
spec:
vlan_id: '100'
# How are addresses assigned?
allocation: static
# Allow MTU to be inherited from link the network rides on
mtu: 1500
# Network address
cidr: 172.16.1.0/24
# Desribe IP address ranges
ranges:
- type: static
start: 172.16.1.15
end: 172.16.1.254
# Static routes to be added for this network
routes:
- subnet: 0.0.0.0/0
# A blank gateway would leave to a static route specifying
# only the interface as a source
gateway: 172.16.1.1
metric: 10
# DNS settings for this network
dns:
# Domain addresses on this network will be registered under
domain: mgmt.sitename.example.com
# DNS servers that a server using this network as its default gateway should use
servers: 172.16.1.9,172.16.1.10
---
apiVersion: 'v1.0'
kind: Network
metadata:
name: private
region: sitename
date: 17-FEB-2017
author: sh8121@att.com
description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
spec:
vlan_id: '101'
allocation: static
mtu: 9000
cidr: 172.16.2.0/24
# Desribe IP address ranges
ranges:
# Type can be reserved (not used for baremetal), static (all explicit
# assignments should fall here), dhcp (will be used by a DHCP server on this network)
- type: static
start: 172.16.2.15
end: 172.16.2.254
dns:
domain: priv.sitename.example.com
servers: 172.16.2.9,172.16.2.10
---
apiVersion: 'v1.0'
kind: Network
metadata:
name: public
region: sitename
date: 17-FEB-2017
author: sh8121@att.com
description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
spec:
vlan_id: '102'
# How are addresses assigned?
allocation: static
# MTU size for the VLAN interface
mtu: 1500
cidr: 172.16.3.0/24
# Desribe IP address ranges
ranges:
- type: static
start: 172.16.3.15
end: 172.16.3.254
routes:
- subnet: 0.0.0.0/0
gateway: 172.16.3.1
metric: 9
dns:
domain: sitename.example.com
servers: 8.8.8.8
---
apiVersion: 'v1.0'
kind: HostProfile
metadata:
name: defaults
region: sitename
date: 17-FEB-2017
author: sh8121@att.com
description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
# No magic to this host_profile, it just provides a way to specify
# sitewide settings. If it is absent from a node's inheritance chain
# then these values will NOT be applied
spec:
# OOB (iLO, iDRAC, etc...) settings. Should prefer open standards such
# as IPMI over vender-specific when possible.
oob:
type: ipmi
# OOB networking should be preconfigured, but we can include a network
# definition for validation or enhancement (DNS registration)
network: oob
account: admin
credential: admin
# Specify storage layout of base OS. Ceph out of scope
storage:
# How storage should be carved up: lvm (logical volumes), flat
# (single partition)
layout: lvm
# Info specific to the boot and root disk/partitions
bootdisk:
# Device will specify an alias defined in hwdefinition.yaml
device: primary_boot
# For LVM, the size of the partition added to VG as a PV
# For flat, the size of the partition formatted as ext4
root_size: 50g
# The /boot partition. If not specified, /boot will in root
boot_size: 2g
# Info for additional partitions. Need to balance between
# flexibility and complexity
partitions:
- name: logs
device: primary_boot
# Partition uuid if needed
part_uuid: 84db9664-f45e-11e6-823d-080027ef795a
size: 10g
# Optional, can carve up unformatted block devices
mountpoint: /var/log
fstype: ext4
mount_options: defaults
# Filesystem UUID or label can be specified. UUID recommended
fs_uuid: cdb74f1c-9e50-4e51-be1d-068b0e9ff69e
fs_label: logs
# Platform (Operating System) settings
platform:
image: ubuntu_16.04_hwe
kernel_params: default
# Additional metadata to apply to a node
metadata:
# Base URL of the introspection service - may go in curtin data
introspection_url: http://172.16.1.10:9090
---
apiVersion: 'v1.0'
kind: HostProfile
metadata:
name: k8-node
region: sitename
date: 17-FEB-2017
author: sh8121@att.com
description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
spec:
# host_profile inheritance allows for deduplication of common CIs
# Inheritance is additive for CIs that are lists of multiple items
# To remove an inherited list member, prefix the primary key value
# with '!'.
host_profile: defaults
# Hardware profile will map hardware specific details to the abstract
# names uses in the host profile as well as specify hardware specific
# configs. A viable model should be to build a host profile without a
# hardware_profile and then for each node inherit the host profile and
# specify a hardware_profile to map that node's hardware to the abstract
# settings of the host_profile
hardware_profile: HPGen9v3
# Network interfaces.
interfaces:
# Keyed on device_name
# pxe is a special marker indicating which device should be used for pxe boot
- device_name: pxe
# The network link attached to this
network_link: pxe
# Slaves will specify aliases from hwdefinition.yaml
slaves:
- prim_nic01
# Which networks will be configured on this interface
networks:
- pxe
- device_name: bond0
network_link: gp
# If multiple slaves are specified, but no bonding config
# is applied to the link, design validation will fail
slaves:
- prim_nic01
- prim_nic02
# If multiple networks are specified, but no trunking
# config is applied to the link, design validation will fail
networks:
- mgmt
- private
metadata:
# Explicit tag assignment
tags:
- 'test'
# MaaS supports key/value pairs. Not sure of the use yet
owner_data:
foo: bar
---
apiVersion: 'v1.0'
kind: HostProfile
metadata:
name: k8-node-public
region: sitename
date: 17-FEB-2017
author: sh8121@att.com
description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
spec:
host_profile: k8-node
interfaces:
- device_name: bond0
networks:
# This is additive, so adds a network to those defined in the host_profile
# inheritance chain
- public
---
apiVersion: 'v1.0'
kind: BaremetalNode
metadata:
name: controller01
region: sitename
date: 17-FEB-2017
author: sh8121@att.com
description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
spec:
host_profile: k8-node-public
# the hostname for a server, could be used in multiple DNS domains to
# represent different interfaces
interfaces:
- device_name: bond0
networks:
# '!' prefix for the value of the primary key indicates a record should be removed
- '!private'
# Addresses assigned to network interfaces
addressing:
# Which network the address applies to. If a network appears in addressing
# that isn't assigned to an interface, design validation will fail
- network: pxe
# The address assigned. Either a explicit IPv4 or IPv6 address
# or dhcp or slaac
address: dhcp
- network: mgmt
address: 172.16.1.20
- network: public
address: 172.16.3.20
metadata:
roles: os_ctl
rack: rack01
---
apiVersion: 'v1.0'
kind: BaremetalNode
metadata:
name: compute01
region: sitename
date: 17-FEB-2017
author: sh8121@att.com
description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
spec:
host_profile: k8-node
addressing:
- network: pxe
address: dhcp
- network: mgmt
address: 172.16.1.21
- network: private
address: 172.16.2.21
---
apiVersion: 'v1.0'
kind: HardwareProfile
metadata:
name: HPGen9v3
region: sitename
date: 17-FEB-2017
author: Scott Hussey
spec:
# Vendor of the server chassis
vendor: HP
# Generation of the chassis model
generation: '8'
# Version of the chassis model within its generation - not version of the hardware definition
hw_version: '3'
# The certified version of the chassis BIOS
bios_version: '2.2.3'
# Mode of the default boot of hardware - bios, uefi
boot_mode: bios
# Protocol of boot of the hardware - pxe, usb, hdd
bootstrap_protocol: pxe
# Which interface to use for network booting within the OOB manager, not OS device
pxe_interface: 0
# Map hardware addresses to aliases/roles to allow a mix of hardware configs
# in a site to result in a consistent configuration
device_aliases:
pci:
- address: pci@0000:00:03.0
alias: prim_nic01
# type could identify expected hardware - used for hardware manifest validation
type: '82540EM Gigabit Ethernet Controller'
- address: pci@0000:00:04.0
alias: prim_nic02
type: '82540EM Gigabit Ethernet Controller'
scsi:
- address: scsi@2:0.0.0
alias: primary_boot
type: 'VBOX HARDDISK'

29
tests/aicyaml_samples/invalid.yaml Normal file
View File

@ -0,0 +1,29 @@
---
apiVersion:
: Network
metadata:
name: public
region: sitename
date: 17-FEB-2017
name: Sample network definition
author: sh8121@att.com
description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
spec:
vlan_id: '102'
# How are addresses assigned?
allocation: static
# MTU size for the VLAN interface
mtu: 1500
cidr: 172.16.3.0/24
# Desribe IP address ranges
ranges:
- type: static
start: 172.16.3.15
end: 172.16.3.254
routes:
- subnet: 0.0.0.0/0
gateway: 172.16.3.1
metric: 9
dns:
domain: sitename.example.com
servers: 8.8.8.8

77
tests/aicyaml_samples/multidoc.yaml Normal file
View File

@ -0,0 +1,77 @@
---
apiVersion: 'v1.0'
kind: NetworkLink
metadata:
name: oob
region: sitename
date: 17-FEB-2017
name: Sample network link
author: sh8121@att.com
description: Describe layer 1 attributes. Primary key is 'name'. These settings will generally be things the switch and server have to agree on
spec:
bonding:
mode: none
mtu: 1500
linkspeed: 100full
trunking:
mode: none
default_network: oob
---
# pxe is a bit of 'magic' indicating the link config used when PXE booting
# a node. All other links indicate network configs applied when the node
# is deployed.
apiVersion: 'v1.0'
kind: NetworkLink
metadata:
name: pxe
region: sitename
date: 17-FEB-2017
name: Sample network link
author: sh8121@att.com
description: Describe layer 1 attributes. Primary key is 'name'. These settings will generally be things the switch and server have to agree on
spec:
bonding:
mode: none
mtu: 1500
linkspeed: auto
# Is this link supporting multiple layer 2 networks?
# none is a port-based VLAN identified by default_network
# tagged is is using 802.1q VLAN tagging. Untagged packets will default to default_netwokr
trunking:
mode: none
# use name, will translate to VLAN ID
default_network: pxe
---
apiVersion: 'v1.0'
kind: NetworkLink
metadata:
name: gp
region: sitename
date: 17-FEB-2017
name: Sample network link
author: sh8121@att.com
description: Describe layer 1 attributes. These CIs will generally be things the switch and server have to agree on
# pxe is a bit of 'magic' indicating the link config used when PXE booting
# a node. All other links indicate network configs applied when the node
# is deployed.
spec:
# If this link is a bond of physical links, how is it configured
# 802.3ad
# active-backup
# balance-rr
# Can add support for others down the road
bonding:
mode: 802.3ad
# For LACP (802.3ad) xmit hashing policy: layer2, layer2+3, layer3+4, encap3+4
hash: layer3+4
# 802.3ad specific options
peer_rate: slow
mon_rate: default
up_delay: default
down_delay: default
mtu: 9000
linkspeed: auto
# Is this link supporting multiple layer 2 networks?
trunking:
mode: tagged
default_network: mgmt

39
tests/aicyaml_samples/singledoc.yaml Normal file
View File

@ -0,0 +1,39 @@
---
apiVersion: 'v1.0'
kind: HardwareProfile
metadata:
name: HPGen8v3
region: sitename
date: 17-FEB-2017
name: Sample hardware definition
author: Scott Hussey
spec:
# Vendor of the server chassis
vendor: HP
# Generation of the chassis model
generation: '8'
# Version of the chassis model within its generation - not version of the hardware definition
hw_version: '3'
# The certified version of the chassis BIOS
bios_version: '2.2.3'
# Mode of the default boot of hardware - bios, uefi
boot_mode: bios
# Protocol of boot of the hardware - pxe, usb, hdd
bootstrap_protocol: pxe
# Which interface to use for network booting within the OOB manager, not OS device
pxe_interface: 0
# Map hardware addresses to aliases/roles to allow a mix of hardware configs
# in a site to result in a consistent configuration
device_aliases:
pci:
- address: pci@0000:00:03.0
alias: prim_nic01
# type could identify expected hardware - used for hardware manifest validation
type: '82540EM Gigabit Ethernet Controller'
- address: pci@0000:00:04.0
alias: prim_nic02
type: '82540EM Gigabit Ethernet Controller'
scsi:
- address: scsi@2:0.0.0
alias: primary_boot
type: 'VBOX HARDDISK'

60
tests/aicyaml_samples/unknown_kind.yaml Normal file
View File

@ -0,0 +1,60 @@
---
apiVersion: 'v1.0'
kind: FooBar
metadata:
name: default
region: sitename
date: 17-FEB-2017
name: Sample network definition
author: sh8121@att.com
description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
# No magic to this host_profile, it just provides a way to specify
# sitewide settings. If it is absent from a node's inheritance chain
# then these values will NOT be applied
spec:
# OOB (iLO, iDRAC, etc...) settings. Should prefer open standards such
# as IPMI over vender-specific when possible.
oob:
type: ipmi
# OOB networking should be preconfigured, but we can include a network
# definition for validation or enhancement (DNS registration)
network: oob
account: admin
credential: admin
# Specify storage layout of base OS. Ceph out of scope
storage:
# How storage should be carved up: lvm (logical volumes), flat
# (single partition)
layout: lvm
# Info specific to the boot and root disk/partitions
bootdisk:
# Device will specify an alias defined in hwdefinition.yaml
device: primary_boot
# For LVM, the size of the partition added to VG as a PV
# For flat, the size of the partition formatted as ext4
root_size: 50g
# The /boot partition. If not specified, /boot will in root
boot_size: 2g
# Info for additional partitions. Need to balance between
# flexibility and complexity
partitions:
- name: logs
device: primary_boot
# Partition uuid if needed
part_uuid: 84db9664-f45e-11e6-823d-080027ef795a
size: 10g
# Optional, can carve up unformatted block devices
mountpoint: /var/log
fstype: ext4
mount_options: defaults
# Filesystem UUID or label can be specified. UUID recommended
fs_uuid: cdb74f1c-9e50-4e51-be1d-068b0e9ff69e
fs_label: logs
# Platform (Operating System) settings
platform:
image: ubuntu_16.04_hwe
kernel_params: default
# Additional metadata to apply to a node
metadata:
# Base URL of the introspection service - may go in curtin data
introspection_url: http://172.16.1.10:9090

70
tests/test_design_inheritance.py Normal file
View File

@ -0,0 +1,70 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from helm_drydock.ingester import Ingester
from helm_drydock.statemgmt import DesignState
from helm_drydock.orchestrator.designdata import DesignStateClient
from copy import deepcopy
import pytest
import shutil
import os
import helm_drydock.ingester.plugins.aicyaml
import yaml
class TestClass(object):
def setup_method(self, method):
print("Running test {0}".format(method.__name__))
def test_design_inheritance(self, loaded_design):
client = DesignStateClient()
design_data = client.load_design_data(design_state=loaded_design)
design_data = client.compute_model_inheritance(design_data)
print(yaml.dump(design_data, default_flow_style=False))
node = design_data.get_baremetal_node("controller01")
assert node.hardware_profile == 'HPGen9v3'
@pytest.fixture(scope='module')
def loaded_design(self, input_files):
input_file = input_files.join("fullsite.yaml")
module_design_state = DesignState()
ingester = Ingester()
ingester.enable_plugins([helm_drydock.ingester.plugins.aicyaml.AicYamlIngester])
ingester.ingest_data(plugin_name='aic_yaml', design_state=module_design_state, filenames=[str(input_file)])
return module_design_state
@pytest.fixture(scope='module')
def input_files(self, tmpdir_factory, request):
tmpdir = tmpdir_factory.mktemp('data')
samples_dir = os.path.dirname(str(request.fspath)) + "/aicyaml_samples"
samples = os.listdir(samples_dir)
for f in samples:
src_file = samples_dir + "/" + f
dst_file = str(tmpdir) + "/" + f
shutil.copyfile(src_file, dst_file)
return tmpdir

53
tests/test_ingester.py Normal file
View File

@ -0,0 +1,53 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from helm_drydock.ingester import Ingester
from helm_drydock.statemgmt import DesignState
import pytest
import shutil
import os
import helm_drydock.ingester.plugins.aicyaml
class TestClass(object):
def setup_method(self, method):
print("Running test {0}".format(method.__name__))
def test_ingest_full_site(self, design_state,input_files):
input_file = input_files.join("fullsite.yaml")
ingester = Ingester()
ingester.enable_plugins([helm_drydock.ingester.plugins.aicyaml.AicYamlIngester])
ingester.ingest_data(plugin_name='aic_yaml', design_state=design_state, filenames=[str(input_file)])
assert len(design_state.host_profiles) == 3
@pytest.fixture(scope='module')
def input_files(self, tmpdir_factory, request):
tmpdir = tmpdir_factory.mktemp('data')
samples_dir = os.path.dirname(str(request.fspath)) + "/aicyaml_samples"
samples = os.listdir(samples_dir)
for f in samples:
src_file = samples_dir + "/" + f
dst_file = str(tmpdir) + "/" + f
shutil.copyfile(src_file, dst_file)
return tmpdir
@pytest.fixture(scope='session')
def design_state(self):
design_state = DesignState()
return design_state

54
tests/test_ingester_aicyaml.py Normal file
View File

@ -0,0 +1,54 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from helm_drydock.ingester.plugins.aicyaml import AicYamlIngester
import pytest
import shutil
import os
class TestClass(object):
def setup_method(self, method):
print("Running test {0}".format(method.__name__))
def test_ingest_singledoc(self, input_files):
input_file = input_files.join("singledoc.yaml")
ingester = AicYamlIngester()
models = ingester.ingest_data(filenames=[str(input_file)])
assert len(models) == 1
def test_ingest_multidoc(self, input_files):
input_file = input_files.join("multidoc.yaml")
ingester = AicYamlIngester()
models = ingester.ingest_data(filenames=[str(input_file)])
assert len(models) == 3
@pytest.fixture(scope='module')
def input_files(self, tmpdir_factory, request):
tmpdir = tmpdir_factory.mktemp('data')
samples_dir = os.path.dirname(str(request.fspath)) + "/aicyaml_samples"
samples = os.listdir(samples_dir)
for f in samples:
src_file = samples_dir + "/" + f
dst_file = str(tmpdir) + "/" + f
shutil.copyfile(src_file, dst_file)
return tmpdir

69
tests/test_models.py Normal file
View File

@ -0,0 +1,69 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import pytest
import yaml
from helm_drydock.model import HardwareProfile
class TestClass(object):
def setup_method(self, method):
print("Running test {0}".format(method.__name__))
def test_hardwareprofile(self):
yaml_snippet = ("---\n"
"apiVersion: 'v1.0'\n"
"kind: HardwareProfile\n"
"metadata:\n"
" name: HPGen8v3\n"
" region: sitename\n"
" date: 17-FEB-2017\n"
" name: Sample hardware definition\n"
" author: Scott Hussey\n"
"spec:\n"
" # Vendor of the server chassis\n"
" vendor: HP\n"
" # Generation of the chassis model\n"
" generation: '8'\n"
" # Version of the chassis model within its generation - not version of the hardware definition\n"
" hw_version: '3'\n"
" # The certified version of the chassis BIOS\n"
" bios_version: '2.2.3'\n"
" # Mode of the default boot of hardware - bios, uefi\n"
" boot_mode: bios\n"
" # Protocol of boot of the hardware - pxe, usb, hdd\n"
" bootstrap_protocol: pxe\n"
" # Which interface to use for network booting within the OOB manager, not OS device\n"
" pxe_interface: 0\n"
" # Map hardware addresses to aliases/roles to allow a mix of hardware configs\n"
" # in a site to result in a consistent configuration\n"
" device_aliases:\n"
" pci:\n"
" - address: pci@0000:00:03.0\n"
" alias: prim_nic01\n"
" # type could identify expected hardware - used for hardware manifest validation\n"
" type: '82540EM Gigabit Ethernet Controller'\n"
" - address: pci@0000:00:04.0\n"
" alias: prim_nic02\n"
" type: '82540EM Gigabit Ethernet Controller'\n"
" scsi:\n"
" - address: scsi@2:0.0.0\n"
" alias: primary_boot\n"
" type: 'VBOX HARDDISK'\n")
hw_profile = yaml.load(yaml_snippet)
hw_profile_model = HardwareProfile(**hw_profile)
assert hasattr(hw_profile_model, 'bootstrap_protocol')

15
tox.ini Normal file
View File

@ -0,0 +1,15 @@
[tox]
envlist = py35
[testenv]
deps=
-rrequirements.txt
-rtestrequirements.txt
setenv=
PYTHONWARNING=all
commands=
py.test \
{posargs}
[flake8]
ignore=E302,H306