From 238eba89665b527fe763cbee28e7c7dac20af804 Mon Sep 17 00:00:00 2001 From: Scott Hussey Date: Tue, 28 Feb 2017 08:52:52 -0600 Subject: [PATCH] Add methods to the model for computing inheritance Skeleton design state client for the orchestrator Tests for design inheritance --- README.md | 10 +- examples/bootstrap_hwdefinition.yaml | 2 +- examples/bootstrap_seed.yaml | 28 +- helm_drydock/model/__init__.py | 466 +++++++++++++++++++++--- helm_drydock/orchestrator/__init__.py | 13 + helm_drydock/orchestrator/designdata.py | 24 +- helm_drydock/statemgmt/__init__.py | 64 ++-- setup.py | 3 +- tests/aicyaml_samples/fullsite.yaml | 60 ++- tests/aicyaml_samples/multidoc.yaml | 6 +- tests/aicyaml_samples/singledoc.yaml | 2 +- tests/aicyaml_samples/unknown_kind.yaml | 2 +- tests/test_design_inheritance.py | 70 ++++ tests/test_models.py | 2 +- 14 files changed, 604 insertions(+), 148 deletions(-) create mode 100644 helm_drydock/orchestrator/__init__.py create mode 100644 tests/test_design_inheritance.py diff --git a/README.md b/README.md index 67e88f85..321649bb 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ A python REST orchestrator to translate a YAML host topology to a provisioned se ### Design Consumer ### -aka smelter +aka ingester Pluggable service to ingest a inventory/design specification, convert it to a standard internal representaion, and persist it to the Design State API. Initial implementation @@ -13,21 +13,21 @@ is the consumer of AIC YAML schema. ### Design State API ### -aka tarot +aka statemgmt API for querying and updating the current design specification and persisted orchestration status. CRUD support of CIs that are not bootstrap-related, but can be used by other automation. ### Control API ### -aka cockpit +aka control User-approachable API for initiating orchestration actions or accessing other internal APIs ### Infrastructure Orchestrator ### -aka alchemist +aka orchestrator Handle validation of complete design, ordering and managing downstream API calls for hardware provisioning/bootstrapping @@ -44,7 +44,7 @@ Pluggable provisioner for network provisioning. Initial implementation is Noop. ### Introspection API ### -aka jabberwocky +aka introspection API for bootstrapping nodes to load self data. Possibly pluggable as this is basically an authenticated bridge to the Design State API \ No newline at end of file diff --git a/examples/bootstrap_hwdefinition.yaml b/examples/bootstrap_hwdefinition.yaml index aaff67be..8476ff6d 100644 --- a/examples/bootstrap_hwdefinition.yaml +++ b/examples/bootstrap_hwdefinition.yaml @@ -18,7 +18,7 @@ ############################################################################# # version the schema in this file so consumers can rationally parse it --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: HardwareProfile metadata: name: HPGen8v3 diff --git a/examples/bootstrap_seed.yaml b/examples/bootstrap_seed.yaml index f5c20c14..e6d7d7b0 100644 --- a/examples/bootstrap_seed.yaml +++ b/examples/bootstrap_seed.yaml @@ -18,7 +18,7 @@ #################### # version the schema in this file so consumers can rationally parse it --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: Region metadata: name: sitename @@ -28,7 +28,7 @@ metadata: spec: # Not sure if we have site wide data that doesn't fall into another 'Kind' --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: NetworkLink metadata: name: oob @@ -49,7 +49,7 @@ spec: # pxe is a bit of 'magic' indicating the link config used when PXE booting # a node. All other links indicate network configs applied when the node # is deployed. -apiVersion: '1.0' +apiVersion: 'v1.0' kind: NetworkLink metadata: name: pxe @@ -71,7 +71,7 @@ spec: # use name, will translate to VLAN ID default_network: pxe --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: NetworkLink metadata: name: gp @@ -105,7 +105,7 @@ spec: mode: tagged default_network: mgmt --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: Network metadata: name: oob @@ -125,7 +125,7 @@ spec: domain: ilo.sitename.att.com servers: 172.16.100.10 --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: Network metadata: name: pxe @@ -155,7 +155,7 @@ spec: # DNS servers that a server using this network as its default gateway should use servers: 172.16.0.10 --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: Network metadata: name: mgmt @@ -191,7 +191,7 @@ spec: # DNS servers that a server using this network as its default gateway should use servers: 172.16.1.9,172.16.1.10 --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: Network metadata: name: private @@ -216,7 +216,7 @@ spec: domain: priv.sitename.example.com servers: 172.16.2.9,172.16.2.10 --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: Network metadata: name: public @@ -245,7 +245,7 @@ spec: domain: sitename.example.com servers: 8.8.8.8 --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: HostProfile metadata: name: default @@ -305,7 +305,7 @@ spec: # Base URL of the introspection service - may go in curtin data introspection_url: http://172.16.1.10:9090 --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: HostProfile metadata: name: k8-node @@ -360,7 +360,7 @@ spec: owner_data: foo: bar --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: HostProfile metadata: name: k8-node-public @@ -378,7 +378,7 @@ spec: # inheritance chain - name: public --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: BaremetalNode metadata: name: controller01 @@ -413,7 +413,7 @@ spec: - os_ctl rack: rack01 --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: BaremetalNode metadata: name: compute01 diff --git a/helm_drydock/model/__init__.py b/helm_drydock/model/__init__.py index a6d32570..e5cec4bf 100644 --- a/helm_drydock/model/__init__.py +++ b/helm_drydock/model/__init__.py @@ -14,6 +14,7 @@ # # Models for helm_drydock # +from copy import deepcopy class HardwareProfile(object): @@ -21,7 +22,7 @@ class HardwareProfile(object): def __init__(self, **kwargs): self.api_version = kwargs.get('apiVersion', '') - if self.api_version == "1.0": + if self.api_version == "v1.0": metadata = kwargs.get('metadata', {}) spec = kwargs.get('spec', {}) @@ -29,13 +30,14 @@ class HardwareProfile(object): # valid for now self.name = metadata.get('name', '') self.site = metadata.get('region', '') - self.vendor = spec.get('vendor', '') - self.generation = spec.get('generation', '') - self.hw_version = spec.get('hw_version', '') - self.bios_version = spec.get('bios_version', '') - self.boot_mode = spec.get('boot_mode', '') - self.bootstrap_protocol = spec.get('bootstrap_protocol', '') - self.pxe_interface = spec.get('pxe_interface', '') + + self.vendor = spec.get('vendor', None) + self.generation = spec.get('generation', None) + self.hw_version = spec.get('hw_version', None) + self.bios_version = spec.get('bios_version', None) + self.boot_mode = spec.get('boot_mode', None) + self.bootstrap_protocol = spec.get('bootstrap_protocol', None) + self.pxe_interface = spec.get('pxe_interface', None) self.devices = [] device_aliases = spec.get('device_aliases', {}) @@ -57,29 +59,34 @@ class HardwareProfile(object): return + def resolve_alias(self, alias_type, alias): + for d in self.devices: + if d.alias == alias and d.bus_type == alias_type: + return deepcopy(d) + + return None + class HardwareDeviceAlias(object): def __init__(self, api_version, **kwargs): self.api_version = api_version - if self.api_version == "1.0": - self.bus_type = kwargs.get('bus_type', '') - self.address = kwargs.get('address', '') - self.alias = kwargs.get('alias', '') - self.type = kwargs.get('type', '') + if self.api_version == "v1.0": + self.bus_type = kwargs.get('bus_type', None) + self.address = kwargs.get('address', None) + self.alias = kwargs.get('alias', None) + self.type = kwargs.get('type', None) else: raise ValueError('Unknown API version of object') - return - class Site(object): def __init__(self, **kwargs): self.api_version = kwargs.get('apiVersion', '') - if self.api_version == "1.0": + if self.api_version == "v1.0": metadata = kwargs.get('metadata', {}) # Need to add validation logic, we'll assume the input is @@ -95,13 +102,47 @@ class Site(object): else: raise ValueError('Unknown API version of object') + def get_network(self, network_name): + for n in self.networks: + if n.name == network_name: + return n + + return None + + def get_network_link(self, link_name): + for l in self.network_links: + if l.name == link_name: + return l + + return None + + def get_host_profile(self, profile_name): + for p in self.host_profiles: + if p.name == profile_name: + return p + + return None + + def get_hardware_profile(self, profile_name): + for p in self.hardware_profiles: + if p.name == profile_name: + return p + + return None + + def get_baremetal_node(self, node_name): + for n in self.baremetal_nodes: + if n.name == node_name: + return n + + return None class NetworkLink(object): def __init__(self, **kwargs): self.api_version = kwargs.get('apiVersion', '') - if self.api_version == "1.0": + if self.api_version == "v1.0": metadata = kwargs.get('metadata', {}) spec = kwargs.get('spec', {}) @@ -135,20 +176,21 @@ class Network(object): def __init__(self, **kwargs): self.api_version = kwargs.get('apiVersion', '') - if self.api_version == "1.0": + if self.api_version == "v1.0": metadata = kwargs.get('metadata', {}) spec = kwargs.get('spec', {}) self.name = metadata.get('name', '') self.site = metadata.get('region', '') - self.cidr = spec.get('cidr', '') + + self.cidr = spec.get('cidr', None) self.allocation_strategy = spec.get('allocation', 'static') self.vlan_id = spec.get('vlan_id', 1) self.mtu = spec.get('mtu', 0) dns = spec.get('dns', {}) self.dns_domain = dns.get('domain', 'local') - self.dns_servers = dns.get('servers', '') + self.dns_servers = dns.get('servers', None) ranges = spec.get('ranges', []) self.ranges = [] @@ -170,10 +212,10 @@ class NetworkAddressRange(object): def __init__(self, api_version, **kwargs): self.api_version = api_version - if self.api_version == "1.0": - self.type = kwargs.get('type', 'static') - self.start = kwargs.get('start', '') - self.end = kwargs.get('end', '') + if self.api_version == "v1.0": + self.type = kwargs.get('type', None) + self.start = kwargs.get('start', None) + self.end = kwargs.get('end', None) else: raise ValueError('Unknown API version of object') @@ -183,9 +225,9 @@ class NetworkRoute(object): def __init__(self, api_version, **kwargs): self.api_version = api_version - if self.api_version == "1.0": - self.type = kwargs.get('subnet', '') - self.start = kwargs.get('gateway', '') + if self.api_version == "v1.0": + self.type = kwargs.get('subnet', None) + self.start = kwargs.get('gateway', None) self.end = kwargs.get('metric', 100) else: raise ValueError('Unknown API version of object') @@ -196,26 +238,29 @@ class HostProfile(object): def __init__(self, **kwargs): self.api_version = kwargs.get('apiVersion', '') - if self.api_version == "1.0": + if self.api_version == "v1.0": metadata = kwargs.get('metadata', {}) spec = kwargs.get('spec', {}) self.name = metadata.get('name', '') self.site = metadata.get('region', '') + self.parent_profile = spec.get('host_profile', None) + self.hardware_profile = spec.get('hardware_profile', None) + oob = spec.get('oob', {}) - self.oob_type = oob.get('type', 'ipmi') - self.oob_network = oob.get('network', 'oob') - self.oob_account = oob.get('account', '') - self.oob_credential = oob.get('credential', '') + self.oob_type = oob.get('type', None) + self.oob_network = oob.get('network', None) + self.oob_account = oob.get('account', None) + self.oob_credential = oob.get('credential', None) storage = spec.get('storage', {}) self.storage_layout = storage.get('layout', 'lvm') bootdisk = storage.get('bootdisk', {}) - self.bootdisk_device = bootdisk.get('device', '') - self.bootdisk_root_size = bootdisk.get('root_size', '') - self.bootdisk_boot_size = bootdisk.get('boot_size', '') + self.bootdisk_device = bootdisk.get('device', None) + self.bootdisk_root_size = bootdisk.get('root_size', None) + self.bootdisk_boot_size = bootdisk.get('boot_size', None) partitions = storage.get('partitions', []) self.partitions = [] @@ -243,23 +288,61 @@ class HostProfile(object): for k, v in owner_data.items(): self.owner_data[k] = v - self.rack = metadata.get('rack', '') + self.rack = metadata.get('rack', None) else: raise ValueError('Unknown API version of object') - def inherit_parent(self, site): + def apply_inheritance(self, site): + # We return a deep copy of the profile so as not to corrupt + # the original model + self_copy = deepcopy(self) + + if self.parent_profile is None: + return self_copy + + parent = site.get_host_profile(self.parent_profile) + + if parent is None: + return self_copy + + parent = parent.apply_inheritance(site) + + # First compute inheritance for simple fields + inheritable_field_list = [ + "hardware_profile", "oob_type", "oob_network", + "oob_credential", "oob_account", "storage_layout", + "bootdisk_device", "bootdisk_root_size", "bootdisk_boot_size", + "rack"] + + for f in inheritable_field_list: + setattr(self_copy, f, + Utils.apply_field_inheritance(getattr(self, f, None), + getattr(parent, f, None))) + + # Now compute inheritance for complex types + self_copy.tags = Utils.merge_lists(self.tags, parent.tags) + + self_copy.owner_data = Utils.merge_dicts( + self.owner_data, parent.owner_data) + + self_copy.interfaces = HostInterface.merge_lists( + self.interfaces, parent.interfaces) + + self_copy.partitions = HostPartition.merge_lists( + self.partitions, parent.partitions) + + return self_copy - def apply_hardware_profile(self, site): class HostInterface(object): def __init__(self, api_version, **kwargs): self.api_version = api_version - if self.api_version == "1.0": - self.device_name = kwargs.get('device_name', '') - self.network_link = kwargs.get('device_link', '') + if self.api_version == "v1.0": + self.device_name = kwargs.get('device_name', None) + self.network_link = kwargs.get('device_link', None) self.hardware_slaves = [] slaves = kwargs.get('slaves', []) @@ -276,25 +359,155 @@ class HostInterface(object): else: raise ValueError('Unknown API version of object') + # The device attribute may be hardware alias that translates to a + # physical device address. If the device attribute does not match an + # alias, we assume it directly identifies a OS device name. When the + # apply_hardware_profile method is called on the parent Node of this + # device, the selector will be decided and applied + + def add_selector(self, sel_type, selector): + if getattr(self, 'selectors', None) is None: + self.selectors = [] + + new_selector = {} + new_selector['selector_type'] = sel_type + new_selector['selector'] = selector + + self.selectors.append(new_selector) + + """ + Merge two lists of HostInterface models with child_list taking + priority when conflicts. If a member of child_list has a device_name + beginning with '!' it indicates that HostInterface should be + removed from the merged list + """ + + @staticmethod + def merge_lists(child_list, parent_list): + if len(child_list) == 0: + return deepcopy(parent_list) + + effective_list = [] + if len(parent_list) == 0: + for i in child_list: + if i.device_name.startswith('!'): + continue + else: + effective_list.append(deepcopy(i)) + + parent_interfaces = [] + for i in parent_list: + parent_name = i.device_name + parent_interfaces.append(parent_name) + for j in child_list: + if j.device_name == ("!" + parent_name): + break + elif j.device_name == parent_name: + m = HostInterface(j.api_version) + m.device_name = j.device_name + m.network_link = \ + Utils.apply_field_inheritance(j.network_link, + i.network_link) + s = filter(lambda x: ("!" + x) not in j.hardware_slaves, + i.hardware_slaves) + s = list(s) + + s.extend(filter(lambda x: not x.startswith("!"), + j.hardware_slaves)) + m.hardware_slaves = s + + n = filter(lambda x: ("!" + x) not in j.networks, + i.networks) + n = list(n) + + n.extend(filter(lambda x: not x.startswith("!"), + j.networks)) + m.networks = n + + effective_list.append(m) + + for j in child_list: + if j.device_name not in parent_list: + effective_list.append(deepcopy(j)) + + return effective_list + class HostPartition(object): def __init__(self, api_version, **kwargs): self.api_version = api_version - if self.api_version == "1.0": - self.name = kwargs.get('name', '') - self.device = kwargs.get('device', '') - self.part_uuid = kwargs.get('part_uuid', '') - self.size = kwargs.get('size', '') - self.mountpoint = kwargs.get('mountpoint', '') + if self.api_version == "v1.0": + self.name = kwargs.get('name', None) + self.device = kwargs.get('device', None) + self.part_uuid = kwargs.get('part_uuid', None) + self.size = kwargs.get('size', None) + self.mountpoint = kwargs.get('mountpoint', None) self.fstype = kwargs.get('fstype', 'ext4') self.mount_options = kwargs.get('mount_options', 'defaults') - self.fs_uuid = kwargs.get('fs_uuid', '') - self.fs_label = kwargs.get('fs_label', '') + self.fs_uuid = kwargs.get('fs_uuid', None) + self.fs_label = kwargs.get('fs_label', None) else: raise ValueError('Unknown API version of object') + # The device attribute may be hardware alias that translates to a + # physical device address. If the device attribute does not match an + # alias, we assume it directly identifies a OS device name. When the + # apply_hardware_profile method is called on the parent Node of this + # device, the selector will be decided and applied + + def set_selector(self, sel_type, selector): + self.selector_type = sel_type + self.selector = selector + + """ + Merge two lists of HostPartition models with child_list taking + priority when conflicts. If a member of child_list has a name + beginning with '!' it indicates that HostPartition should be + removed from the merged list + """ + + @staticmethod + def merge_lists(child_list, parent_list): + if len(child_list) == 0: + return deepcopy(parent_list) + + effective_list = [] + if len(parent_list) == 0: + for i in child_list: + if i.name.startswith('!'): + continue + else: + effective_list.append(deepcopy(i)) + + inherit_field_list = ["device", "part_uuid", "size", + "mountpoint", "fstype", "mount_options", + "fs_uuid", "fs_label"] + + parent_partitions = [] + for i in parent_list: + parent_name = i.name + parent_partitions.append(parent_name) + for j in child_list: + if j.name == ("!" + parent_name): + break + elif j.name == parent_name: + p = HostPartition(j.api_version) + p.name = j.name + + for f in inherit_field_list: + setattr(p, Utils.apply_field_inheritance(getattr(j, f), + getattr(i, f)) + ) + + effective_list.append(p) + + for j in child_list: + if j.name not in parent_list: + effective_list.append(deepcopy(j)) + + return effective_list # A BaremetalNode is really nothing more than a physical # instantiation of a HostProfile, so they both represent @@ -302,4 +515,157 @@ class HostPartition(object): class BaremetalNode(HostProfile): def __init__(self, **kwargs): - super(BaremetalNode, self).__init__() + super(BaremetalNode, self).__init__(**kwargs) + + def apply_host_profile(self, site): + return self.apply_inheritance(site) + + # Translate device alises to physical selectors and copy + # other hardware attributes into this object + def apply_hardware_profile(self, site): + self_copy = deepcopy(self) + + if self.hardware_profile is None: + raise ValueError("Hardware profile not set") + + hw_profile = site.get_hardware_profile(self.hardware_profile) + + for i in self_copy.interfaces: + for s in i.hardware_slaves: + selector = hw_profile.resolve_alias("pci", s) + if selector is None: + i.add_selector("name", s) + else: + i.add_selector("address", selector) + + for p in self_copy.partitions: + selector = hw_profile.resolve_alias("scsi", p.device) + if selector is None: + p.set_selector("name", p.device) + else: + p.set_selector("address", selector) + + + hardware = {"vendor": getattr(hw_profile, 'vendor', None), + "generation": getattr(hw_profile, 'generation', None), + "hw_version": getattr(hw_profile, 'hw_version', None), + "bios_version": getattr(hw_profile, 'bios_version', None), + "boot_mode": getattr(hw_profile, 'boot_mode', None), + "bootstrap_protocol": getattr(hw_profile, + 'bootstrap_protocol', + None), + "pxe_interface": getattr(hw_profile, 'pxe_interface', None) + } + + self_copy.hardware = hardware + + return self_copy + + +# Utility class for calculating inheritance + +class Utils(object): + + """ + apply_field_inheritance - apply inheritance rules to a single field value + + param child_field - value of the child field, or the field receiving + the inheritance + param parent_field - value of the parent field, or the field supplying + the inheritance + + return the correct value for child_field based on the inheritance algorithm + + Inheritance algorithm + + 1. If child_field is not None, '!' for string vals or -1 for numeric + vals retain the value + of child_field + + 2. If child_field is '!' return None to unset the field value + + 3. If child_field is -1 return None to unset the field value + + 4. If child_field is None return parent_field + """ + + @staticmethod + def apply_field_inheritance(child_field, parent_field): + + if child_field is not None: + if child_field != '!' and child_field != -1: + return child_field + else: + return None + else: + return parent_field + + """ + merge_lists - apply inheritance rules to a list of simple values + + param child_list - list of values from the child + param parent_list - list of values from the parent + + return a merged list with child values taking prority + + 1. All members in the child list not starting with '!' + + 2. If a member in the parent list has a corresponding member in the + chid list prefixed with '!' it is removed + + 3. All remaining members of the parent list + """ + @staticmethod + def merge_lists(child_list, parent_list): + + if type(child_list) is not list or type(parent_list) is not list: + raise ValueError("One parameter is not a list") + + effective_list = [] + + # Probably should handle non-string values + effective_list.extend( + filter(lambda x: not x.startswith("!"), child_list)) + + effective_list.extend( + filter(lambda x: ("!" + x) not in child_list, + filter(lambda x: x not in effective_list, parent_list))) + + return effective_list + + """ + merge_dicts - apply inheritance rules to a dict + + param child_dict - dict of k:v from child + param parent_dict - dict of k:v from the parent + + return a merged dict with child values taking prority + + 1. All members in the child dict with a key not starting with '!' + + 2. If a member in the parent dict has a corresponding member in the + chid dict where the key is prefixed with '!' it is removed + + 3. All remaining members of the parent dict + """ + @staticmethod + def merge_dicts(child_dict, parent_dict): + + if type(child_dict) is not dict or type(parent_dict) is not dict: + raise ValueError("One parameter is not a dict") + + effective_dict = {} + + # Probably should handle non-string keys + use_keys = filter(lambda x: ("!" + x) not in child_dict.keys(), + parent_dict) + + for k in use_keys: + effective_dict[k] = deepcopy(parent_dict[k]) + + use_keys = filter(lambda x: not x.startswith("!"), child_dict) + + for k in use_keys: + effective_dict[k] = deepcopy(child_dict[k]) + + return effective_dict diff --git a/helm_drydock/orchestrator/__init__.py b/helm_drydock/orchestrator/__init__.py new file mode 100644 index 00000000..2a385a45 --- /dev/null +++ b/helm_drydock/orchestrator/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. \ No newline at end of file diff --git a/helm_drydock/orchestrator/designdata.py b/helm_drydock/orchestrator/designdata.py index fe7a268a..2ec2cf65 100644 --- a/helm_drydock/orchestrator/designdata.py +++ b/helm_drydock/orchestrator/designdata.py @@ -14,6 +14,8 @@ import logging +from copy import deepcopy + class DesignStateClient(object): def __init__(self): @@ -67,7 +69,6 @@ class DesignStateClient(object): if n.site == site_name: site.baremetal_nodes.append(n) - return site """ @@ -77,4 +78,23 @@ class DesignStateClient(object): return a Site model reflecting the effective design for the site """ def compute_model_inheritance(self, site_root): - \ No newline at end of file + + # For now the only thing that really incorporates inheritance is + # host profiles and baremetal nodes. So we'll just resolve it for + # the baremetal nodes which recursively resolves it for host profiles + # assigned to those nodes + + site_copy = deepcopy(site_root) + + effective_nodes = [] + + for n in site_copy.baremetal_nodes: + resolved = n.apply_host_profile(site_copy) + resolved = resolved.apply_hardware_profile(site_copy) + effective_nodes.append(resolved) + + site_copy.baremetal_nodes = effective_nodes + + return site_copy + + diff --git a/helm_drydock/statemgmt/__init__.py b/helm_drydock/statemgmt/__init__.py index 110f974d..2fb13a51 100644 --- a/helm_drydock/statemgmt/__init__.py +++ b/helm_drydock/statemgmt/__init__.py @@ -16,20 +16,20 @@ import helm_drydock.model as model class DesignState(object): - def __init__(self): - self.sites = [] - self.networks = [] - self.network_links = [] - self.host_profiles = [] - self.hardware_profiles = [] - self.baremetal_nodes = [] - return + def __init__(self): + self.sites = [] + self.networks = [] + self.network_links = [] + self.host_profiles = [] + self.hardware_profiles = [] + self.baremetal_nodes = [] + return - def add_site(self, new_site): - if new_site is None or not isinstance(new_site, model.Site): - raise Exception("Invalid Site model") + def add_site(self, new_site): + if new_site is None or not isinstance(new_site, model.Site): + raise Exception("Invalid Site model") - self.sites.append(new_site) + self.sites.append(new_site) def get_sites(self): return self.sites @@ -41,11 +41,11 @@ class DesignState(object): raise NameError("Site %s not found in design state" % site_name) - def add_network(self, new_network): - if new_network is None or not isinstance(new_network, model.Network): - raise Exception("Invalid Network model") + def add_network(self, new_network): + if new_network is None or not isinstance(new_network, model.Network): + raise Exception("Invalid Network model") - self.networks.append(new_network) + self.networks.append(new_network) def get_networks(self): return self.networks @@ -57,11 +57,11 @@ class DesignState(object): raise NameError("Network %s not found in design state" % network_name) - def add_network_link(self, new_network_link): - if new_network_link is None or not isinstance(new_network_link, model.NetworkLink): - raise Exception("Invalid NetworkLink model") + def add_network_link(self, new_network_link): + if new_network_link is None or not isinstance(new_network_link, model.NetworkLink): + raise Exception("Invalid NetworkLink model") - self.network_links.append(new_network_link) + self.network_links.append(new_network_link) def get_network_links(self): return self.network_links @@ -73,11 +73,11 @@ class DesignState(object): raise NameError("NetworkLink %s not found in design state" % link_name) - def add_host_profile(self, new_host_profile): - if new_host_profile is None or not isinstance(new_host_profile, model.HostProfile): - raise Exception("Invalid HostProfile model") + def add_host_profile(self, new_host_profile): + if new_host_profile is None or not isinstance(new_host_profile, model.HostProfile): + raise Exception("Invalid HostProfile model") - self.host_profiles.append(new_host_profile) + self.host_profiles.append(new_host_profile) def get_host_profiles(self): return self.host_profiles @@ -89,11 +89,11 @@ class DesignState(object): raise NameError("HostProfile %s not found in design state" % profile_name) - def add_hardware_profile(self, new_hardware_profile): - if new_hardware_profile is None or not isinstance(new_hardware_profile, model.HardwareProfile): - raise Exception("Invalid HardwareProfile model") + def add_hardware_profile(self, new_hardware_profile): + if new_hardware_profile is None or not isinstance(new_hardware_profile, model.HardwareProfile): + raise Exception("Invalid HardwareProfile model") - self.hardware_profiles.append(new_hardware_profile) + self.hardware_profiles.append(new_hardware_profile) def get_hardware_profiles(self): return self.hardware_profiles @@ -105,11 +105,11 @@ class DesignState(object): raise NameError("HardwareProfile %s not found in design state" % profile_name) - def add_baremetal_node(self, new_baremetal_node): - if new_baremetal_node is None or not isinstance(new_baremetal_node, model.BaremetalNode): - raise Exception("Invalid BaremetalNode model") + def add_baremetal_node(self, new_baremetal_node): + if new_baremetal_node is None or not isinstance(new_baremetal_node, model.BaremetalNode): + raise Exception("Invalid BaremetalNode model") - self.baremetal_nodes.append(new_baremetal_node) + self.baremetal_nodes.append(new_baremetal_node) def get_baremetal_nodes(self): return self.baremetal_nodes diff --git a/setup.py b/setup.py index caa39b96..adc96bd7 100644 --- a/setup.py +++ b/setup.py @@ -43,7 +43,8 @@ setup(name='helm_drydock', 'helm_drydock.model', 'helm_drydock.ingester', 'helm_drydock.ingester.plugins', - 'helm_drydock.statemgmt'], + 'helm_drydock.statemgmt', + 'helm_drydock.orchestrator'], install_requires=[ 'PyYAML', 'oauth', diff --git a/tests/aicyaml_samples/fullsite.yaml b/tests/aicyaml_samples/fullsite.yaml index 08ebf4de..3a7f8ede 100644 --- a/tests/aicyaml_samples/fullsite.yaml +++ b/tests/aicyaml_samples/fullsite.yaml @@ -18,23 +18,22 @@ #################### # version the schema in this file so consumers can rationally parse it --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: Region metadata: name: sitename date: 17-FEB-2017 - name: Sample site design + description: Sample site design author: sh8121@att.com spec: # Not sure if we have site wide data that doesn't fall into another 'Kind' --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: NetworkLink metadata: name: oob region: sitename date: 17-FEB-2017 - name: Sample network link author: sh8121@att.com description: Describe layer 1 attributes. Primary key is 'name'. These settings will generally be things the switch and server have to agree on spec: @@ -49,13 +48,12 @@ spec: # pxe is a bit of 'magic' indicating the link config used when PXE booting # a node. All other links indicate network configs applied when the node # is deployed. -apiVersion: '1.0' +apiVersion: 'v1.0' kind: NetworkLink metadata: name: pxe region: sitename date: 17-FEB-2017 - name: Sample network link author: sh8121@att.com description: Describe layer 1 attributes. Primary key is 'name'. These settings will generally be things the switch and server have to agree on spec: @@ -71,13 +69,12 @@ spec: # use name, will translate to VLAN ID default_network: pxe --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: NetworkLink metadata: name: gp region: sitename date: 17-FEB-2017 - name: Sample network link author: sh8121@att.com description: Describe layer 1 attributes. These CIs will generally be things the switch and server have to agree on # pxe is a bit of 'magic' indicating the link config used when PXE booting @@ -105,13 +102,12 @@ spec: mode: tagged default_network: mgmt --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: Network metadata: name: oob region: sitename date: 17-FEB-2017 - name: Sample network definition author: sh8121@att.com description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces spec: @@ -125,13 +121,12 @@ spec: domain: ilo.sitename.att.com servers: 172.16.100.10 --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: Network metadata: name: pxe region: sitename date: 17-FEB-2017 - name: Sample network definition author: sh8121@att.com description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces spec: @@ -155,13 +150,12 @@ spec: # DNS servers that a server using this network as its default gateway should use servers: 172.16.0.10 --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: Network metadata: name: mgmt region: sitename date: 17-FEB-2017 - name: Sample network definition author: sh8121@att.com description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces spec: @@ -191,13 +185,12 @@ spec: # DNS servers that a server using this network as its default gateway should use servers: 172.16.1.9,172.16.1.10 --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: Network metadata: name: private region: sitename date: 17-FEB-2017 - name: Sample network definition author: sh8121@att.com description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces spec: @@ -216,13 +209,12 @@ spec: domain: priv.sitename.example.com servers: 172.16.2.9,172.16.2.10 --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: Network metadata: name: public region: sitename date: 17-FEB-2017 - name: Sample network definition author: sh8121@att.com description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces spec: @@ -245,13 +237,12 @@ spec: domain: sitename.example.com servers: 8.8.8.8 --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: HostProfile metadata: - name: default + name: defaults region: sitename date: 17-FEB-2017 - name: Sample network definition author: sh8121@att.com description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces # No magic to this host_profile, it just provides a way to specify @@ -305,13 +296,12 @@ spec: # Base URL of the introspection service - may go in curtin data introspection_url: http://172.16.1.10:9090 --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: HostProfile metadata: name: k8-node region: sitename date: 17-FEB-2017 - name: Sample network definition author: sh8121@att.com description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces spec: @@ -339,7 +329,7 @@ spec: - prim_nic01 # Which networks will be configured on this interface networks: - - name: pxe + - pxe - device_name: bond0 network_link: gp # If multiple slaves are specified, but no bonding config @@ -350,8 +340,8 @@ spec: # If multiple networks are specified, but no trunking # config is applied to the link, design validation will fail networks: - - name: mgmt - - name: private + - mgmt + - private metadata: # Explicit tag assignment tags: @@ -360,13 +350,12 @@ spec: owner_data: foo: bar --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: HostProfile metadata: name: k8-node-public region: sitename date: 17-FEB-2017 - name: Sample network definition author: sh8121@att.com description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces spec: @@ -376,15 +365,14 @@ spec: networks: # This is additive, so adds a network to those defined in the host_profile # inheritance chain - - name: public + - public --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: BaremetalNode metadata: name: controller01 region: sitename date: 17-FEB-2017 - name: Sample network definition author: sh8121@att.com description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces spec: @@ -395,7 +383,7 @@ spec: - device_name: bond0 networks: # '!' prefix for the value of the primary key indicates a record should be removed - - name: '!private' + - '!private' # Addresses assigned to network interfaces addressing: # Which network the address applies to. If a network appears in addressing @@ -412,13 +400,12 @@ spec: roles: os_ctl rack: rack01 --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: BaremetalNode metadata: name: compute01 region: sitename date: 17-FEB-2017 - name: Sample network definition author: sh8121@att.com description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces spec: @@ -431,13 +418,12 @@ spec: - network: private address: 172.16.2.21 --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: HardwareProfile metadata: - name: HPGen8v3 + name: HPGen9v3 region: sitename date: 17-FEB-2017 - name: Sample hardware definition author: Scott Hussey spec: # Vendor of the server chassis diff --git a/tests/aicyaml_samples/multidoc.yaml b/tests/aicyaml_samples/multidoc.yaml index 693afd1d..0bd35190 100644 --- a/tests/aicyaml_samples/multidoc.yaml +++ b/tests/aicyaml_samples/multidoc.yaml @@ -1,5 +1,5 @@ --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: NetworkLink metadata: name: oob @@ -20,7 +20,7 @@ spec: # pxe is a bit of 'magic' indicating the link config used when PXE booting # a node. All other links indicate network configs applied when the node # is deployed. -apiVersion: '1.0' +apiVersion: 'v1.0' kind: NetworkLink metadata: name: pxe @@ -42,7 +42,7 @@ spec: # use name, will translate to VLAN ID default_network: pxe --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: NetworkLink metadata: name: gp diff --git a/tests/aicyaml_samples/singledoc.yaml b/tests/aicyaml_samples/singledoc.yaml index 8ad16094..6d24c8c8 100644 --- a/tests/aicyaml_samples/singledoc.yaml +++ b/tests/aicyaml_samples/singledoc.yaml @@ -1,5 +1,5 @@ --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: HardwareProfile metadata: name: HPGen8v3 diff --git a/tests/aicyaml_samples/unknown_kind.yaml b/tests/aicyaml_samples/unknown_kind.yaml index bd39bfc0..bfcd4c2b 100644 --- a/tests/aicyaml_samples/unknown_kind.yaml +++ b/tests/aicyaml_samples/unknown_kind.yaml @@ -1,5 +1,5 @@ --- -apiVersion: '1.0' +apiVersion: 'v1.0' kind: FooBar metadata: name: default diff --git a/tests/test_design_inheritance.py b/tests/test_design_inheritance.py new file mode 100644 index 00000000..991289bb --- /dev/null +++ b/tests/test_design_inheritance.py @@ -0,0 +1,70 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from helm_drydock.ingester import Ingester +from helm_drydock.statemgmt import DesignState +from helm_drydock.orchestrator.designdata import DesignStateClient + +from copy import deepcopy + +import pytest +import shutil +import os +import helm_drydock.ingester.plugins.aicyaml +import yaml + +class TestClass(object): + + def setup_method(self, method): + print("Running test {0}".format(method.__name__)) + + + def test_design_inheritance(self, loaded_design): + client = DesignStateClient() + + design_data = client.load_design_data(design_state=loaded_design) + design_data = client.compute_model_inheritance(design_data) + + print(yaml.dump(design_data, default_flow_style=False)) + + node = design_data.get_baremetal_node("controller01") + + assert node.hardware_profile == 'HPGen9v3' + + @pytest.fixture(scope='module') + def loaded_design(self, input_files): + input_file = input_files.join("fullsite.yaml") + + module_design_state = DesignState() + + ingester = Ingester() + ingester.enable_plugins([helm_drydock.ingester.plugins.aicyaml.AicYamlIngester]) + ingester.ingest_data(plugin_name='aic_yaml', design_state=module_design_state, filenames=[str(input_file)]) + + return module_design_state + + + + @pytest.fixture(scope='module') + def input_files(self, tmpdir_factory, request): + tmpdir = tmpdir_factory.mktemp('data') + samples_dir = os.path.dirname(str(request.fspath)) + "/aicyaml_samples" + samples = os.listdir(samples_dir) + + for f in samples: + src_file = samples_dir + "/" + f + dst_file = str(tmpdir) + "/" + f + shutil.copyfile(src_file, dst_file) + + return tmpdir \ No newline at end of file diff --git a/tests/test_models.py b/tests/test_models.py index c8bfdfab..3e993c44 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -23,7 +23,7 @@ class TestClass(object): def test_hardwareprofile(self): yaml_snippet = ("---\n" - "apiVersion: '1.0'\n" + "apiVersion: 'v1.0'\n" "kind: HardwareProfile\n" "metadata:\n" " name: HPGen8v3\n"