From f3c2a7e2de0abf3d2d065c0d1553c4cd89769ce8 Mon Sep 17 00:00:00 2001 From: Scott Hussey Date: Thu, 23 Feb 2017 16:08:00 -0600 Subject: [PATCH] Complete Ingester basic test cases Begin orchestrator site design data management --- examples/bootstrap_seed.yaml | 3 +- helm_drydock/ingester/__init__.py | 18 +++-- helm_drydock/ingester/plugins/__init__.py | 2 +- helm_drydock/model/__init__.py | 27 ++++++-- helm_drydock/orchestrator/designdata.py | 80 +++++++++++++++++++++++ helm_drydock/statemgmt/__init__.py | 58 ++++++++++++++++ tests/aicyaml_samples/fullsite.yaml | 12 ++-- tests/test_ingester.py | 7 +- 8 files changed, 183 insertions(+), 24 deletions(-) create mode 100644 helm_drydock/orchestrator/designdata.py diff --git a/examples/bootstrap_seed.yaml b/examples/bootstrap_seed.yaml index 9c97a040..f5c20c14 100644 --- a/examples/bootstrap_seed.yaml +++ b/examples/bootstrap_seed.yaml @@ -409,7 +409,8 @@ spec: - network: public address: 172.16.3.20 metadata: - roles: os_ctl + tags: + - os_ctl rack: rack01 --- apiVersion: '1.0' diff --git a/helm_drydock/ingester/__init__.py b/helm_drydock/ingester/__init__.py index 44823078..beae66a3 100644 --- a/helm_drydock/ingester/__init__.py +++ b/helm_drydock/ingester/__init__.py @@ -21,14 +21,12 @@ import helm_drydock.model as model from helm_drydock.statemgmt import DesignState - class Ingester(object): - registered_plugins = {} - def __init__(self): logging.basicConfig(format="%(asctime)-15s [%(levelname)] %(module)s %(process)d %(message)s") self.log = logging.Logger("ingester") + self.registered_plugins = {} """ enable_plugins @@ -41,18 +39,18 @@ class Ingester(object): it will throw an exception """ def enable_plugins(self, plugins=[]): - if len(plugin) == 0: + if len(plugins) == 0: self.log.error("Cannot have an empty plugin list.") for plugin in plugins: try: new_plugin = plugin() plugin_name = new_plugin.get_name() - registered_plugins[plugin_name] = new_plugin + self.registered_plugins[plugin_name] = new_plugin except: self.log.error("Could not enable plugin %s" % (plugin.__name__)) - if len(registered_plugins) == 0: + if len(self.registered_plugins) == 0: self.log.error("Could not enable at least one plugin") raise Exception("Could not enable at least one plugin") @@ -64,13 +62,13 @@ class Ingester(object): Execute a data ingestion using the named plugin (assuming it is enabled) """ - def ingest_data(self, plugin_name, design_state=None, **kwargs): + def ingest_data(self, plugin_name='', design_state=None, **kwargs): if design_state is None: self.log.error("ingest_data called without valid DesignState handler") raise Exception("Invalid design_state handler") - if plugin_name in registered_plugins: - design_data = registered_plugins[plugin_name].ingest_data(kwargs) + if plugin_name in self.registered_plugins: + design_data = self.registered_plugins[plugin_name].ingest_data(**kwargs) # Need to persist data here, but we don't yet have the statemgmt service working for m in design_data: if type(m) is model.Site: @@ -85,7 +83,7 @@ class Ingester(object): design_state.add_hardware_profile(m) elif type(m) is model.BaremetalNode: design_state.add_baremetal_node(m) - else + else: self.log.error("Could not find plugin %s to ingest data." % (plugin_name)) raise LookupError("Could not find plugin %s" % plugin_name) diff --git a/helm_drydock/ingester/plugins/__init__.py b/helm_drydock/ingester/plugins/__init__.py index 7281f22e..cf92a0e9 100644 --- a/helm_drydock/ingester/plugins/__init__.py +++ b/helm_drydock/ingester/plugins/__init__.py @@ -13,7 +13,7 @@ # limitations under the License. # # Plugins to parse incoming topology and translate it to helm-drydock's -# intermediate representation +# model representation import logging diff --git a/helm_drydock/model/__init__.py b/helm_drydock/model/__init__.py index 1be9d722..a6d32570 100644 --- a/helm_drydock/model/__init__.py +++ b/helm_drydock/model/__init__.py @@ -28,7 +28,7 @@ class HardwareProfile(object): # Need to add validation logic, we'll assume the input is # valid for now self.name = metadata.get('name', '') - self.region = metadata.get('region', '') + self.site = metadata.get('region', '') self.vendor = spec.get('vendor', '') self.generation = spec.get('generation', '') self.hw_version = spec.get('hw_version', '') @@ -106,7 +106,7 @@ class NetworkLink(object): spec = kwargs.get('spec', {}) self.name = metadata.get('name', '') - self.region = metadata.get('region', '') + self.site = metadata.get('region', '') bonding = spec.get('bonding', {}) self.bonding_mode = bonding.get('mode', 'none') @@ -140,7 +140,7 @@ class Network(object): spec = kwargs.get('spec', {}) self.name = metadata.get('name', '') - self.region = metadata.get('region', '') + self.site = metadata.get('region', '') self.cidr = spec.get('cidr', '') self.allocation_strategy = spec.get('allocation', 'static') self.vlan_id = spec.get('vlan_id', 1) @@ -201,7 +201,7 @@ class HostProfile(object): spec = kwargs.get('spec', {}) self.name = metadata.get('name', '') - self.region = metadata.get('region', '') + self.site = metadata.get('region', '') oob = spec.get('oob', {}) self.oob_type = oob.get('type', 'ipmi') @@ -229,9 +229,28 @@ class HostProfile(object): for i in interfaces: self.interfaces.append(HostInterface(self.api_version, **i)) + metadata = spec.get('metadata', {}) + + metadata_tags = metadata.get('tags', []) + self.tags = [] + + for t in metadata_tags: + self.tags.append(t) + + owner_data = metadata.get('owner_data', {}) + self.owner_data = {} + + for k, v in owner_data.items(): + self.owner_data[k] = v + + self.rack = metadata.get('rack', '') + else: raise ValueError('Unknown API version of object') + def inherit_parent(self, site): + + def apply_hardware_profile(self, site): class HostInterface(object): diff --git a/helm_drydock/orchestrator/designdata.py b/helm_drydock/orchestrator/designdata.py new file mode 100644 index 00000000..fe7a268a --- /dev/null +++ b/helm_drydock/orchestrator/designdata.py @@ -0,0 +1,80 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +class DesignStateClient(object): + + def __init__(self): + self.log = logging.Logger('orchestrator') + + """ + load_design_data - Pull all the defined models in statemgmt and assemble + them into a representation of the site. Does not compute inheritance. + Throws an exception if multiple Site models are found. + + param design_state - Instance of statemgmt.DesignState to load data from + + return a Site model populated with all components from the design state + """ + + def load_design_data(self, design_state=None): + if len(design_state.get_sites()) != 1: + self.log.error("Invalid design state, should only have 1 Site model") + raise Exception("Invalid design state, should only have 1 Site model") + + site = design_state.get_sites()[0] + site_name = site.name + + networks = design_state.get_networks() + + for n in networks: + if n.site == site_name: + site.networks.append(n) + + network_links = design_state.get_network_links() + + for l in network_links: + if l.site == site_name: + site.network_links.append(l) + + host_profiles = design_state.get_host_profiles() + + for p in host_profiles: + if p.site == site_name: + site.host_profiles.append(p) + + hardware_profiles = design_state.get_hardware_profiles() + + for p in hardware_profiles: + if p.site == site_name: + site.hardware_profiles.append(p) + + baremetal_nodes = design_state.get_baremetal_nodes() + + for n in baremetal_nodes: + if n.site == site_name: + site.baremetal_nodes.append(n) + + + return site + + """ + compute_model_inheritance - given a fully populated Site model, compute the effecitve + design by applying inheritance and references + + return a Site model reflecting the effective design for the site + """ + def compute_model_inheritance(self, site_root): + \ No newline at end of file diff --git a/helm_drydock/statemgmt/__init__.py b/helm_drydock/statemgmt/__init__.py index 1fba0192..110f974d 100644 --- a/helm_drydock/statemgmt/__init__.py +++ b/helm_drydock/statemgmt/__init__.py @@ -31,34 +31,92 @@ class DesignState(object): self.sites.append(new_site) + def get_sites(self): + return self.sites + + def get_site(self, site_name): + for s in self.sites: + if s.name == site_name: + return s + + raise NameError("Site %s not found in design state" % site_name) + def add_network(self, new_network): if new_network is None or not isinstance(new_network, model.Network): raise Exception("Invalid Network model") self.networks.append(new_network) + def get_networks(self): + return self.networks + + def get_network(self, network_name): + for n in self.networks: + if n.name == network_name: + return n + + raise NameError("Network %s not found in design state" % network_name) + def add_network_link(self, new_network_link): if new_network_link is None or not isinstance(new_network_link, model.NetworkLink): raise Exception("Invalid NetworkLink model") self.network_links.append(new_network_link) + def get_network_links(self): + return self.network_links + + def get_network_link(self, link_name): + for l in self.network_links: + if l.name == link_name: + return l + + raise NameError("NetworkLink %s not found in design state" % link_name) + def add_host_profile(self, new_host_profile): if new_host_profile is None or not isinstance(new_host_profile, model.HostProfile): raise Exception("Invalid HostProfile model") self.host_profiles.append(new_host_profile) + def get_host_profiles(self): + return self.host_profiles + + def get_host_profile(self, profile_name): + for p in self.host_profiles: + if p.name == profile_name: + return p + + raise NameError("HostProfile %s not found in design state" % profile_name) + def add_hardware_profile(self, new_hardware_profile): if new_hardware_profile is None or not isinstance(new_hardware_profile, model.HardwareProfile): raise Exception("Invalid HardwareProfile model") self.hardware_profiles.append(new_hardware_profile) + def get_hardware_profiles(self): + return self.hardware_profiles + + def get_hardware_profile(self, profile_name): + for p in self.hardware_profiles: + if p.name == profile_name: + return p + + raise NameError("HardwareProfile %s not found in design state" % profile_name) + def add_baremetal_node(self, new_baremetal_node): if new_baremetal_node is None or not isinstance(new_baremetal_node, model.BaremetalNode): raise Exception("Invalid BaremetalNode model") self.baremetal_nodes.append(new_baremetal_node) + def get_baremetal_nodes(self): + return self.baremetal_nodes + def get_baremetal_node(self, node_name): + for n in self.baremetal_nodes: + if n.name == node_name: + return n + + raise NameError("BaremetalNode %s not found in design state" % node_name) diff --git a/tests/aicyaml_samples/fullsite.yaml b/tests/aicyaml_samples/fullsite.yaml index 92e275ce..08ebf4de 100644 --- a/tests/aicyaml_samples/fullsite.yaml +++ b/tests/aicyaml_samples/fullsite.yaml @@ -118,12 +118,12 @@ spec: allocation: static cidr: 172.16.100.0/24 ranges: - - type: static - start: 172.16.100.15 - end: 172.16.100.254 - dns: - domain: ilo.sitename.att.com - servers: 172.16.100.10 + - type: static + start: 172.16.100.15 + end: 172.16.100.254 + dns: + domain: ilo.sitename.att.com + servers: 172.16.100.10 --- apiVersion: '1.0' kind: Network diff --git a/tests/test_ingester.py b/tests/test_ingester.py index fc5c3ed0..edece137 100644 --- a/tests/test_ingester.py +++ b/tests/test_ingester.py @@ -18,6 +18,7 @@ from helm_drydock.statemgmt import DesignState import pytest import shutil import os +import helm_drydock.ingester.plugins.aicyaml class TestClass(object): @@ -28,8 +29,10 @@ class TestClass(object): input_file = input_files.join("fullsite.yaml") ingester = Ingester() - ingester.enable_plugins([helm_drydock.ingester.plugins.aicyaml.AicYamlPlugin]) - ingester.ingest_data('aic_yaml', design_state=design_state, filenames=) + ingester.enable_plugins([helm_drydock.ingester.plugins.aicyaml.AicYamlIngester]) + ingester.ingest_data(plugin_name='aic_yaml', design_state=design_state, filenames=[str(input_file)]) + + assert len(design_state.host_profiles) == 3 @pytest.fixture(scope='module') def input_files(self, tmpdir_factory, request):