Merge pull request #12 from sh8121att/master

Python skeleton and initial test cases

README.md

@@ -1,50 +1,52 @@
-# drydock
+# helm_drydock
 A python REST orchestrator to translate a YAML host topology to a provisioned set of hosts and provide a set of cloud-init post-provisioning instructions.
 
 ## Modular service
 
 ### Design Consumer ###
 
-aka smelter
+aka ingester
 
 Pluggable service to ingest a inventory/design specification, convert it to a standard
 internal representaion, and persist it to the Design State API. Initial implementation
-is the consumer of AIC YAML schema.
+is the consumer of YAML schema.
 
 ### Design State API ###
 
-aka tarot
+aka statemgmt
 
 API for querying and updating the current design specification and persisted orchestration status.
 CRUD support of CIs that are not bootstrap-related, but can be used by other automation.
 
 ### Control API ###
 
-aka cockpit
+aka control
 
 User-approachable API for initiating orchestration actions or accessing other internal
 APIs
 
 ### Infrastructure Orchestrator ###
 
-aka alchemist
+aka orchestrator
 
 Handle validation of complete design, ordering and managing downstream API calls for hardware
 provisioning/bootstrapping
 
-### Server Driver ### 
+### OOB Driver ###
 
-aka maasdriver
+Pluggable provider for server OOB (ILO) management
+
+aka driver/oob
+
+### Node Driver ###
+
+aka driver/node
 
 Pluggable provisioner for server bootstrapping. Initial implementation is MaaS client.
 
-### Network Driver ###
-
-Pluggable provisioner for network provisioning. Initial implementation is Noop.
-
 ### Introspection API ###
 
-aka jabberwocky
+aka introspection
 
 API for bootstrapping nodes to load self data. Possibly pluggable as this is basically an
 authenticated bridge to the Design State API

examples/bootstrap_seed.yaml

@@ -397,7 +397,8 @@ spec:
   - network: public
     address: 172.16.3.20
   metadata:
-    roles: os_ctl
+    tags:
+    - os_ctl
     rack: rack01
 ---
 apiVersion: 'v1.0'

helm_drydock/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

helm_drydock/config.py

@@ -0,0 +1,41 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# Read application configuration
+#
+
+# configuration map with defaults
+
+class DrydockConfig(object):
+
+    def __init__(self):
+        self.server_driver_config = {
+            selected_driver = helm_drydock.drivers.server.maasdriver,
+            params = {
+                maas_api_key = "" 
+                maas_api_url = ""
+            }
+        }
+        self.selected_network_driver = helm_drydock.drivers.network.noopdriver
+        self.control_config = {}
+        self.ingester_config = {
+            plugins = [helm_drydock.ingester.plugins.aicyaml.AicYamlIngester]
+        }
+        self.introspection_config = {}
+        self.orchestrator_config = {}
+        self.statemgmt_config = {
+            backend_driver = helm_drydock.drivers.statemgmt.etcd,
+        }

helm_drydock/control/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

helm_drydock/control/readme.md

@@ -0,0 +1,6 @@
+# Control #
+
+This is the external facing API service to control the rest
+of Drydock and query Drydock-managed data.
+
+Anticipate basing this service on the falcon Python library

helm_drydock/drivers/__init__.py

@@ -0,0 +1,21 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class ProviderDriver(object):
+
+	__init__(self):
+        pass
+		
+
+class DriverTask(object):

helm_drydock/drivers/node/__init__.py

@@ -0,0 +1,29 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from helm_drydock.drivers import ProviderDriver
+
+class NodeDriver(ProviderDriver):
+
+class NodeAction(Enum):
+    PrepareNode = 'prepare_node'
+    ApplyNetworkConfig = 'apply_network_config'
+    ApplyStorageConfig = 'apply_storage_config'
+    InterrogateNode = 'interrogate_node'
+    DeployNode = 'deploy_node'
+
+
+
+	

helm_drydock/drivers/node/maasdriver/__init__.py

@@ -0,0 +1,20 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from helm_drydock.drivers.node import NodeDriver
+
+class MaasNodeDriver(NodeDriver):
+
+    def __init__(self, kwargs):
+        super(MaasNodeDriver, self).__init__(**kwargs)
+	

helm_drydock/drivers/oob/__init__.py

@@ -0,0 +1,42 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# OOB:
+# sync_hardware_clock
+# collect_chassis_sysinfo
+# enable_netboot
+# initiate_reboot
+# set_power_off
+# set_power_on
+
+from helm_drydock.drivers import ProviderDriver
+
+class OobDriver(ProviderDriver):
+
+    def __init__(self):
+        pass
+
+    def execute_action(self, action, **kwargs):
+        if action == 
+
+
+
+class OobAction(Enum):
+    ConfigNodePxe = 'config_node_pxe'
+    SetNodeBoot = 'set_node_boot'
+    PowerOffNode = 'power_off_node'
+    PowerOnNode = 'power_on_node'
+    PowerCycleNode = 'power_cycle_node'
+    InterrogateNode = 'interrogate_node'
+

helm_drydock/drivers/readme.md

@@ -0,0 +1,26 @@
+# Drivers #
+
+Drivers are downstream actors that Drydock will use to actually execute
+orchestration actions. It is intended to be a pluggable architecture
+so that various downstream automation can be used.
+
+## oob ##
+
+The oob drivers will interface with physical servers' out-of-band
+management system (e.g. Dell iDRAC, HP iLO, etc...). OOB management
+will be used for setting a system to use PXE boot and power cycling
+servers.
+
+## node ##
+
+The node drivers will interface with an external bootstrapping system
+for loading the base OS on a server and configuring hardware, network,
+and storage.
+
+## network ##
+
+The network drivers will interface with switches for managing port
+configuration to support the bootstrapping of physical nodes. This is not
+intended to be a network provisioner, but instead is a support driver
+for node bootstrapping where temporary changes to network configurations
+are required.

helm_drydock/error.py

@@ -0,0 +1,16 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class DesignError(Exception):
+    pass

helm_drydock/ingester/__init__.py

@@ -0,0 +1,104 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ingester - Ingest host topologies to define site design and
+# 			persist design to helm-drydock's statemgmt service
+
+import logging
+import yaml
+
+import helm_drydock.model.site as site
+import helm_drydock.model.network as network
+import helm_drydock.model.hwprofile as hwprofile
+import helm_drydock.model.node as node
+import helm_drydock.model.hostprofile as hostprofile
+
+from helm_drydock.statemgmt import DesignState, SiteDesign, DesignError
+
+class Ingester(object):
+
+    def __init__(self):
+        logging.basicConfig(format="%(asctime)-15s [%(levelname)] %(module)s %(process)d %(message)s")
+        self.log = logging.Logger("ingester")
+        self.registered_plugins = {}
+
+    def enable_plugins(self, plugins=[]):
+        if len(plugins) == 0:
+            self.log.error("Cannot have an empty plugin list.")
+
+        for plugin in plugins:
+            try:
+                new_plugin = plugin()
+                plugin_name = new_plugin.get_name()
+                self.registered_plugins[plugin_name] = new_plugin
+            except:
+                self.log.error("Could not enable plugin %s" % (plugin.__name__))
+
+        if len(self.registered_plugins) == 0:
+            self.log.error("Could not enable at least one plugin")
+            raise Exception("Could not enable at least one plugin")
+    """
+      enable_plugins
+
+      params: plugins - A list of class objects denoting the ingester plugins to be enabled
+
+      Enable plugins that can be used for ingest_data calls. Each plugin should use
+      helm_drydock.ingester.plugins.IngesterPlugin as its base class. As long as one
+      enabled plugin successfully initializes, the call is considered successful. Otherwise
+      it will throw an exception
+    """
+    
+    def ingest_data(self, plugin_name='', design_state=None, **kwargs):
+        if design_state is None:
+            self.log.error("ingest_data called without valid DesignState handler")
+            raise Exception("Invalid design_state handler")
+
+        # TODO this method needs refactored to handle design base vs change
+
+        design_data = None
+
+        try:
+            design_data = design_state.get_design_base()
+        except DesignError:
+            design_data = SiteDesign()
+
+        if plugin_name in self.registered_plugins:
+            design_items = self.registered_plugins[plugin_name].ingest_data(**kwargs)
+            # Need to persist data here, but we don't yet have the statemgmt service working
+            for m in design_items:
+                if type(m) is site.Site:
+                    design_data.add_site(m)
+                elif type(m) is network.Network:
+                    design_data.add_network(m)
+                elif type(m) is network.NetworkLink:
+                    design_data.add_network_link(m)
+                elif type(m) is hostprofile.HostProfile:
+                    design_data.add_host_profile(m)
+                elif type(m) is hwprofile.HardwareProfile:
+                    design_data.add_hardware_profile(m)
+                elif type(m) is node.BaremetalNode:
+                    design_data.add_baremetal_node(m)
+            design_state.put_design_base(design_data)
+        else:
+            self.log.error("Could not find plugin %s to ingest data." % (plugin_name))
+            raise LookupError("Could not find plugin %s" % plugin_name)
+    """
+      ingest_data
+
+      params: plugin_name - Which plugin should be used for ingestion
+      params: params - A map of parameters that will be passed to the plugin's ingest_data method
+
+      Execute a data ingestion using the named plugin (assuming it is enabled) 
+    """
+

helm_drydock/ingester/plugins/__init__.py

@@ -0,0 +1,30 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Plugins to parse incoming topology and translate it to helm-drydock's
+# model representation
+
+import logging
+
+class IngesterPlugin(object):
+
+    def __init__(self):
+        self.log = logging.Logger('ingester')
+        return
+
+    def get_data(self):
+        return "ingester_skeleton"
+
+    def ingest_data(self, **kwargs):
+        return {}

helm_drydock/ingester/plugins/yaml.py

@@ -0,0 +1,113 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# AIC YAML Ingester - This data ingester will consume a AIC YAML design
+#                   file 
+#   
+import yaml
+import logging
+
+import helm_drydock.model.hwprofile as hwprofile
+import helm_drydock.model.node as node
+import helm_drydock.model.site as site
+import helm_drydock.model.hostprofile as hostprofile
+import helm_drydock.model.network as network
+
+from helm_drydock.ingester.plugins import IngesterPlugin
+
+class YamlIngester(IngesterPlugin):
+
+    kind_map = {
+        "Region": site.Site,
+        "NetworkLink": network.NetworkLink,
+        "HardwareProfile": hwprofile.HardwareProfile,
+        "Network": network.Network,
+        "HostProfile": hostprofile.HostProfile,
+        "BaremetalNode": node.BaremetalNode,
+    }
+
+    def __init__(self):
+        super(YamlIngester, self).__init__()
+
+    def get_name(self):
+        return "yaml"
+
+    """
+    AIC YAML ingester params
+
+    filenames - Array of absolute path to the YAML files to ingest
+
+    returns an array of objects from helm_drydock.model
+
+    """
+    def ingest_data(self, **kwargs):
+        models = []
+
+        if 'filenames' in kwargs:
+            # TODO validate filenames is array
+            for f in kwargs.get('filenames'):
+                try:
+                    file = open(f,'rt')
+                    contents = file.read()
+                    file.close()
+                    models.extend(self.parse_docs(contents))
+                except OSError as err:
+                    self.log.error(
+                        "Error opening input file %s for ingestion: %s" 
+                        % (filename, err))
+                    continue
+        
+                
+        elif 'content' in kwargs:
+            models.extend(self.parse_docs(kwargs.get('content')))
+        else:
+            raise ValueError('Missing parameter "filename"')
+        
+        return models
+
+    """
+    Translate a YAML string into the internal Drydock model
+    """
+    def parse_docs(self, yaml_string):
+        models = []
+
+        try:
+            parsed_data = yaml.load_all(yaml_string)
+        except yaml.YAMLError as err:
+            raise ValueError("Error parsing YAML in %s: %s" % (f,err))
+
+        for d in parsed_data:
+            kind = d.get('kind', '')
+            if kind != '':
+                if kind in YamlIngester.kind_map:
+                    try:
+                        model = YamlIngester.kind_map[kind](**d)
+                        models.append(model)
+                    except Exception as err:
+                        self.log.error("Error building model %s: %s" 
+                                       % (kind, str(err)))
+                        continue
+                else:
+                    self.log.error(
+                        "Error processing document, unknown kind %s" 
+                        % (kind))
+                    continue
+            else:
+                self.log.error(
+                    "Error processing document in %s, no kind field"
+                    % (f))
+                continue
+
+        return models

helm_drydock/ingester/readme.md

@@ -0,0 +1,13 @@
+# Ingester #
+
+Ingester is a pluggable consumer of site design data. It
+will support consuming data in different formats from
+different sources.
+
+Each ingester plugin should be able source data
+based on user-provided parameters and parse that data
+into the Drydock internal model (helm_drydock.model).
+
+Each plugin does not need to support every type of design
+data as a single site design may be federated from multiple
+sources.

helm_drydock/introspection/readme.md

@@ -0,0 +1,9 @@
+# Introspection #
+
+Introspection is a cloud-init compatible metadata service
+that is used to make a node self-aware. After a full
+deployment by the node driver, the newly installed OS
+will contact the introspection API to gain a package of
+declaritive data defining the node's role in the site and
+enough initial data to start the promenade process of
+Kubernetes assimilation

helm_drydock/model/__init__.py

@@ -0,0 +1,127 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Models for helm_drydock
+#
+import logging
+
+from copy import deepcopy
+
+# Utility class for calculating inheritance
+
+class Utils(object):
+
+    """
+    apply_field_inheritance - apply inheritance rules to a single field value
+
+    param child_field - value of the child field, or the field receiving
+                        the inheritance
+    param parent_field - value of the parent field, or the field supplying
+                        the inheritance
+
+    return the correct value for child_field based on the inheritance algorithm
+
+    Inheritance algorithm
+
+    1. If child_field is not None, '!' for string vals or -1 for numeric
+       vals retain the value
+    of child_field
+
+    2. If child_field is '!' return None to unset the field value
+
+    3. If child_field is -1 return None to unset the field value
+
+    4. If child_field is None return parent_field
+    """
+
+    @staticmethod
+    def apply_field_inheritance(child_field, parent_field):
+
+        if child_field is not None:
+            if child_field != '!' and child_field != -1:
+                return child_field
+            else:
+                return None
+        else:
+            return parent_field
+
+    """
+    merge_lists - apply inheritance rules to a list of simple values
+
+    param child_list - list of values from the child
+    param parent_list - list of values from the parent
+
+    return a merged list with child values taking prority
+
+    1. All members in the child list not starting with '!'
+
+    2. If a member in the parent list has a corresponding member in the
+    chid list prefixed with '!' it is removed
+
+    3. All remaining members of the parent list
+    """
+    @staticmethod
+    def merge_lists(child_list, parent_list):
+
+        if type(child_list) is not list or type(parent_list) is not list:
+            raise ValueError("One parameter is not a list")
+
+        effective_list = []
+
+        # Probably should handle non-string values
+        effective_list.extend(
+            filter(lambda x: not x.startswith("!"), child_list))
+
+        effective_list.extend(
+            filter(lambda x: ("!" + x) not in child_list,
+                   filter(lambda x: x not in effective_list, parent_list)))
+
+        return effective_list
+
+    """
+    merge_dicts - apply inheritance rules to a dict
+
+    param child_dict - dict of k:v from child
+    param parent_dict - dict of k:v from the parent
+
+    return a merged dict with child values taking prority
+
+    1. All members in the child dict with a key not starting with '!'
+
+    2. If a member in the parent dict has a corresponding member in the
+    chid dict where the key is prefixed with '!' it is removed
+
+    3. All remaining members of the parent dict
+    """
+    @staticmethod
+    def merge_dicts(child_dict, parent_dict):
+
+        if type(child_dict) is not dict or type(parent_dict) is not dict:
+            raise ValueError("One parameter is not a dict")
+
+        effective_dict = {}
+
+        # Probably should handle non-string keys
+        use_keys = filter(lambda x: ("!" + x) not in child_dict.keys(),
+                          parent_dict)
+
+        for k in use_keys:
+            effective_dict[k] = deepcopy(parent_dict[k])
+
+        use_keys = filter(lambda x: not x.startswith("!"), child_dict)
+
+        for k in use_keys:
+            effective_dict[k] = deepcopy(child_dict[k])
+
+        return effective_dict

helm_drydock/model/hostprofile.py

@@ -0,0 +1,378 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Models for helm_drydock
+#
+import logging
+
+from copy import deepcopy
+
+from helm_drydock.orchestrator.enum import SiteStatus
+from helm_drydock.orchestrator.enum import NodeStatus
+from helm_drydock.model.network import Network
+from helm_drydock.model.network import NetworkLink
+from helm_drydock.model import Utils
+
+class HostProfile(object):
+
+    def __init__(self, **kwargs):
+        self.log = logging.Logger('model')
+
+        self.api_version = kwargs.get('apiVersion', '')
+
+        if self.api_version == "v1.0":
+            metadata = kwargs.get('metadata', {})
+            spec = kwargs.get('spec', {})
+
+            self.name = metadata.get('name', '')
+            self.site = metadata.get('region', '')
+
+            self.parent_profile = spec.get('host_profile', None)
+            self.hardware_profile = spec.get('hardware_profile', None)
+
+            oob = spec.get('oob', {})
+            self.oob_type = oob.get('type', None)
+            self.oob_network = oob.get('network', None)
+            self.oob_account = oob.get('account', None)
+            self.oob_credential = oob.get('credential', None)
+
+            storage = spec.get('storage', {})
+            self.storage_layout = storage.get('layout', 'lvm')
+
+            bootdisk = storage.get('bootdisk', {})
+            self.bootdisk_device = bootdisk.get('device', None)
+            self.bootdisk_root_size = bootdisk.get('root_size', None)
+            self.bootdisk_boot_size = bootdisk.get('boot_size', None)
+
+            partitions = storage.get('partitions', [])
+            self.partitions = []
+
+            for p in partitions:
+                self.partitions.append(HostPartition(self.api_version, **p))
+
+            interfaces = spec.get('interfaces', [])
+            self.interfaces = []
+
+            for i in interfaces:
+                self.interfaces.append(HostInterface(self.api_version, **i))
+
+            node_metadata = spec.get('metadata', {})
+
+            metadata_tags = node_metadata.get('tags', [])
+            self.tags = []
+
+            for t in metadata_tags:
+                self.tags.append(t)
+
+            owner_data = node_metadata.get('owner_data', {})
+            self.owner_data = {}
+
+            for k, v in owner_data.items():
+                self.owner_data[k] = v
+
+            self.rack = node_metadata.get('rack', None)
+
+        else:
+            self.log.error("Unknown API version %s of %s" %
+                (self.api_version, self.__class__))
+            raise ValueError('Unknown API version of object')
+
+    def get_rack(self):
+        return self.rack
+
+    def get_name(self):
+        return self.name
+
+    def has_tag(self, tag):
+        if tag in self.tags:
+            return True
+
+        return False
+
+    def apply_inheritance(self, site):
+        # We return a deep copy of the profile so as not to corrupt
+        # the original model
+        self_copy = deepcopy(self)
+
+        if self.parent_profile is None:
+            return self_copy
+
+        parent = site.get_host_profile(self.parent_profile)
+
+        if parent is None:
+            raise NameError("Cannot find parent profile %s for %s"
+                            % (self.parent_profile, self.name))
+
+        parent = parent.apply_inheritance(site)
+
+        # First compute inheritance for simple fields
+        inheritable_field_list = [
+            "hardware_profile", "oob_type", "oob_network",
+            "oob_credential", "oob_account", "storage_layout",
+            "bootdisk_device", "bootdisk_root_size", "bootdisk_boot_size",
+            "rack"]
+
+        for f in inheritable_field_list:
+            setattr(self_copy, f,
+                    Utils.apply_field_inheritance(getattr(self, f, None),
+                                                  getattr(parent, f, None)))
+
+        # Now compute inheritance for complex types
+        self_copy.tags = Utils.merge_lists(self.tags, parent.tags)
+
+        self_copy.owner_data = Utils.merge_dicts(
+            self.owner_data, parent.owner_data)
+
+        self_copy.interfaces = HostInterface.merge_lists(
+            self.interfaces, parent.interfaces)
+
+        self_copy.partitions = HostPartition.merge_lists(
+            self.partitions, parent.partitions)
+
+        return self_copy
+
+
+class HostInterface(object):
+
+    def __init__(self, api_version, **kwargs):
+        self.log = logging.Logger('model')
+
+        self.api_version = api_version
+
+        if self.api_version == "v1.0":
+            self.device_name = kwargs.get('device_name', None)
+            self.network_link = kwargs.get('device_link', None)
+
+            self.hardware_slaves = []
+            slaves = kwargs.get('slaves', [])
+
+            for s in slaves:
+                self.hardware_slaves.append(s)
+
+            self.networks = []
+            networks = kwargs.get('networks', [])
+
+            for n in networks:
+                self.networks.append(n)
+        else:
+            self.log.error("Unknown API version %s of %s" %
+                (self.api_version, self.__class__))
+            raise ValueError('Unknown API version of object')
+
+    # The device attribute may be hardware alias that translates to a
+    # physical device address. If the device attribute does not match an
+    # alias, we assume it directly identifies a OS device name. When the
+    # apply_hardware_profile method is called on the parent Node of this
+    # device, the selector will be decided and applied
+
+    def add_selector(self, sel_type, address='', dev_type=''):
+        if getattr(self, 'selectors', None) is None:
+            self.selectors = []
+
+        new_selector = {}
+        new_selector['selector_type'] = sel_type
+        new_selector['address'] = address
+        new_selector['device_type'] = dev_type
+
+        self.selectors.append(new_selector)
+
+    def get_slave_selectors(self):
+        return self.selectors
+
+    # Return number of slaves for this interface
+    def get_slave_count(self):
+        return len(self.hardware_slaves)
+
+    def apply_link_config(self, net_link):
+        if (net_link is not None and
+            isinstance(net_link, NetworkLink) and 
+            net_link.name == self.network_link):
+
+            self.attached_link = deepcopy(net_link)
+            return True
+        return False
+
+    def apply_network_config(self, network):
+        if network in self.networks:
+            if getattr(self, 'attached_networks', None) is None:
+                self.attached_networks = []
+            self.attached_networks.append(deepcopy(network))
+            return True
+        else:
+            return False
+
+    def set_network_address(self, network_name, address):
+        if getattr(self, 'attached_networks', None) is None:
+            return False
+
+        for n in self.attached_neteworks:
+            if n.name == network_name:
+                n.assigned_address = address
+
+    def get_network_configs(self):
+        return self.attached_networks
+
+    """
+    Merge two lists of HostInterface models with child_list taking
+    priority when conflicts. If a member of child_list has a device_name
+    beginning with '!' it indicates that HostInterface should be
+    removed from the merged list
+    """
+
+    @staticmethod
+    def merge_lists(child_list, parent_list):
+        if len(child_list) == 0:
+            return deepcopy(parent_list)
+
+        effective_list = []
+        if len(parent_list) == 0:
+            for i in child_list:
+                if i.device_name.startswith('!'):
+                    continue
+                else:
+                    effective_list.append(deepcopy(i))
+            return effective_list
+
+        parent_interfaces = []
+        for i in parent_list:
+            parent_name = i.device_name
+            parent_interfaces.append(parent_name)
+            add = True
+            for j in child_list:
+                if j.device_name == ("!" + parent_name):
+                    add = False
+                    break
+                elif j.device_name == parent_name:
+                    m = HostInterface(j.api_version)
+                    m.device_name = j.device_name
+                    m.network_link = \
+                        Utils.apply_field_inheritance(j.network_link,
+                                                      i.network_link)
+                    s = filter(lambda x: ("!" + x) not in j.hardware_slaves,
+                               i.hardware_slaves)
+                    s = list(s)
+
+                    s.extend(filter(lambda x: not x.startswith("!"),
+                                    j.hardware_slaves))
+                    m.hardware_slaves = s
+
+                    n = filter(lambda x: ("!" + x) not in j.networks,
+                               i.networks)
+                    n = list(n)
+
+                    n.extend(filter(lambda x: not x.startswith("!"),
+                                    j.networks))
+                    m.networks = n
+
+                    effective_list.append(m)
+                    add = False
+                    break
+
+            if add:
+                effective_list.append(deepcopy(i))
+
+        for j in child_list:
+            if (j.device_name not in parent_interfaces
+                and not j.device_name.startswith("!")):
+                effective_list.append(deepcopy(j))
+
+        return effective_list
+
+
+class HostPartition(object):
+
+    def __init__(self, api_version, **kwargs):
+        self.api_version = api_version
+
+        if self.api_version == "v1.0":
+            self.name = kwargs.get('name', None)
+            self.device = kwargs.get('device', None)
+            self.part_uuid = kwargs.get('part_uuid', None)
+            self.size = kwargs.get('size', None)
+            self.mountpoint = kwargs.get('mountpoint', None)
+            self.fstype = kwargs.get('fstype', 'ext4')
+            self.mount_options = kwargs.get('mount_options', 'defaults')
+            self.fs_uuid = kwargs.get('fs_uuid', None)
+            self.fs_label = kwargs.get('fs_label', None)
+        else:
+            raise ValueError('Unknown API version of object')
+
+    # The device attribute may be hardware alias that translates to a
+    # physical device address. If the device attribute does not match an
+    # alias, we assume it directly identifies a OS device name. When the
+    # apply_hardware_profile method is called on the parent Node of this
+    # device, the selector will be decided and applied
+
+    def set_selector(self, sel_type, address='', dev_type=''):
+        selector = {}
+        selector['type'] = sel_type
+        selector['address'] = address
+        selector['device_type'] = dev_type
+
+        self.selector = selector
+
+    def get_selector(self):
+        return self.selector
+
+    """
+    Merge two lists of HostPartition models with child_list taking
+    priority when conflicts. If a member of child_list has a name
+    beginning with '!' it indicates that HostPartition should be
+    removed from the merged list
+    """
+
+    @staticmethod
+    def merge_lists(child_list, parent_list):
+        if len(child_list) == 0:
+            return deepcopy(parent_list)
+
+        effective_list = []
+        if len(parent_list) == 0:
+            for i in child_list:
+                if i.name.startswith('!'):
+                    continue
+                else:
+                    effective_list.append(deepcopy(i))
+
+        inherit_field_list = ["device", "part_uuid", "size",
+                              "mountpoint", "fstype", "mount_options",
+                              "fs_uuid", "fs_label"]
+
+        parent_partitions = []
+        for i in parent_list:
+            parent_name = i.name
+            parent_partitions.append(parent_name)
+            add = True
+            for j in child_list:
+                if j.name == ("!" + parent_name):
+                    add = False
+                    break
+                elif j.name == parent_name:
+                    p = HostPartition(j.api_version)
+                    p.name = j.name
+
+                    for f in inherit_field_list:
+                        setattr(p, Utils.apply_field_inheritance(getattr(j, f),
+                                                                 getattr(i, f))
+                                )
+                    add = False
+                    effective_list.append(p)
+            if add:
+                effective_list.append(deepcopy(i))
+
+        for j in child_list:
+            if j.name not in parent_list:
+                effective_list.append(deepcopy(j))
+
+        return effective_list

helm_drydock/model/hwprofile.py

@@ -0,0 +1,95 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Models for helm_drydock
+#
+import logging
+
+from copy import deepcopy
+
+from helm_drydock.orchestrator.enum import SiteStatus
+from helm_drydock.orchestrator.enum import NodeStatus
+
+class HardwareProfile(object):
+
+    def __init__(self, **kwargs):
+        self.log = logging.Logger('model')
+
+        self.api_version = kwargs.get('apiVersion', '')
+
+        if self.api_version == "v1.0":
+            metadata = kwargs.get('metadata', {})
+            spec = kwargs.get('spec', {})
+
+            # Need to add validation logic, we'll assume the input is
+            # valid for now
+            self.name = metadata.get('name', '')
+            self.site = metadata.get('region', '')
+
+            self.vendor = spec.get('vendor', None)
+            self.generation = spec.get('generation', None)
+            self.hw_version = spec.get('hw_version', None)
+            self.bios_version = spec.get('bios_version', None)
+            self.boot_mode = spec.get('boot_mode', None)
+            self.bootstrap_protocol = spec.get('bootstrap_protocol', None)
+            self.pxe_interface = spec.get('pxe_interface', None)
+            self.devices = []
+
+            device_aliases = spec.get('device_aliases', {})
+
+            pci_devices = device_aliases.get('pci', [])
+            scsi_devices = device_aliases.get('scsi', [])
+
+            for d in pci_devices:
+                d['bus_type'] = 'pci'
+                self.devices.append(
+                    HardwareDeviceAlias(self.api_version, **d))
+
+            for d in scsi_devices:
+                d['bus_type'] = 'scsi'
+                self.devices.append(
+                    HardwareDeviceAlias(self.api_version, **d))
+        else:
+            self.log.error("Unknown API version %s of %s" %
+                (self.api_version, self.__class__))
+            raise ValueError('Unknown API version of object')
+
+        return
+
+    def resolve_alias(self, alias_type, alias):
+        selector = {}
+        for d in self.devices:
+            if d.alias == alias and d.bus_type == alias_type:
+                selector['address'] = d.address
+                selector['device_type'] = d.type
+                return selector
+
+        return None
+
+class HardwareDeviceAlias(object):
+
+    def __init__(self, api_version, **kwargs):
+        self.log = logging.Logger('model')
+
+        self.api_version = api_version
+
+        if self.api_version == "v1.0":
+            self.bus_type = kwargs.get('bus_type', None)
+            self.address = kwargs.get('address', None)
+            self.alias = kwargs.get('alias', None)
+            self.type = kwargs.get('type', None)
+        else:
+            self.log.error("Unknown API version %s of %s" %
+                (self.api_version, self.__class__))
+            raise ValueError('Unknown API version of object')

helm_drydock/model/network.py

@@ -0,0 +1,133 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Models for helm_drydock
+#
+import logging
+
+from copy import deepcopy
+
+from helm_drydock.orchestrator.enum import SiteStatus
+from helm_drydock.orchestrator.enum import NodeStatus
+
+class NetworkLink(object):
+
+    def __init__(self, **kwargs):
+        self.log = logging.Logger('model')
+
+        self.api_version = kwargs.get('apiVersion', '')
+
+        if self.api_version == "v1.0":
+            metadata = kwargs.get('metadata', {})
+            spec = kwargs.get('spec', {})
+
+            self.name = metadata.get('name', '')
+            self.site = metadata.get('region', '')
+
+            bonding = spec.get('bonding', {})
+            self.bonding_mode = bonding.get('mode', 'none')
+
+            # How should we define defaults for CIs not in the input?
+            if self.bonding_mode == '802.3ad':
+                self.bonding_xmit_hash = bonding.get('hash', 'layer3+4')
+                self.bonding_peer_rate = bonding.get('peer_rate', 'fast')
+                self.bonding_mon_rate = bonding.get('mon_rate', '100')
+                self.bonding_up_delay = bonding.get('up_delay', '200')
+                self.bonding_down_delay = bonding.get('down_delay', '200')
+
+            self.mtu = spec.get('mtu', 1500)
+            self.linkspeed = spec.get('linkspeed', 'auto')
+
+            trunking = spec.get('trunking', {})
+            self.trunk_mode = trunking.get('mode', 'none')
+
+            self.native_network = spec.get('default_network', '')
+        else:
+            self.log.error("Unknown API version %s of %s" %
+                (self.api_version, self.__class__))
+            raise ValueError('Unknown API version of object')
+
+
+class Network(object):
+
+    def __init__(self, **kwargs):
+        self.log = logging.Logger('model')
+
+        self.api_version = kwargs.get('apiVersion', '')
+
+        if self.api_version == "v1.0":
+            metadata = kwargs.get('metadata', {})
+            spec = kwargs.get('spec', {})
+
+            self.name = metadata.get('name', '')
+            self.site = metadata.get('region', '')
+
+            self.cidr = spec.get('cidr', None)
+            self.allocation_strategy = spec.get('allocation', 'static')
+            self.vlan_id = spec.get('vlan_id', 1)
+            self.mtu = spec.get('mtu', 0)
+
+            dns = spec.get('dns', {})
+            self.dns_domain = dns.get('domain', 'local')
+            self.dns_servers = dns.get('servers', None)
+
+            ranges = spec.get('ranges', [])
+            self.ranges = []
+
+            for r in ranges:
+                self.ranges.append(NetworkAddressRange(self.api_version, **r))
+
+            routes = spec.get('routes', [])
+            self.routes = []
+
+            for r in routes:
+                self.routes.append(NetworkRoute(self.api_version, **r))
+        else:
+            self.log.error("Unknown API version %s of %s" %
+                (self.api_version, self.__class__))
+            raise ValueError('Unknown API version of object')
+
+
+class NetworkAddressRange(object):
+
+    def __init__(self, api_version, **kwargs):
+        self.log = logging.Logger('model')
+
+        self.api_version = api_version
+
+        if self.api_version == "v1.0":
+            self.type = kwargs.get('type', None)
+            self.start = kwargs.get('start', None)
+            self.end = kwargs.get('end', None)
+        else:
+            self.log.error("Unknown API version %s of %s" %
+                (self.api_version, self.__class__))
+            raise ValueError('Unknown API version of object')
+
+
+class NetworkRoute(object):
+
+    def __init__(self, api_version, **kwargs):
+        self.log = logging.Logger('model')
+
+        self.api_version = api_version
+
+        if self.api_version == "v1.0":
+            self.type = kwargs.get('subnet', None)
+            self.start = kwargs.get('gateway', None)
+            self.end = kwargs.get('metric', 100)
+        else:
+            self.log.error("Unknown API version %s of %s" %
+                (self.api_version, self.__class__))
+            raise ValueError('Unknown API version of object')

helm_drydock/model/node.py

@@ -0,0 +1,154 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Models for helm_drydock
+#
+import logging
+
+from copy import deepcopy
+
+
+from helm_drydock.orchestrator.enum import SiteStatus
+from helm_drydock.orchestrator.enum import NodeStatus
+from helm_drydock.model.hostprofile import HostProfile
+from helm_drydock.model import Utils
+
+class BaremetalNode(HostProfile):
+
+    # A BaremetalNode is really nothing more than a physical
+    # instantiation of a HostProfile, so they both represent
+    # the same set of CIs
+    def __init__(self, **kwargs):
+        super(BaremetalNode, self).__init__(**kwargs)
+
+        if self.api_version == "v1.0":
+            self.addressing = []
+
+            spec = kwargs.get('spec', {})
+            addresses = spec.get('addressing', [])
+
+            if len(addresses) == 0:
+                raise ValueError('BaremetalNode needs at least' \
+                                 ' 1 assigned address')
+            for a in addresses:
+                assignment = {}
+                address = a.get('address', '')
+                if address == 'dhcp':
+                    assignment['type'] = 'dhcp'
+                    assignment['address'] = None
+                    assignment['network'] = a.get('network')
+                    self.addressing.append(assignment)
+                elif address != '':
+                    assignment['type'] = 'static'
+                    assignment['address'] = a.get('address')
+                    assignment['network'] = a.get('network')
+                    self.addressing.append(assignment)
+                else:
+                    self.log.error("Invalid address assignment %s on Node %s" 
+                                    % (address, self.name))
+
+        self.build = kwargs.get('build', {})
+
+    def start_build(self):
+        if self.build.get('status','') == '':
+            self.build['status'] = NodeStatus.Unknown
+
+    def apply_host_profile(self, site):
+        return self.apply_inheritance(site)
+
+    # Translate device alises to physical selectors and copy
+    # other hardware attributes into this object
+    def apply_hardware_profile(self, site):
+        self_copy = deepcopy(self)
+
+        if self.hardware_profile is None:
+            raise ValueError("Hardware profile not set")
+
+        hw_profile = site.get_hardware_profile(self.hardware_profile)
+
+        for i in self_copy.interfaces:
+            for s in i.hardware_slaves:
+                selector = hw_profile.resolve_alias("pci", s)
+                if selector is None:
+                    i.add_selector("name", address=p.device)
+                else:
+                    i.add_selector("address", address=selector['address'],
+                                   dev_type=selector['device_type'])
+
+        for p in self_copy.partitions:
+            selector = hw_profile.resolve_alias("scsi", p.device)
+            if selector is None:
+                p.set_selector("name", address=p.device)
+            else:
+                p.set_selector("address", address=selector['address'],
+                               dev_type=selector['device_type'])
+
+
+        hardware = {"vendor": getattr(hw_profile, 'vendor', None),
+                    "generation": getattr(hw_profile, 'generation', None),
+                    "hw_version": getattr(hw_profile, 'hw_version', None),
+                    "bios_version": getattr(hw_profile, 'bios_version', None),
+                    "boot_mode": getattr(hw_profile, 'boot_mode', None),
+                    "bootstrap_protocol": getattr(hw_profile, 
+                                                  'bootstrap_protocol',
+                                                   None),
+                    "pxe_interface": getattr(hw_profile, 'pxe_interface', None)
+                    }
+
+        self_copy.hardware = hardware
+
+        return self_copy
+
+    def apply_network_connections(self, site):
+        self_copy = deepcopy(self)
+
+        for n in site.network_links:
+            for i in self_copy.interfaces:
+                i.apply_link_config(n)
+
+        for n in site.networks:
+            for i in self_copy.interfaces:
+                i.apply_network_config(n)
+
+        for a in self_copy.addressing:
+            for i in self_copy.interfaces:
+                i.set_network_address(a.get('network'), a.get('address'))
+
+        return self_copy
+
+    def get_interface(self, iface_name):
+        for i in self.interfaces:
+            if i.device_name == iface_name:
+                return i
+        return None
+
+    def get_status(self):
+        return self.build['status']
+
+    def set_status(self, status):
+        if isinstance(status, NodeStatus):
+            self.build['status'] = status
+
+    def get_last_build_action(self):
+        return self.build.get('last_action', None)
+
+    def set_last_build_action(self, action, result, detail=None):
+        last_action = self.build.get('last_action', None)
+        if last_action is None:
+            self.build['last_action'] = {}
+            last_action = self.build['last_action']
+            last_action['action'] = action
+            last_action['result'] = result
+            if detail is not None:
+                last_action['detail'] = detail

helm_drydock/model/site.py

@@ -0,0 +1,122 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Models for helm_drydock
+#
+import logging
+
+from copy import deepcopy
+
+from helm_drydock.orchestrator.enum import SiteStatus
+from helm_drydock.orchestrator.enum import NodeStatus
+
+class Site(object):
+
+    def __init__(self, **kwargs):
+        self.log = logging.Logger('model')
+
+        if kwargs is None:
+            raise ValueError("Empty arguments")
+
+        self.api_version = kwargs.get('apiVersion', '')
+
+        self.build = kwargs.get('build', {})
+
+        if self.api_version == "v1.0":
+            metadata = kwargs.get('metadata', {})
+
+            # Need to add validation logic, we'll assume the input is
+            # valid for now
+            self.name = metadata.get('name', '')
+
+            spec = kwargs.get('spec', {})
+
+            self.tag_definitions = []
+            tag_defs = spec.get('tag_definitions', [])
+
+            for t in tag_defs:
+                self.tag_definitions.append(
+                    NodeTagDefinition(self.api_version, **t))
+
+            self.networks = []
+            self.network_links = []
+            self.host_profiles = []
+            self.hardware_profiles = []
+            self.baremetal_nodes = []
+
+        else:
+            self.log.error("Unknown API version %s of %s" %
+                (self.api_version, self.__class__))
+            raise ValueError('Unknown API version of object')
+
+    def start_build(self):
+        if self.build.get('status', '') == '':
+            self.build['status'] = SiteStatus.Unknown
+            
+    def get_network(self, network_name):
+        for n in self.networks:
+            if n.name == network_name:
+                return n
+
+        return None
+
+    def get_network_link(self, link_name):
+        for l in self.network_links:
+            if l.name == link_name:
+                return l
+
+        return None
+
+    def get_host_profile(self, profile_name):
+        for p in self.host_profiles:
+            if p.name == profile_name:
+                return p
+
+        return None
+
+    def get_hardware_profile(self, profile_name):
+        for p in self.hardware_profiles:
+            if p.name == profile_name:
+                return p
+
+        return None
+
+    def get_baremetal_node(self, node_name):
+        for n in self.baremetal_nodes:
+            if n.name == node_name:
+                return n
+
+        return None
+
+    def set_status(self, status):
+        if isinstance(status, SiteStatus):
+            self.build['status'] = status
+
+class NodeTagDefinition(object):
+
+    def __init__(self, api_version, **kwargs):
+        self.api_version = api_version
+
+        if self.api_version == "v1.0":
+            self.tag = kwargs.get('tag', '')
+            self.definition_type = kwargs.get('definition_type', '')
+            self.definition = kwargs.get('definition', '')
+
+            if self.definition_type not in ['lshw_xpath']:
+                raise ValueError('Unknown definition type in ' \
+                    'NodeTagDefinition: %s' % (self.definition_type))
+        else:
+            self.log.error("Unknown API version %s of %s" %
+                (self.api_version, self.__class__))
+            raise ValueError('Unknown API version of object')

helm_drydock/orchestrator/__init__.py

@@ -0,0 +1,71 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from enum import Enum, unique
+
+import uuid
+
+class Orchestrator(object):
+
+    # enabled_drivers is a map which provider drivers
+    # should be enabled for use by this orchestrator
+
+    def __init__(self, enabled_drivers=None, design_state=None):
+        self.enabled_drivers = {}
+
+        self.enabled_drivers['oob'] = enabled_drivers.get('oob', None)
+        self.enabled_drivers['server'] = enabled_drivers.get('server', None)
+        self.enabled_drivers['network'] = enabled_drivers.get('network', None)
+
+        self.design_state = design_state
+
+    """
+    execute_task
+
+    This is the core of the orchestrator. The task will describe the action
+    to take and the context/scope of the command. We will then source
+    the current designed state and current built state from the statemgmt
+    module. Based on those 3 inputs, we'll decide what is needed next.
+    """
+    def execute_task(self, task):
+        if design_state is None:
+            raise Exception("Cannot execute task without initialized state manager")
+
+
+class OrchestrationTask(object):
+
+    def __init__(self, action, **kwargs):
+        self.taskid = uuid.uuid4()
+
+        self.action = action
+
+        parent_task = kwargs.get('parent_task','')
+
+        # Validate parameters based on action
+        self.site = kwargs.get('site', '')
+
+
+        if self.site == '':
+            raise ValueError("Task requires 'site' parameter")
+
+        if action in [Action.VerifyNode, Action.PrepareNode,
+            Action.DeployNode, Action.DestroyNode]:
+            self.node_filter = kwargs.get('node_filter', None)
+
+    def child_task(self, action, **kwargs):
+        child_task = OrchestrationTask(action, parent_task=self.taskid, site=self.site, **kwargs)
+        return child_task
+
+
+
+

helm_drydock/orchestrator/designdata.py

@@ -0,0 +1,109 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+from copy import deepcopy
+
+from helm_drydock.error import DesignError
+
+class DesignStateClient(object):
+
+    def __init__(self):
+        self.log = logging.Logger('orchestrator')
+
+    """
+    load_design_data - Pull all the defined models in statemgmt and assemble
+    them into a representation of the site. Does not compute inheritance.
+    Throws an exception if multiple Site models are found.
+
+    param design_state - Instance of statemgmt.DesignState to load data from
+
+    return a Site model populated with all components from the design state
+    """
+
+    def load_design_data(self, site_name, design_state=None, change_id=None):
+        if design_state is None:
+            raise ValueError("Design state is None")
+
+        design_data = None
+
+        if change_id is None:
+            try:
+                design_data = design_state.get_design_base()
+            except DesignError(e):
+                raise e
+        else:
+            design_data = design_state.get_design_change(change_id)
+
+        site = design_data.get_site(site_name)
+
+        networks = design_data.get_networks()
+
+        for n in networks:
+            if n.site == site_name:
+                site.networks.append(n)
+
+        network_links = design_data.get_network_links()
+
+        for l in network_links:
+            if l.site == site_name:
+                site.network_links.append(l)
+
+        host_profiles = design_data.get_host_profiles()
+
+        for p in host_profiles:
+            if p.site == site_name:
+                site.host_profiles.append(p)
+
+        hardware_profiles = design_data.get_hardware_profiles()
+
+        for p in hardware_profiles:
+            if p.site == site_name:
+                site.hardware_profiles.append(p)
+
+        baremetal_nodes = design_data.get_baremetal_nodes()
+
+        for n in baremetal_nodes:
+            if n.site == site_name:
+                site.baremetal_nodes.append(n)
+
+        return site
+
+    def compute_model_inheritance(self, site_root):
+        
+        # For now the only thing that really incorporates inheritance is
+        # host profiles and baremetal nodes. So we'll just resolve it for
+        # the baremetal nodes which recursively resolves it for host profiles
+        # assigned to those nodes
+
+        site_copy = deepcopy(site_root)
+
+        effective_nodes = []
+
+        for n in site_copy.baremetal_nodes:
+            resolved = n.apply_host_profile(site_copy)
+            resolved = resolved.apply_hardware_profile(site_copy)
+            resolved = resolved.apply_network_connections(site_copy)
+            effective_nodes.append(resolved)
+
+        site_copy.baremetal_nodes = effective_nodes
+        
+        return site_copy
+    """
+    compute_model_inheritance - given a fully populated Site model,
+    compute the effecitve design by applying inheritance and references
+
+    return a Site model reflecting the effective design for the site
+    """

helm_drydock/orchestrator/enum.py

@@ -0,0 +1,64 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from enum import Enum, unique
+
+@unique
+class Action(Enum):
+    Noop = 'noop'
+    ValidateDesign = 'validate_design'
+    VerifySite = 'verify_site'
+    PrepareSite = 'prepare_site'
+    VerifyNode = 'verify_node'
+    PrepareNode = 'prepare_node'
+    DeployNode = 'deploy_node'
+    DestroyNode = 'destroy_node'
+
+@unique
+class ActionResult(Enum):
+    Success = 'success'
+    PartialSuccess = 'partial_success'
+    Failure = 'failure'
+    DependentFailure = 'dependent_failure'
+
+@unique
+class SiteStatus(Enum):
+    Unknown = 'unknown'
+    DesignStarted = 'design_started'
+    DesignAvailable = 'design_available'
+    DesignValidated = 'design_validated'
+    Deploying = 'deploying'
+    Deployed = 'deployed'
+    DesignUpdated = 'design_updated'
+
+@unique
+class NodeStatus(Enum):
+    Unknown = 'unknown'
+    Designed = 'designed'
+    Present = 'present' # IPMI access verified
+    BasicVerifying = 'basic_verifying' # Base node verification in process
+    FailedBasicVerify = 'failed_basic_verify' # Base node verification failed
+    BasicVerified = 'basic_verified' # Base node verification successful
+    Preparing = 'preparing' # Node preparation in progress
+    FailedPrepare = 'failed_prepare' # Node preparation failed
+    Prepared = 'prepared' # Node preparation complete
+    FullyVerifying = 'fully_verifying' # Node full verification in progress
+    FailedFullVerify = 'failed_full_verify' # Node full verification failed
+    FullyVerified = 'fully_verified' # Deeper verification successful
+    Deploying  = 'deploy' # Node deployment in progress
+    FailedDeploy = 'failed_deploy' # Node deployment failed
+    Deployed = 'deployed' # Node deployed successfully
+    Bootstrapping = 'bootstrapping' # Node bootstrapping
+    FailedBootstrap = 'failed_bootstrap' # Node bootstrapping failed
+    Bootstrapped = 'bootstrapped' # Node fully bootstrapped
+    Complete = 'complete' # Node is complete

helm_drydock/orchestrator/readme.md

@@ -0,0 +1,83 @@
+# Orchestrator #
+
+The orchestrator is the core of drydock and will manage
+the ordering of driver actions to implement the main Drydock
+actions. Each of these actions will be started by the
+external cLCP orchestrator with different parameters to
+control the scope of the action.
+
+Orchestrator should persist the state of each task
+such that on failure the task can retried and only the
+steps needed will be executed.
+
+Bullet points listed below are not exhaustive and will
+change as we move through testing
+
+## ValidateDesign ##
+
+Load design data from the statemgmt persistent store and
+validate that the current state of design data represents
+a valid site design. No claim is made that the design data
+is compatible with the physical state of the site.
+
+## VerifySite ##
+
+Verify site-wide resources are in a useful state
+
+* Driver downstream resources are reachable (e.g. MaaS)
+* OS images needed for bootstrapping are available
+* Promenade or other next-step services are up and available
+* Verify credentials are available
+
+## PrepareSite ##
+
+Begin preparing site-wide resources for bootstrapping. This
+action will lock site design data for changes.
+
+* Configure bootstrapper with site network configs
+* Shuffle images so they are correctly configured for bootstrapping
+
+## VerifyNode ##
+
+Verification of per-node configurations within the context
+of the current node status
+
+* Status: Present
+    * Basic hardware verification as available via OOB driver
+        - BIOS firmware
+        - PCI layout
+        - Drives
+        - Hardware alarms
+    * IPMI connectivity
+* Status: Prepared
+    - Full hardware manifest
+    - Possibly network connectivity
+    - Firmware versions
+
+## PrepareNode ##
+
+Prepare a node for bootstrapping
+
+* Configure network port for PXE
+* Configure a node for PXE boot
+* Power-cycle the node
+* Setup commissioning configuration
+    - Hardware drivers
+    - Hardware configuration (e.g. RAID)
+* Configure node networking
+* Configure node storage
+
+## DeployNode ##
+
+Begin bootstrapping the node and monitor
+success
+
+* Initialize the Introspection service for the node
+* Bootstrap the node (i.e. Write persistent OS install)
+* Ensure network port is returned to production configuration
+* Reboot node from local disk
+* Monitor platform bootstrapping
+
+## DestroyNode ##
+
+Destroy current node configuration and rebootstrap from scratch

helm_drydock/statemgmt/__init__.py

@@ -0,0 +1,294 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from copy import deepcopy
+from datetime import datetime
+from datetime import timezone
+
+import uuid
+
+import helm_drydock.model.node as node
+import helm_drydock.model.hostprofile as hostprofile
+import helm_drydock.model.network as network
+import helm_drydock.model.site as site
+import helm_drydock.model.hwprofile as hwprofile
+
+from helm_drydock.error import DesignError
+
+class DesignState(object):
+
+    def __init__(self):
+        self.design_base = None
+
+        self.design_changes = []
+
+        self.builds = []
+        return
+
+    # TODO Need to lock a design base or change once implementation
+    # has started
+    def get_design_base(self):
+        if self.design_base is None:
+            raise DesignError("No design base submitted")
+
+        return deepcopy(self.design_base)
+
+    def post_design_base(self, site_design):
+        if site_design is not None and isinstance(site_design, SiteDesign):
+            self.design_base = deepcopy(site_design)
+            return True
+
+    def put_design_base(self, site_design):
+        # TODO Support merging
+        if site_design is not None and isinstance(site_design, SiteDesign):
+            self.design_base = deepcopy(site_design)
+            return True
+
+    def get_design_change(self, changeid):
+        match = [x for x in self.design_changes if x.changeid == changeid]
+
+        if len(match) == 0:
+            raise DesignError("No design change %s found." % (changeid))
+        else:
+            return deepcopy(match[0])
+
+    def post_design_change(self, site_design):
+        if site_design is not None and isinstance(site_design, SiteDesign):
+            exists = [(x) for x
+                      in self.design_changes
+                      if x.changeid == site_design.changeid]
+            if len(exists) > 0:
+                raise DesignError("Existing change %s found" %
+                    (site_design.changeid))
+            self.design_changes.append(deepcopy(site_design))
+            return True
+        else:
+            raise DesignError("Design change must be a SiteDesign instance")
+
+    def put_design_change(self, site_design):
+        # TODO Support merging
+        if site_design is not None and isinstance(site_design, SiteDesign):
+            design_copy = deepcopy(site_design)
+            self.design_changes = [design_copy
+                                   if x.changeid == design_copy.changeid
+                                   else x
+                                   for x
+                                   in self.design_changes]
+            return True
+        else:
+            raise DesignError("Design change must be a SiteDesign instance")
+
+    def get_current_build(self):
+        latest_stamp = 0
+        current_build = None
+
+        for b in self.builds:
+            if b.build_id > latest_stamp:
+                latest_stamp = b.build_id
+                current_build = b
+
+        return deepcopy(current_build)
+
+    def get_build(self, build_id):
+        for b in self.builds:
+            if b.build_id == build_id:
+                return b
+
+        return None
+
+    def post_build(self, site_build):
+        if site_build is not None and isinstance(site_build, SiteBuild):
+            exists = [b for b in self.builds
+                      if b.build_id == site_build.build_id]
+
+        if len(exists) > 0:
+            raise DesignError("Already a site build with ID %s" %
+                (str(site_build.build_id)))
+        else:
+            self.builds.append(deepcopy(site_build))
+            return True
+
+
+class SiteDesign(object):
+
+    def __init__(self, ischange=False):
+        if ischange:
+            self.changeid = uuid.uuid4()
+        else:
+            self.changeid = 0
+
+        self.sites = []
+        self.networks = []
+        self.network_links = []
+        self.host_profiles = []
+        self.hardware_profiles = []
+        self.baremetal_nodes = []
+
+    def add_site(self, new_site):
+        if new_site is None or not isinstance(new_site, site.Site):
+            raise DesignError("Invalid Site model")
+
+        self.sites.append(new_site)
+
+    def get_sites(self):
+        return self.sites
+
+    def get_site(self, site_name):
+        for s in self.sites:
+            if s.name == site_name:
+                return s
+
+        raise DesignError("Site %s not found in design state" % site_name)
+
+    def add_network(self, new_network):
+        if new_network is None or not isinstance(new_network, network.Network):
+            raise DesignError("Invalid Network model")
+
+        self.networks.append(new_network)
+
+    def get_networks(self):
+        return self.networks
+
+    def get_network(self, network_name):
+        for n in self.networks:
+            if n.name == network_name:
+                return n
+
+        raise DesignError("Network %s not found in design state"
+                          % network_name)
+
+    def add_network_link(self, new_network_link):
+        if new_network_link is None or not isinstance(new_network_link,
+                                                      network.NetworkLink):
+            raise DesignError("Invalid NetworkLink model")
+
+        self.network_links.append(new_network_link)
+
+    def get_network_links(self):
+        return self.network_links
+
+    def get_network_link(self, link_name):
+        for l in self.network_links:
+            if l.name == link_name:
+                return l
+
+        raise DesignError("NetworkLink %s not found in design state"
+                          % link_name)
+
+    def add_host_profile(self, new_host_profile):
+        if new_host_profile is None or not isinstance(new_host_profile,
+                                                      hostprofile.HostProfile):
+            raise DesignError("Invalid HostProfile model")
+
+        self.host_profiles.append(new_host_profile)
+
+    def get_host_profiles(self):
+        return self.host_profiles
+
+    def get_host_profile(self, profile_name):
+        for p in self.host_profiles:
+            if p.name == profile_name:
+                return p
+
+        raise DesignError("HostProfile %s not found in design state"
+                          % profile_name)
+
+    def add_hardware_profile(self, new_hardware_profile):
+        if (new_hardware_profile is None or
+            not isinstance(new_hardware_profile, hwprofile.HardwareProfile)):
+            raise DesignError("Invalid HardwareProfile model")
+
+        self.hardware_profiles.append(new_hardware_profile)
+
+    def get_hardware_profiles(self):
+        return self.hardware_profiles
+
+    def get_hardware_profile(self, profile_name):
+        for p in self.hardware_profiles:
+            if p.name == profile_name:
+                return p
+
+        raise DesignError("HardwareProfile %s not found in design state"
+                          % profile_name)
+
+    def add_baremetal_node(self, new_baremetal_node):
+        if (new_baremetal_node is None or
+            not isinstance(new_baremetal_node, node.BaremetalNode)):
+            raise DesignError("Invalid BaremetalNode model")
+
+        self.baremetal_nodes.append(new_baremetal_node)
+
+    def get_baremetal_nodes(self):
+        return self.baremetal_nodes
+
+    def get_baremetal_node(self, node_name):
+        for n in self.baremetal_nodes:
+            if n.name == node_name:
+                return n
+
+        raise DesignError("BaremetalNode %s not found in design state"
+                          % node_name)
+
+
+class SiteBuild(SiteDesign):
+
+    def __init__(self, build_id=None):
+        super(SiteBuild, self).__init__()
+
+        if build_id is None:
+            self.build_id = datetime.datetime.now(timezone.utc).timestamp()
+        else:
+            self.build_id = build_id
+
+    def get_filtered_nodes(self, node_filter):
+        effective_nodes = self.get_baremetal_nodes()
+
+        # filter by rack
+        rack_filter = node_filter.get('rackname', None)
+
+        if rack_filter is not None:
+            rack_list = rack_filter.split(',')
+            effective_nodes = [x 
+                               for x in effective_nodes
+                               if x.get_rack() in rack_list]
+        # filter by name
+        name_filter = node_filter.get('nodename', None)
+
+        if name_filter is not None:
+            name_list = name_filter.split(',')
+            effective_nodes = [x
+                               for x in effective_nodes
+                               if x.get_name() in name_list]
+        # filter by tag
+        tag_filter = node_filter.get('tags', None)
+
+        if tag_filter is not None:
+            tag_list = tag_filter.split(',')
+            effective_nodes = [x
+                               for x in effective_nodes
+                               for t in tag_list
+                               if x.has_tag(t)]
+
+        return effective_nodes
+    """
+    Support filtering on rack name, node name or node tag
+    for now. Each filter can be a comma-delimited list of
+    values. The final result is an intersection of all the
+    filters
+    """
+
+    def set_nodes_status(self, node_filter, status):
+        target_nodes = self.get_filtered_nodes(node_filter)
+
+        for n in target_nodes:
+            n.set_status(status)

helm_drydock/statemgmt/readme.md

@@ -0,0 +1,41 @@
+# Statemgmt #
+
+Statemgmt is the interface to the persistence store
+for holding site design data as well as build status data
+
+/drydock - Base namespace for drydock data
+
+## As Designed ##
+
+Serialization of Drydock internal model as ingested. Not externally writable.
+
+/drydock/design
+/drydock/design/base - The base site design used for the first deployment
+/drydock/design/[changeID] - A change layer on top of the base site design. Chrono ordered
+
+## As Built ##
+
+Serialization of Drydock internal model as rendered to effective implementation including build status. Not externally writable.
+
+/drydock/build
+/drydock/build/[datestamp] - A point-in-time view of what was deployed with deployment results
+
+## Node data ##
+
+Per-node data that can drive introspection as well as accept updates from nodes
+
+/drydock/nodedata
+/drydock/nodedata/[nodename] - Per-node data, can be seeded with Ingested metadata but can be updated during by orchestrator or node
+
+## Service data ##
+
+Per-service data (may not be needed)
+
+/drydock/servicedata
+/drydock/servicedata/[servicename]
+
+## Global data ##
+
+Generic k:v store that can be produced/consumed by non-Drydock services via the Drydock API
+
+/drydock/globaldata

requirements.txt

@@ -0,0 +1,5 @@
+PyYAML==3.12
+oauth==1.0.1
+requests-oauthlib==0.8.0
+netaddr==0.7.19
+python-libmaas==0.4.1

setup.py

@@ -43,16 +43,20 @@ setup(name='helm_drydock',
                 'helm_drydock.model',
                 'helm_drydock.ingester',
                 'helm_drydock.ingester.plugins',
-                'helm_drydock.statemgmt'],
+                'helm_drydock.statemgmt',
+                'helm_drydock.orchestrator',
+                'helm_drydock.control'],
       install_requires=[
         'PyYAML',
         'oauth',
         'requests-oauthlib',
-        'pyipmi',
+        'pyghmi',
         'netaddr',
-        'pecan'
+        'pecan',
+        'webob'
       ],
       dependency_link=[
         'git+https://github.com/maas/python-libmaas.git'
       ]
      )
+

testrequirements.txt

@@ -0,0 +1,2 @@
+pytest
+tox

tests/test_design_inheritance.py

@@ -0,0 +1,85 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from helm_drydock.ingester import Ingester
+from helm_drydock.statemgmt import DesignState, SiteDesign
+from helm_drydock.orchestrator.designdata import DesignStateClient
+
+from copy import deepcopy
+
+import pytest
+import shutil
+import os
+import helm_drydock.ingester.plugins.yaml
+import yaml
+
+class TestClass(object):
+
+    def setup_method(self, method):
+        print("Running test {0}".format(method.__name__))
+
+
+    def test_design_inheritance(self, loaded_design):
+        client = DesignStateClient()
+
+        design_data = client.load_design_data("sitename", design_state=loaded_design)
+
+        assert len(design_data.baremetal_nodes) == 2
+
+        print(yaml.dump(design_data, default_flow_style=False))
+
+        design_data = client.compute_model_inheritance(design_data)
+
+        node = design_data.get_baremetal_node("controller01")
+
+        print(yaml.dump(node, default_flow_style=False))
+        
+        assert node.hardware_profile == 'HPGen9v3'
+
+        iface = node.get_interface('bond0')
+
+        assert iface.get_slave_count() == 2
+
+        iface = node.get_interface('pxe')
+
+        assert iface.get_slave_count() == 1
+
+    @pytest.fixture(scope='module')
+    def loaded_design(self, input_files):
+        input_file = input_files.join("fullsite.yaml")
+
+        design_state = DesignState()
+        design_data = SiteDesign()
+        design_state.post_design_base(design_data)
+
+        ingester = Ingester()
+        ingester.enable_plugins([helm_drydock.ingester.plugins.yaml.YamlIngester])
+        ingester.ingest_data(plugin_name='yaml', design_state=design_state, filenames=[str(input_file)])
+
+        return design_state
+
+        
+
+    @pytest.fixture(scope='module')
+    def input_files(self, tmpdir_factory, request):
+        tmpdir = tmpdir_factory.mktemp('data')
+        samples_dir = os.path.dirname(str(request.fspath)) + "/yaml_samples"
+        samples = os.listdir(samples_dir)
+
+        for f in samples:
+            src_file = samples_dir + "/" + f
+            dst_file = str(tmpdir) + "/" + f
+            shutil.copyfile(src_file, dst_file)
+
+        return tmpdir

tests/test_ingester.py

@@ -0,0 +1,73 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from helm_drydock.ingester import Ingester
+from helm_drydock.statemgmt import DesignState, SiteDesign
+
+import pytest
+import shutil
+import os
+import helm_drydock.ingester.plugins.yaml
+
+class TestClass(object):
+
+    def setup_method(self, method):
+        print("Running test {0}".format(method.__name__))
+
+    def test_ingest_full_site(self, input_files):
+        input_file = input_files.join("fullsite.yaml")
+
+        design_state = DesignState()
+        design_data = SiteDesign()
+        design_state.post_design_base(design_data)
+
+        ingester = Ingester()
+        ingester.enable_plugins([helm_drydock.ingester.plugins.yaml.YamlIngester])
+        ingester.ingest_data(plugin_name='yaml', design_state=design_state, filenames=[str(input_file)])
+
+        design_data = design_state.get_design_base()
+
+        assert len(design_data.get_host_profiles()) == 3
+        assert len(design_data.get_baremetal_nodes()) == 2
+
+    def test_ingest_federated_design(self, input_files):
+        profiles_file = input_files.join("fullsite_profiles.yaml")
+        networks_file = input_files.join("fullsite_networks.yaml")
+        nodes_file = input_files.join("fullsite_nodes.yaml")
+
+        design_state = DesignState()
+        design_data = SiteDesign()
+        design_state.post_design_base(design_data)
+
+        ingester = Ingester()
+        ingester.enable_plugins([helm_drydock.ingester.plugins.yaml.YamlIngester])
+        ingester.ingest_data(plugin_name='yaml', design_state=design_state,
+            filenames=[str(profiles_file), str(networks_file), str(nodes_file)])
+
+        design_data = design_state.get_design_base()
+
+        assert len(design_data.host_profiles) == 3
+
+    @pytest.fixture(scope='module')
+    def input_files(self, tmpdir_factory, request):
+        tmpdir = tmpdir_factory.mktemp('data')
+        samples_dir = os.path.dirname(str(request.fspath)) + "/yaml_samples"
+        samples = os.listdir(samples_dir)
+
+        for f in samples:
+            src_file = samples_dir + "/" + f
+            dst_file = str(tmpdir) + "/" + f
+            shutil.copyfile(src_file, dst_file)
+
+        return tmpdir

tests/test_ingester_yaml.py

@@ -0,0 +1,54 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from helm_drydock.ingester.plugins.yaml import YamlIngester
+import pytest
+import shutil
+import os
+
+class TestClass(object):
+
+    def setup_method(self, method):
+        print("Running test {0}".format(method.__name__))
+
+    def test_ingest_singledoc(self, input_files):
+        input_file = input_files.join("singledoc.yaml")
+
+        ingester = YamlIngester()
+
+        models = ingester.ingest_data(filenames=[str(input_file)])
+
+        assert len(models) == 1
+
+    def test_ingest_multidoc(self, input_files):
+        input_file = input_files.join("multidoc.yaml")
+ 
+        ingester = YamlIngester()
+
+        models = ingester.ingest_data(filenames=[str(input_file)])
+
+        assert len(models) == 3
+
+    @pytest.fixture(scope='module')
+    def input_files(self, tmpdir_factory, request):
+        tmpdir = tmpdir_factory.mktemp('data')
+        samples_dir = os.path.dirname(str(request.fspath)) + "/yaml_samples"
+        samples = os.listdir(samples_dir)
+
+        for f in samples:
+            src_file = samples_dir + "/" + f
+            dst_file = str(tmpdir) + "/" + f
+            shutil.copyfile(src_file, dst_file)
+
+        return tmpdir

tests/test_models.py

@@ -0,0 +1,69 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+import yaml
+from helm_drydock.model.hwprofile import HardwareProfile
+
+class TestClass(object):
+
+    def setup_method(self, method):
+        print("Running test {0}".format(method.__name__))
+
+    def test_hardwareprofile(self):
+        yaml_snippet = ("---\n"
+                        "apiVersion: 'v1.0'\n"
+                        "kind: HardwareProfile\n"
+                        "metadata:\n"
+                        "  name: HPGen8v3\n"
+                        "  region: sitename\n"
+                        "  date: 17-FEB-2017\n"
+                        "  name: Sample hardware definition\n"
+                        "  author: Scott Hussey\n"
+                        "spec:\n"
+                        "  # Vendor of the server chassis\n"
+                        "  vendor: HP\n"
+                        "  # Generation of the chassis model\n"
+                        "  generation: '8'\n"
+                        "  # Version of the chassis model within its generation - not version of the hardware definition\n"
+                        "  hw_version: '3'\n"
+                        "  # The certified version of the chassis BIOS\n"
+                        "  bios_version: '2.2.3'\n"
+                        "  # Mode of the default boot of hardware - bios, uefi\n"
+                        "  boot_mode: bios\n"
+                        "  # Protocol of boot of the hardware - pxe, usb, hdd\n"
+                        "  bootstrap_protocol: pxe\n"
+                        "  # Which interface to use for network booting within the OOB manager, not OS device\n"
+                        "  pxe_interface: 0\n"
+                        "  # Map hardware addresses to aliases/roles to allow a mix of hardware configs\n"
+                        "  # in a site to result in a consistent configuration\n"
+                        "  device_aliases:\n"
+                        "    pci:\n"
+                        "    - address: pci@0000:00:03.0\n"
+                        "      alias: prim_nic01\n"
+                        "      # type could identify expected hardware - used for hardware manifest validation\n"
+                        "      type: '82540EM Gigabit Ethernet Controller'\n"
+                        "    - address: pci@0000:00:04.0\n"
+                        "      alias: prim_nic02\n"
+                        "      type: '82540EM Gigabit Ethernet Controller'\n"
+                        "    scsi:\n"
+                        "    - address: scsi@2:0.0.0\n"
+                        "      alias: primary_boot\n"
+                        "      type: 'VBOX HARDDISK'\n")
+
+        hw_profile = yaml.load(yaml_snippet)
+        hw_profile_model = HardwareProfile(**hw_profile)
+
+        assert hasattr(hw_profile_model, 'bootstrap_protocol')
+

tests/yaml_samples/fullsite.yaml

@@ -0,0 +1,456 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+####################
+#
+# bootstrap_seed.yaml - Site server design definition for physical layer
+#
+####################
+# version the schema in this file so consumers can rationally parse it
+---
+apiVersion: 'v1.0'
+kind: Region
+metadata:
+  name: sitename
+  date: 17-FEB-2017
+  description: Sample site design
+  author: sh8121@att.com
+# Not sure if we have site wide data that doesn't fall into another 'Kind'
+---
+apiVersion: 'v1.0'
+kind: NetworkLink
+metadata:
+  name: oob
+  region: sitename
+  date: 17-FEB-2017
+  author: sh8121@att.com
+  description: Describe layer 1 attributes. Primary key is 'name'. These settings will generally be things the switch and server have to agree on
+spec:
+  bonding:
+    mode: none
+  mtu: 1500
+  linkspeed: 100full
+  trunking:
+    mode: none
+    default_network: oob
+---
+# pxe is a bit of 'magic' indicating the link config used when PXE booting
+# a node. All other links indicate network configs applied when the node
+# is deployed.
+apiVersion: 'v1.0'
+kind: NetworkLink
+metadata:
+  name: pxe
+  region: sitename
+  date: 17-FEB-2017
+  author: sh8121@att.com
+  description: Describe layer 1 attributes. Primary key is 'name'. These settings will generally be things the switch and server have to agree on
+spec:
+  bonding:
+    mode: none
+  mtu: 1500
+  linkspeed: auto
+  # Is this link supporting multiple layer 2 networks?
+  # none is a port-based VLAN identified by default_network
+  # tagged is is using 802.1q VLAN tagging. Untagged packets will default to default_netwokr
+  trunking:
+    mode: none
+    # use name, will translate to VLAN ID
+    default_network: pxe
+---
+apiVersion: 'v1.0'
+kind: NetworkLink
+metadata:
+  name: gp
+  region: sitename
+  date: 17-FEB-2017
+  author: sh8121@att.com
+  description: Describe layer 1 attributes. These CIs will generally be things the switch and server have to agree on
+  # pxe is a bit of 'magic' indicating the link config used when PXE booting
+  # a node. All other links indicate network configs applied when the node
+  # is deployed.
+spec:
+  # If this link is a bond of physical links, how is it configured
+  # 802.3ad
+  # active-backup
+  # balance-rr 
+  # Can add support for others down the road
+  bonding:
+    mode: 802.3ad
+    # For LACP (802.3ad) xmit hashing policy: layer2, layer2+3, layer3+4, encap3+4
+    hash: layer3+4
+    # 802.3ad specific options
+    peer_rate: slow
+    mon_rate: default
+    up_delay: default
+    down_delay: default
+  mtu: 9000
+  linkspeed: auto
+  # Is this link supporting multiple layer 2 networks?
+  trunking:
+    mode: tagged
+    default_network: mgmt
+---
+apiVersion: 'v1.0'
+kind: Network
+metadata:
+  name: oob
+  region: sitename
+  date: 17-FEB-2017
+  author: sh8121@att.com
+  description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
+spec:
+  allocation: static
+  cidr: 172.16.100.0/24
+  ranges:
+  - type: static
+    start: 172.16.100.15
+    end: 172.16.100.254
+  dns:
+     domain: ilo.sitename.att.com
+     servers: 172.16.100.10
+---
+apiVersion: 'v1.0'
+kind: Network
+metadata:
+  name: pxe
+  region: sitename
+  date: 17-FEB-2017
+  author: sh8121@att.com
+  description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces  
+spec:
+    # Layer 2 VLAN segment id, could support other segmentations. Optional
+  vlan_id: '99'
+    # How are addresses assigned?
+  allocation: dhcp
+    # MTU for this VLAN interface, if not specified it will be inherited from the link
+  mtu: 1500
+    # Network address
+  cidr: 172.16.0.0/24
+    # Desribe IP address ranges
+  ranges: 
+  - type: dhcp
+    start: 172.16.0.5
+    end: 172.16.0.254
+    # DNS settings for this network
+  dns:
+      # Domain addresses on this network will be registered under
+    domain: admin.sitename.att.com
+      # DNS servers that a server using this network as its default gateway should use
+    servers: 172.16.0.10
+---
+apiVersion: 'v1.0'
+kind: Network
+metadata:
+  name: mgmt
+  region: sitename
+  date: 17-FEB-2017
+  author: sh8121@att.com
+  description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
+spec:
+  vlan_id: '100'
+    # How are addresses assigned?
+  allocation: static
+    # Allow MTU to be inherited from link the network rides on
+  mtu: 1500
+    # Network address
+  cidr: 172.16.1.0/24
+    # Desribe IP address ranges
+  ranges: 
+  - type: static
+    start: 172.16.1.15
+    end: 172.16.1.254
+    # Static routes to be added for this network
+  routes:
+  - subnet: 0.0.0.0/0
+        # A blank gateway would leave to a static route specifying
+        # only the interface as a source
+    gateway: 172.16.1.1
+    metric: 10
+    # DNS settings for this network
+  dns:
+      # Domain addresses on this network will be registered under
+    domain: mgmt.sitename.example.com
+      # DNS servers that a server using this network as its default gateway should use
+    servers: 172.16.1.9,172.16.1.10
+---
+apiVersion: 'v1.0'
+kind: Network
+metadata:
+  name: private
+  region: sitename
+  date: 17-FEB-2017
+  author: sh8121@att.com
+  description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
+spec:
+  vlan_id: '101'
+  allocation: static
+  mtu: 9000
+  cidr: 172.16.2.0/24
+    # Desribe IP address ranges
+  ranges: 
+      # Type can be reserved (not used for baremetal), static (all explicit
+      # assignments should fall here), dhcp (will be used by a DHCP server on this network)
+  - type: static
+    start: 172.16.2.15
+    end: 172.16.2.254
+  dns:
+    domain: priv.sitename.example.com
+    servers: 172.16.2.9,172.16.2.10
+---
+apiVersion: 'v1.0'
+kind: Network
+metadata:
+  name: public
+  region: sitename
+  date: 17-FEB-2017
+  author: sh8121@att.com
+  description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
+spec:
+  vlan_id: '102'
+    # How are addresses assigned?
+  allocation: static
+    # MTU size for the VLAN interface
+  mtu: 1500
+  cidr: 172.16.3.0/24
+    # Desribe IP address ranges
+  ranges: 
+  - type: static
+    start: 172.16.3.15
+    end: 172.16.3.254
+  routes:
+  - subnet: 0.0.0.0/0
+    gateway: 172.16.3.1
+    metric: 9
+  dns:
+    domain: sitename.example.com
+    servers: 8.8.8.8
+---
+apiVersion: 'v1.0'
+kind: HostProfile
+metadata:
+  name: defaults
+  region: sitename
+  date: 17-FEB-2017
+  author: sh8121@att.com
+  description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
+  # No magic to this host_profile, it just provides a way to specify
+  # sitewide settings. If it is absent from a node's inheritance chain
+  # then these values will NOT be applied
+spec:
+  # OOB (iLO, iDRAC, etc...) settings. Should prefer open standards such
+  # as IPMI over vender-specific when possible.
+  oob:
+    type: ipmi
+    # OOB networking should be preconfigured, but we can include a network
+    # definition for validation or enhancement (DNS registration)
+    network: oob
+    account: admin
+    credential: admin
+  # Specify storage layout of base OS. Ceph out of scope
+  storage:
+    # How storage should be carved up: lvm (logical volumes), flat 
+    # (single partition)
+    layout: lvm
+    # Info specific to the boot and root disk/partitions
+    bootdisk:
+      # Device will specify an alias defined in hwdefinition.yaml
+      device: primary_boot
+      # For LVM, the size of the partition added to VG as a PV
+      # For flat, the size of the partition formatted as ext4
+      root_size: 50g
+      # The /boot partition. If not specified, /boot will in root
+      boot_size: 2g
+    # Info for additional partitions. Need to balance between
+    # flexibility and complexity
+    partitions:
+    - name: logs
+      device: primary_boot
+      # Partition uuid if needed
+      part_uuid: 84db9664-f45e-11e6-823d-080027ef795a
+      size: 10g
+      # Optional, can carve up unformatted block devices
+      mountpoint: /var/log
+      fstype: ext4
+      mount_options: defaults
+      # Filesystem UUID or label can be specified. UUID recommended
+      fs_uuid: cdb74f1c-9e50-4e51-be1d-068b0e9ff69e
+      fs_label: logs
+  # Platform (Operating System) settings
+  platform:
+    image: ubuntu_16.04_hwe
+    kernel_params: default
+  # Additional metadata to apply to a node
+  metadata:
+    # Base URL of the introspection service - may go in curtin data
+    introspection_url: http://172.16.1.10:9090
+---
+apiVersion: 'v1.0'
+kind: HostProfile
+metadata:
+  name: k8-node
+  region: sitename
+  date: 17-FEB-2017
+  author: sh8121@att.com
+  description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
+spec:
+  # host_profile inheritance allows for deduplication of common CIs
+  # Inheritance is additive for CIs that are lists of multiple items
+  # To remove an inherited list member, prefix the primary key value
+  # with '!'. 
+  host_profile: defaults
+  # Hardware profile will map hardware specific details to the abstract
+  # names uses in the host profile as well as specify hardware specific
+  # configs. A viable model should be to build a host profile without a
+  # hardware_profile and then for each node inherit the host profile and
+  # specify a hardware_profile to map that node's hardware to the abstract
+  # settings of the host_profile
+  hardware_profile: HPGen9v3
+  # Network interfaces.
+  interfaces:
+  # Keyed on device_name
+  # pxe is a special marker indicating which device should be used for pxe boot
+  - device_name: pxe
+    # The network link attached to this 
+    network_link: pxe
+    # Slaves will specify aliases from hwdefinition.yaml
+    slaves:
+    - prim_nic01
+    # Which networks will be configured on this interface
+    networks: 
+    - pxe
+  - device_name: bond0
+    network_link: gp
+    # If multiple slaves are specified, but no bonding config
+    # is applied to the link, design validation will fail
+    slaves:
+    - prim_nic01
+    - prim_nic02
+    # If multiple networks are specified, but no trunking
+    # config is applied to the link, design validation will fail
+    networks:
+    - mgmt
+    - private
+  metadata:
+    # Explicit tag assignment
+    tags:
+    - 'test'
+    # MaaS supports key/value pairs. Not sure of the use yet
+    owner_data:
+      foo: bar
+---
+apiVersion: 'v1.0'
+kind: HostProfile
+metadata:
+  name: k8-node-public
+  region: sitename
+  date: 17-FEB-2017
+  author: sh8121@att.com
+  description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
+spec:
+  host_profile: k8-node
+  interfaces:
+    - device_name: bond0
+      networks:
+        # This is additive, so adds a network to those defined in the host_profile
+        # inheritance chain
+      - public
+---
+apiVersion: 'v1.0'
+kind: BaremetalNode
+metadata:
+  name: controller01
+  region: sitename
+  date: 17-FEB-2017
+  author: sh8121@att.com
+  description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
+spec:
+  host_profile: k8-node-public
+  # the hostname for a server, could be used in multiple DNS domains to
+  # represent different interfaces
+  interfaces:
+  - device_name: bond0
+    networks:
+    # '!' prefix for the value of the primary key indicates a record should be removed
+    - '!private'
+  # Addresses assigned to network interfaces
+  addressing:
+    # Which network the address applies to. If a network appears in addressing
+    # that isn't assigned to an interface, design validation will fail
+  - network: pxe
+    # The address assigned. Either a explicit IPv4 or IPv6 address
+    # or dhcp or slaac
+    address: dhcp
+  - network: mgmt
+    address: 172.16.1.20
+  - network: public
+    address: 172.16.3.20
+  metadata:
+    roles: os_ctl
+    rack: rack01
+---
+apiVersion: 'v1.0'
+kind: BaremetalNode
+metadata:
+  name: compute01
+  region: sitename
+  date: 17-FEB-2017
+  author: sh8121@att.com
+  description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
+spec:
+  host_profile: k8-node
+  addressing:
+  - network: pxe
+    address: dhcp
+  - network: mgmt
+    address: 172.16.1.21
+  - network: private
+    address: 172.16.2.21
+---
+apiVersion: 'v1.0'
+kind: HardwareProfile
+metadata:
+  name: HPGen9v3
+  region: sitename
+  date: 17-FEB-2017
+  author: Scott Hussey
+spec:
+  # Vendor of the server chassis
+  vendor: HP
+  # Generation of the chassis model
+  generation: '8'
+  # Version of the chassis model within its generation - not version of the hardware definition
+  hw_version: '3'
+  # The certified version of the chassis BIOS
+  bios_version: '2.2.3'
+  # Mode of the default boot of hardware - bios, uefi
+  boot_mode: bios
+  # Protocol of boot of the hardware - pxe, usb, hdd
+  bootstrap_protocol: pxe
+  # Which interface to use for network booting within the OOB manager, not OS device
+  pxe_interface: 0
+  # Map hardware addresses to aliases/roles to allow a mix of hardware configs
+  # in a site to result in a consistent configuration
+  device_aliases:
+    pci:
+    - address: pci@0000:00:03.0
+      alias: prim_nic01
+      # type could identify expected hardware - used for hardware manifest validation
+      type: '82540EM Gigabit Ethernet Controller'
+    - address: pci@0000:00:04.0
+      alias: prim_nic02
+      type: '82540EM Gigabit Ethernet Controller'
+    scsi:
+    - address: scsi@2:0.0.0
+      alias: primary_boot
+      type: 'VBOX HARDDISK'

tests/yaml_samples/fullsite_networks.yaml

@@ -0,0 +1,228 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+####################
+#
+# bootstrap_seed.yaml - Site server design definition for physical layer
+#
+####################
+# version the schema in this file so consumers can rationally parse it
+---
+apiVersion: 'v1.0'
+kind: NetworkLink
+metadata:
+  name: oob
+  region: sitename
+  date: 17-FEB-2017
+  author: sh8121@att.com
+  description: Describe layer 1 attributes. Primary key is 'name'. These settings will generally be things the switch and server have to agree on
+spec:
+  bonding:
+    mode: none
+  mtu: 1500
+  linkspeed: 100full
+  trunking:
+    mode: none
+    default_network: oob
+---
+# pxe is a bit of 'magic' indicating the link config used when PXE booting
+# a node. All other links indicate network configs applied when the node
+# is deployed.
+apiVersion: 'v1.0'
+kind: NetworkLink
+metadata:
+  name: pxe
+  region: sitename
+  date: 17-FEB-2017
+  author: sh8121@att.com
+  description: Describe layer 1 attributes. Primary key is 'name'. These settings will generally be things the switch and server have to agree on
+spec:
+  bonding:
+    mode: none
+  mtu: 1500
+  linkspeed: auto
+  # Is this link supporting multiple layer 2 networks?
+  # none is a port-based VLAN identified by default_network
+  # tagged is is using 802.1q VLAN tagging. Untagged packets will default to default_netwokr
+  trunking:
+    mode: none
+    # use name, will translate to VLAN ID
+    default_network: pxe
+---
+apiVersion: 'v1.0'
+kind: NetworkLink
+metadata:
+  name: gp
+  region: sitename
+  date: 17-FEB-2017
+  author: sh8121@att.com
+  description: Describe layer 1 attributes. These CIs will generally be things the switch and server have to agree on
+  # pxe is a bit of 'magic' indicating the link config used when PXE booting
+  # a node. All other links indicate network configs applied when the node
+  # is deployed.
+spec:
+  # If this link is a bond of physical links, how is it configured
+  # 802.3ad
+  # active-backup
+  # balance-rr 
+  # Can add support for others down the road
+  bonding:
+    mode: 802.3ad
+    # For LACP (802.3ad) xmit hashing policy: layer2, layer2+3, layer3+4, encap3+4
+    hash: layer3+4
+    # 802.3ad specific options
+    peer_rate: slow
+    mon_rate: default
+    up_delay: default
+    down_delay: default
+  mtu: 9000
+  linkspeed: auto
+  # Is this link supporting multiple layer 2 networks?
+  trunking:
+    mode: tagged
+    default_network: mgmt
+---
+apiVersion: 'v1.0'
+kind: Network
+metadata:
+  name: oob
+  region: sitename
+  date: 17-FEB-2017
+  author: sh8121@att.com
+  description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
+spec:
+  allocation: static
+  cidr: 172.16.100.0/24
+  ranges:
+  - type: static
+    start: 172.16.100.15
+    end: 172.16.100.254
+  dns:
+     domain: ilo.sitename.att.com
+     servers: 172.16.100.10
+---
+apiVersion: 'v1.0'
+kind: Network
+metadata:
+  name: pxe
+  region: sitename
+  date: 17-FEB-2017
+  author: sh8121@att.com
+  description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces  
+spec:
+    # Layer 2 VLAN segment id, could support other segmentations. Optional
+  vlan_id: '99'
+    # How are addresses assigned?
+  allocation: dhcp
+    # MTU for this VLAN interface, if not specified it will be inherited from the link
+  mtu: 1500
+    # Network address
+  cidr: 172.16.0.0/24
+    # Desribe IP address ranges
+  ranges: 
+  - type: dhcp
+    start: 172.16.0.5
+    end: 172.16.0.254
+    # DNS settings for this network
+  dns:
+      # Domain addresses on this network will be registered under
+    domain: admin.sitename.att.com
+      # DNS servers that a server using this network as its default gateway should use
+    servers: 172.16.0.10
+---
+apiVersion: 'v1.0'
+kind: Network
+metadata:
+  name: mgmt
+  region: sitename
+  date: 17-FEB-2017
+  author: sh8121@att.com
+  description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
+spec:
+  vlan_id: '100'
+    # How are addresses assigned?
+  allocation: static
+    # Allow MTU to be inherited from link the network rides on
+  mtu: 1500
+    # Network address
+  cidr: 172.16.1.0/24
+    # Desribe IP address ranges
+  ranges: 
+  - type: static
+    start: 172.16.1.15
+    end: 172.16.1.254
+    # Static routes to be added for this network
+  routes:
+  - subnet: 0.0.0.0/0
+        # A blank gateway would leave to a static route specifying
+        # only the interface as a source
+    gateway: 172.16.1.1
+    metric: 10
+    # DNS settings for this network
+  dns:
+      # Domain addresses on this network will be registered under
+    domain: mgmt.sitename.example.com
+      # DNS servers that a server using this network as its default gateway should use
+    servers: 172.16.1.9,172.16.1.10
+---
+apiVersion: 'v1.0'
+kind: Network
+metadata:
+  name: private
+  region: sitename
+  date: 17-FEB-2017
+  author: sh8121@att.com
+  description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
+spec:
+  vlan_id: '101'
+  allocation: static
+  mtu: 9000
+  cidr: 172.16.2.0/24
+    # Desribe IP address ranges
+  ranges: 
+      # Type can be reserved (not used for baremetal), static (all explicit
+      # assignments should fall here), dhcp (will be used by a DHCP server on this network)
+  - type: static
+    start: 172.16.2.15
+    end: 172.16.2.254
+  dns:
+    domain: priv.sitename.example.com
+    servers: 172.16.2.9,172.16.2.10
+---
+apiVersion: 'v1.0'
+kind: Network
+metadata:
+  name: public
+  region: sitename
+  date: 17-FEB-2017
+  author: sh8121@att.com
+  description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
+spec:
+  vlan_id: '102'
+    # How are addresses assigned?
+  allocation: static
+    # MTU size for the VLAN interface
+  mtu: 1500
+  cidr: 172.16.3.0/24
+    # Desribe IP address ranges
+  ranges: 
+  - type: static
+    start: 172.16.3.15
+    end: 172.16.3.254
+  routes:
+  - subnet: 0.0.0.0/0
+    gateway: 172.16.3.1
+    metric: 9
+  dns:
+    domain: sitename.example.com
+    servers: 8.8.8.8

tests/yaml_samples/fullsite_nodes.yaml

@@ -0,0 +1,70 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+####################
+#
+# bootstrap_seed.yaml - Site server design definition for physical layer
+#
+####################
+# version the schema in this file so consumers can rationally parse it
+---
+apiVersion: 'v1.0'
+kind: BaremetalNode
+metadata:
+  name: controller01
+  region: sitename
+  date: 17-FEB-2017
+  author: sh8121@att.com
+  description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
+spec:
+  host_profile: k8-node-public
+  # the hostname for a server, could be used in multiple DNS domains to
+  # represent different interfaces
+  interfaces:
+  - device_name: bond0
+    networks:
+    # '!' prefix for the value of the primary key indicates a record should be removed
+    - '!private'
+  # Addresses assigned to network interfaces
+  addressing:
+    # Which network the address applies to. If a network appears in addressing
+    # that isn't assigned to an interface, design validation will fail
+  - network: pxe
+    # The address assigned. Either a explicit IPv4 or IPv6 address
+    # or dhcp or slaac
+    address: dhcp
+  - network: mgmt
+    address: 172.16.1.20
+  - network: public
+    address: 172.16.3.20
+  metadata:
+    roles: os_ctl
+    rack: rack01
+---
+apiVersion: 'v1.0'
+kind: BaremetalNode
+metadata:
+  name: compute01
+  region: sitename
+  date: 17-FEB-2017
+  author: sh8121@att.com
+  description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
+spec:
+  host_profile: k8-node
+  addressing:
+  - network: pxe
+    address: dhcp
+  - network: mgmt
+    address: 172.16.1.21
+  - network: private
+    address: 172.16.2.21

tests/yaml_samples/fullsite_profiles.yaml

@@ -0,0 +1,196 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+####################
+#
+# bootstrap_seed.yaml - Site server design definition for physical layer
+#
+####################
+# version the schema in this file so consumers can rationally parse it
+---
+apiVersion: 'v1.0'
+kind: Region
+metadata:
+  name: sitename
+  date: 17-FEB-2017
+  description: Sample site design
+  author: sh8121@att.com
+# Not sure if we have site wide data that doesn't fall into another 'Kind'
+---
+apiVersion: 'v1.0'
+kind: HostProfile
+metadata:
+  name: defaults
+  region: sitename
+  date: 17-FEB-2017
+  author: sh8121@att.com
+  description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
+  # No magic to this host_profile, it just provides a way to specify
+  # sitewide settings. If it is absent from a node's inheritance chain
+  # then these values will NOT be applied
+spec:
+  # OOB (iLO, iDRAC, etc...) settings. Should prefer open standards such
+  # as IPMI over vender-specific when possible.
+  oob:
+    type: ipmi
+    # OOB networking should be preconfigured, but we can include a network
+    # definition for validation or enhancement (DNS registration)
+    network: oob
+    account: admin
+    credential: admin
+  # Specify storage layout of base OS. Ceph out of scope
+  storage:
+    # How storage should be carved up: lvm (logical volumes), flat 
+    # (single partition)
+    layout: lvm
+    # Info specific to the boot and root disk/partitions
+    bootdisk:
+      # Device will specify an alias defined in hwdefinition.yaml
+      device: primary_boot
+      # For LVM, the size of the partition added to VG as a PV
+      # For flat, the size of the partition formatted as ext4
+      root_size: 50g
+      # The /boot partition. If not specified, /boot will in root
+      boot_size: 2g
+    # Info for additional partitions. Need to balance between
+    # flexibility and complexity
+    partitions:
+    - name: logs
+      device: primary_boot
+      # Partition uuid if needed
+      part_uuid: 84db9664-f45e-11e6-823d-080027ef795a
+      size: 10g
+      # Optional, can carve up unformatted block devices
+      mountpoint: /var/log
+      fstype: ext4
+      mount_options: defaults
+      # Filesystem UUID or label can be specified. UUID recommended
+      fs_uuid: cdb74f1c-9e50-4e51-be1d-068b0e9ff69e
+      fs_label: logs
+  # Platform (Operating System) settings
+  platform:
+    image: ubuntu_16.04_hwe
+    kernel_params: default
+  # Additional metadata to apply to a node
+  metadata:
+    # Base URL of the introspection service - may go in curtin data
+    introspection_url: http://172.16.1.10:9090
+---
+apiVersion: 'v1.0'
+kind: HostProfile
+metadata:
+  name: k8-node
+  region: sitename
+  date: 17-FEB-2017
+  author: sh8121@att.com
+  description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
+spec:
+  # host_profile inheritance allows for deduplication of common CIs
+  # Inheritance is additive for CIs that are lists of multiple items
+  # To remove an inherited list member, prefix the primary key value
+  # with '!'. 
+  host_profile: defaults
+  # Hardware profile will map hardware specific details to the abstract
+  # names uses in the host profile as well as specify hardware specific
+  # configs. A viable model should be to build a host profile without a
+  # hardware_profile and then for each node inherit the host profile and
+  # specify a hardware_profile to map that node's hardware to the abstract
+  # settings of the host_profile
+  hardware_profile: HPGen9v3
+  # Network interfaces.
+  interfaces:
+  # Keyed on device_name
+  # pxe is a special marker indicating which device should be used for pxe boot
+  - device_name: pxe
+    # The network link attached to this 
+    network_link: pxe
+    # Slaves will specify aliases from hwdefinition.yaml
+    slaves:
+    - prim_nic01
+    # Which networks will be configured on this interface
+    networks: 
+    - pxe
+  - device_name: bond0
+    network_link: gp
+    # If multiple slaves are specified, but no bonding config
+    # is applied to the link, design validation will fail
+    slaves:
+    - prim_nic01
+    - prim_nic02
+    # If multiple networks are specified, but no trunking
+    # config is applied to the link, design validation will fail
+    networks:
+    - mgmt
+    - private
+  metadata:
+    # Explicit tag assignment
+    tags:
+    - 'test'
+    # MaaS supports key/value pairs. Not sure of the use yet
+    owner_data:
+      foo: bar
+---
+apiVersion: 'v1.0'
+kind: HostProfile
+metadata:
+  name: k8-node-public
+  region: sitename
+  date: 17-FEB-2017
+  author: sh8121@att.com
+  description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
+spec:
+  host_profile: k8-node
+  interfaces:
+    - device_name: bond0
+      networks:
+        # This is additive, so adds a network to those defined in the host_profile
+        # inheritance chain
+      - public
+---
+apiVersion: 'v1.0'
+kind: HardwareProfile
+metadata:
+  name: HPGen9v3
+  region: sitename
+  date: 17-FEB-2017
+  author: Scott Hussey
+spec:
+  # Vendor of the server chassis
+  vendor: HP
+  # Generation of the chassis model
+  generation: '8'
+  # Version of the chassis model within its generation - not version of the hardware definition
+  hw_version: '3'
+  # The certified version of the chassis BIOS
+  bios_version: '2.2.3'
+  # Mode of the default boot of hardware - bios, uefi
+  boot_mode: bios
+  # Protocol of boot of the hardware - pxe, usb, hdd
+  bootstrap_protocol: pxe
+  # Which interface to use for network booting within the OOB manager, not OS device
+  pxe_interface: 0
+  # Map hardware addresses to aliases/roles to allow a mix of hardware configs
+  # in a site to result in a consistent configuration
+  device_aliases:
+    pci:
+    - address: pci@0000:00:03.0
+      alias: prim_nic01
+      # type could identify expected hardware - used for hardware manifest validation
+      type: '82540EM Gigabit Ethernet Controller'
+    - address: pci@0000:00:04.0
+      alias: prim_nic02
+      type: '82540EM Gigabit Ethernet Controller'
+    scsi:
+    - address: scsi@2:0.0.0
+      alias: primary_boot
+      type: 'VBOX HARDDISK'

tests/yaml_samples/invalid.yaml

@@ -0,0 +1,29 @@
+---
+apiVersion: 
+: Network
+metadata:
+  name: public
+  region: sitename
+  date: 17-FEB-2017
+  name: Sample network definition
+  author: sh8121@att.com
+  description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
+spec:
+  vlan_id: '102'
+    # How are addresses assigned?
+  allocation: static
+    # MTU size for the VLAN interface
+  mtu: 1500
+  cidr: 172.16.3.0/24
+    # Desribe IP address ranges
+  ranges: 
+  - type: static
+    start: 172.16.3.15
+    end: 172.16.3.254
+  routes:
+  - subnet: 0.0.0.0/0
+    gateway: 172.16.3.1
+    metric: 9
+  dns:
+    domain: sitename.example.com
+    servers: 8.8.8.8

tests/yaml_samples/multidoc.yaml

@@ -0,0 +1,77 @@
+---
+apiVersion: 'v1.0'
+kind: NetworkLink
+metadata:
+  name: oob
+  region: sitename
+  date: 17-FEB-2017
+  name: Sample network link
+  author: sh8121@att.com
+  description: Describe layer 1 attributes. Primary key is 'name'. These settings will generally be things the switch and server have to agree on
+spec:
+  bonding:
+    mode: none
+  mtu: 1500
+  linkspeed: 100full
+  trunking:
+    mode: none
+    default_network: oob
+---
+# pxe is a bit of 'magic' indicating the link config used when PXE booting
+# a node. All other links indicate network configs applied when the node
+# is deployed.
+apiVersion: 'v1.0'
+kind: NetworkLink
+metadata:
+  name: pxe
+  region: sitename
+  date: 17-FEB-2017
+  name: Sample network link
+  author: sh8121@att.com
+  description: Describe layer 1 attributes. Primary key is 'name'. These settings will generally be things the switch and server have to agree on
+spec:
+  bonding:
+    mode: none
+  mtu: 1500
+  linkspeed: auto
+  # Is this link supporting multiple layer 2 networks?
+  # none is a port-based VLAN identified by default_network
+  # tagged is is using 802.1q VLAN tagging. Untagged packets will default to default_netwokr
+  trunking:
+    mode: none
+    # use name, will translate to VLAN ID
+    default_network: pxe
+---
+apiVersion: 'v1.0'
+kind: NetworkLink
+metadata:
+  name: gp
+  region: sitename
+  date: 17-FEB-2017
+  name: Sample network link
+  author: sh8121@att.com
+  description: Describe layer 1 attributes. These CIs will generally be things the switch and server have to agree on
+  # pxe is a bit of 'magic' indicating the link config used when PXE booting
+  # a node. All other links indicate network configs applied when the node
+  # is deployed.
+spec:
+  # If this link is a bond of physical links, how is it configured
+  # 802.3ad
+  # active-backup
+  # balance-rr 
+  # Can add support for others down the road
+  bonding:
+    mode: 802.3ad
+    # For LACP (802.3ad) xmit hashing policy: layer2, layer2+3, layer3+4, encap3+4
+    hash: layer3+4
+    # 802.3ad specific options
+    peer_rate: slow
+    mon_rate: default
+    up_delay: default
+    down_delay: default
+  mtu: 9000
+  linkspeed: auto
+  # Is this link supporting multiple layer 2 networks?
+  trunking:
+    mode: tagged
+    default_network: mgmt

tests/yaml_samples/singledoc.yaml

@@ -0,0 +1,39 @@
+---
+apiVersion: 'v1.0'
+kind: HardwareProfile
+metadata:
+  name: HPGen8v3
+  region: sitename
+  date: 17-FEB-2017
+  name: Sample hardware definition
+  author: Scott Hussey
+spec:
+  # Vendor of the server chassis
+  vendor: HP
+  # Generation of the chassis model
+  generation: '8'
+  # Version of the chassis model within its generation - not version of the hardware definition
+  hw_version: '3'
+  # The certified version of the chassis BIOS
+  bios_version: '2.2.3'
+  # Mode of the default boot of hardware - bios, uefi
+  boot_mode: bios
+  # Protocol of boot of the hardware - pxe, usb, hdd
+  bootstrap_protocol: pxe
+  # Which interface to use for network booting within the OOB manager, not OS device
+  pxe_interface: 0
+  # Map hardware addresses to aliases/roles to allow a mix of hardware configs
+  # in a site to result in a consistent configuration
+  device_aliases:
+    pci:
+    - address: pci@0000:00:03.0
+      alias: prim_nic01
+      # type could identify expected hardware - used for hardware manifest validation
+      type: '82540EM Gigabit Ethernet Controller'
+    - address: pci@0000:00:04.0
+      alias: prim_nic02
+      type: '82540EM Gigabit Ethernet Controller'
+    scsi:
+    - address: scsi@2:0.0.0
+      alias: primary_boot
+      type: 'VBOX HARDDISK'

tests/yaml_samples/unknown_kind.yaml

@@ -0,0 +1,60 @@
+---
+apiVersion: 'v1.0'
+kind: FooBar
+metadata:
+  name: default
+  region: sitename
+  date: 17-FEB-2017
+  name: Sample network definition
+  author: sh8121@att.com
+  description: Describe layer 2/3 attributes. Primarily CIs used for configuring server interfaces
+  # No magic to this host_profile, it just provides a way to specify
+  # sitewide settings. If it is absent from a node's inheritance chain
+  # then these values will NOT be applied
+spec:
+  # OOB (iLO, iDRAC, etc...) settings. Should prefer open standards such
+  # as IPMI over vender-specific when possible.
+  oob:
+    type: ipmi
+    # OOB networking should be preconfigured, but we can include a network
+    # definition for validation or enhancement (DNS registration)
+    network: oob
+    account: admin
+    credential: admin
+  # Specify storage layout of base OS. Ceph out of scope
+  storage:
+    # How storage should be carved up: lvm (logical volumes), flat 
+    # (single partition)
+    layout: lvm
+    # Info specific to the boot and root disk/partitions
+    bootdisk:
+      # Device will specify an alias defined in hwdefinition.yaml
+      device: primary_boot
+      # For LVM, the size of the partition added to VG as a PV
+      # For flat, the size of the partition formatted as ext4
+      root_size: 50g
+      # The /boot partition. If not specified, /boot will in root
+      boot_size: 2g
+    # Info for additional partitions. Need to balance between
+    # flexibility and complexity
+    partitions:
+    - name: logs
+      device: primary_boot
+      # Partition uuid if needed
+      part_uuid: 84db9664-f45e-11e6-823d-080027ef795a
+      size: 10g
+      # Optional, can carve up unformatted block devices
+      mountpoint: /var/log
+      fstype: ext4
+      mount_options: defaults
+      # Filesystem UUID or label can be specified. UUID recommended
+      fs_uuid: cdb74f1c-9e50-4e51-be1d-068b0e9ff69e
+      fs_label: logs
+  # Platform (Operating System) settings
+  platform:
+    image: ubuntu_16.04_hwe
+    kernel_params: default
+  # Additional metadata to apply to a node
+  metadata:
+    # Base URL of the introspection service - may go in curtin data
+    introspection_url: http://172.16.1.10:9090

tox.ini

@@ -0,0 +1,15 @@
+[tox]
+envlist = py35
+
+[testenv]
+deps=
+  -rrequirements.txt
+  -rtestrequirements.txt
+setenv=
+  PYTHONWARNING=all
+commands=
+  py.test \
+  	{posargs}
+ 
+[flake8]
+ignore=E302,H306