411 lines
15 KiB
Python
411 lines
15 KiB
Python
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import copy
|
|
import json
|
|
import logging
|
|
import pkg_resources
|
|
import pprint
|
|
import sys
|
|
|
|
import jsonschema
|
|
import netaddr
|
|
import yaml
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
|
|
class ProcessDataSource(object):
|
|
def __init__(self, sitetype):
|
|
# Initialize intermediary and save site type
|
|
self._initialize_intermediary()
|
|
self.region_name = sitetype
|
|
|
|
@staticmethod
|
|
def _read_file(file_name):
|
|
with open(file_name, "r") as f:
|
|
raw_data = f.read()
|
|
return raw_data
|
|
|
|
def _initialize_intermediary(self):
|
|
self.host_type = {}
|
|
self.data = {
|
|
"network": {},
|
|
"baremetal": {},
|
|
"region_name": "",
|
|
"storage": {},
|
|
"site_info": {},
|
|
}
|
|
self.sitetype = None
|
|
self.genesis_node = None
|
|
self.region_name = None
|
|
self.network_subnets = None
|
|
|
|
def _get_network_subnets(self):
|
|
"""Extract subnet information for networks.
|
|
|
|
In some networks, there are multiple subnets, in that case
|
|
we assign only the first subnet
|
|
"""
|
|
|
|
LOG.info("Extracting network subnets")
|
|
network_subnets = {}
|
|
for net_type in self.data["network"]["vlan_network_data"]:
|
|
# One of the type is ingress and we don't want that here
|
|
if net_type != "ingress":
|
|
network_subnets[net_type] = netaddr.IPNetwork(
|
|
self.data["network"]["vlan_network_data"][net_type][
|
|
"subnet"
|
|
][0]
|
|
)
|
|
|
|
LOG.debug(
|
|
"Network subnets:\n{}".format(pprint.pformat(network_subnets))
|
|
)
|
|
return network_subnets
|
|
|
|
def _get_genesis_node_details(self):
|
|
# Get genesis host node details from the hosts based on host type
|
|
for racks in self.data["baremetal"].keys():
|
|
rack_hosts = self.data["baremetal"][racks]
|
|
for host in rack_hosts:
|
|
if rack_hosts[host]["type"] == "genesis":
|
|
self.genesis_node = rack_hosts[host]
|
|
self.genesis_node["name"] = host
|
|
LOG.debug(
|
|
"Genesis Node Details:\n{}".format(
|
|
pprint.pformat(self.genesis_node)
|
|
)
|
|
)
|
|
|
|
def _get_genesis_node_ip(self):
|
|
"""Returns the genesis node ip"""
|
|
|
|
ip = "0.0.0.0"
|
|
LOG.info("Getting Genesis Node IP")
|
|
if not self.genesis_node:
|
|
self._get_genesis_node_details()
|
|
ips = self.genesis_node.get("ip", "")
|
|
if ips:
|
|
ip = ips.get("oam", "0.0.0.0")
|
|
return ip
|
|
|
|
def _validate_intermediary_data(self, data):
|
|
"""Validates the intermediary data before generating manifests.
|
|
|
|
It checks wether the data types and data format are as expected.
|
|
The method validates this with regex pattern defined for each
|
|
data type.
|
|
"""
|
|
|
|
LOG.info("Validating Intermediary data")
|
|
temp_data = {}
|
|
# Peforming a deep copy
|
|
temp_data = copy.deepcopy(data)
|
|
# Converting baremetal dict to list.
|
|
baremetal_list = []
|
|
for rack in temp_data["baremetal"].keys():
|
|
temp = [{k: v} for k, v in temp_data["baremetal"][rack].items()]
|
|
baremetal_list = baremetal_list + temp
|
|
|
|
temp_data["baremetal"] = baremetal_list
|
|
schema_dir = pkg_resources.resource_filename("spyglass", "schemas/")
|
|
schema_file = schema_dir + "data_schema.json"
|
|
json_data = json.loads(json.dumps(temp_data))
|
|
with open(schema_file, "r") as f:
|
|
json_schema = json.load(f)
|
|
try:
|
|
# Suppressing writing of data2.json. Can use it for debugging
|
|
# with open('data2.json', 'w') as outfile:
|
|
# json.dump(temp_data, outfile, sort_keys=True, indent=4)
|
|
jsonschema.validate(json_data, json_schema)
|
|
except jsonschema.exceptions.ValidationError as e:
|
|
LOG.error("Validation Error")
|
|
LOG.error("Message:{}".format(e.message))
|
|
LOG.error("Validator_path:{}".format(e.path))
|
|
LOG.error("Validator_pattern:{}".format(e.validator_value))
|
|
LOG.error("Validator:{}".format(e.validator))
|
|
sys.exit()
|
|
except jsonschema.exceptions.SchemaError as e:
|
|
LOG.error("Schema Validation Error!!")
|
|
LOG.error("Message:{}".format(e.message))
|
|
LOG.error("Schema:{}".format(e.schema))
|
|
LOG.error("Validator_value:{}".format(e.validator_value))
|
|
LOG.error("Validator:{}".format(e.validator))
|
|
LOG.error("path:{}".format(e.path))
|
|
sys.exit()
|
|
|
|
LOG.info("Data validation Passed!")
|
|
|
|
def _apply_design_rules(self):
|
|
"""Applies design rules from rules.yaml
|
|
|
|
These rules are used to determine ip address allocation ranges,
|
|
host profile interfaces and also to create hardware profile
|
|
information. The method calls corresponding rule hander function
|
|
based on rule name and applies them to appropriate data objects.
|
|
"""
|
|
|
|
LOG.info("Apply design rules")
|
|
rules_dir = pkg_resources.resource_filename("spyglass", "config/")
|
|
rules_file = rules_dir + "rules.yaml"
|
|
rules_data_raw = self._read_file(rules_file)
|
|
rules_yaml = yaml.safe_load(rules_data_raw)
|
|
rules_data = {}
|
|
rules_data.update(rules_yaml)
|
|
for rule in rules_data.keys():
|
|
rule_name = rules_data[rule]["name"]
|
|
function_str = "_apply_rule_" + rule_name
|
|
rule_data_name = rules_data[rule][rule_name]
|
|
function = getattr(self, function_str)
|
|
function(rule_data_name)
|
|
LOG.info("Applying rule:{}".format(rule_name))
|
|
|
|
def _apply_rule_host_profile_interfaces(self, rule_data):
|
|
# TODO(pg710r)Nothing to do as of now since host profile
|
|
# information is already present in plugin data.
|
|
# This function shall be defined if plugin data source
|
|
# doesn't provide host profile information.
|
|
pass
|
|
|
|
def _apply_rule_hardware_profile(self, rule_data):
|
|
"""Apply rules to define host type from hardware profile info.
|
|
|
|
Host profile will define host types as "controller, compute or
|
|
genesis". The rule_data has pre-defined information to define
|
|
compute or controller based on host_profile. For defining 'genesis'
|
|
the first controller host is defined as genesis.
|
|
"""
|
|
|
|
is_genesis = False
|
|
hardware_profile = rule_data[self.data["site_info"]["sitetype"]]
|
|
# Getting individual racks. The racks are sorted to ensure that the
|
|
# first controller of the first rack is assigned as 'genesis' node.
|
|
for rack in sorted(self.data["baremetal"].keys()):
|
|
# Getting individual hosts in each rack. Sorting of the hosts are
|
|
# done to determine the genesis node.
|
|
for host in sorted(self.data["baremetal"][rack].keys()):
|
|
host_info = self.data["baremetal"][rack][host]
|
|
if (
|
|
host_info["host_profile"]
|
|
== hardware_profile["profile_name"]["ctrl"]
|
|
):
|
|
if not is_genesis:
|
|
host_info["type"] = "genesis"
|
|
is_genesis = True
|
|
else:
|
|
host_info["type"] = "controller"
|
|
else:
|
|
host_info["type"] = "compute"
|
|
|
|
def _apply_rule_ip_alloc_offset(self, rule_data):
|
|
"""Apply offset rules to update baremetal host
|
|
|
|
ip's and vlan network
|
|
"""
|
|
|
|
# Get network subnets
|
|
self.network_subnets = self._get_network_subnets()
|
|
|
|
self._update_vlan_net_data(rule_data)
|
|
self._update_baremetal_host_ip_data(rule_data)
|
|
|
|
def _update_baremetal_host_ip_data(self, rule_data):
|
|
"""Update baremetal host ip's for applicable networks.
|
|
|
|
The applicable networks are oob, oam, ksn, storage and overlay.
|
|
These IPs are assigned based on network subnets ranges.
|
|
If a particular ip exists it is overridden.
|
|
"""
|
|
|
|
# Ger defult ip offset
|
|
default_ip_offset = rule_data["default"]
|
|
|
|
host_idx = 0
|
|
LOG.info("Update baremetal host ip's")
|
|
for racks in self.data["baremetal"].keys():
|
|
rack_hosts = self.data["baremetal"][racks]
|
|
for host in rack_hosts:
|
|
host_networks = rack_hosts[host]["ip"]
|
|
for net in host_networks:
|
|
ips = list(self.network_subnets[net])
|
|
host_networks[net] = str(ips[host_idx + default_ip_offset])
|
|
host_idx = host_idx + 1
|
|
|
|
LOG.debug(
|
|
"Updated baremetal host:\n{}".format(
|
|
pprint.pformat(self.data["baremetal"])
|
|
)
|
|
)
|
|
|
|
def _update_vlan_net_data(self, rule_data):
|
|
"""Offset allocation rules to determine ip address range(s)
|
|
|
|
This rule is applied to incoming network data to determine
|
|
network address, gateway ip and other address ranges
|
|
"""
|
|
|
|
LOG.info("Apply network design rules")
|
|
|
|
# Collect Rules
|
|
default_ip_offset = rule_data["default"]
|
|
oob_ip_offset = rule_data["oob"]
|
|
gateway_ip_offset = rule_data["gateway"]
|
|
ingress_vip_offset = rule_data["ingress_vip"]
|
|
# static_ip_end_offset for non pxe network
|
|
static_ip_end_offset = rule_data["static_ip_end"]
|
|
# dhcp_ip_end_offset for pxe network
|
|
dhcp_ip_end_offset = rule_data["dhcp_ip_end"]
|
|
|
|
# Set ingress vip and CIDR for bgp
|
|
LOG.info("Apply network design rules:bgp")
|
|
subnet = netaddr.IPNetwork(
|
|
self.data["network"]["vlan_network_data"]["ingress"]["subnet"][0]
|
|
)
|
|
ips = list(subnet)
|
|
self.data["network"]["bgp"]["ingress_vip"] = str(
|
|
ips[ingress_vip_offset]
|
|
)
|
|
self.data["network"]["bgp"]["public_service_cidr"] = self.data[
|
|
"network"
|
|
]["vlan_network_data"]["ingress"]["subnet"][0]
|
|
LOG.debug(
|
|
"Updated network bgp data:\n{}".format(
|
|
pprint.pformat(self.data["network"]["bgp"])
|
|
)
|
|
)
|
|
|
|
LOG.info("Apply network design rules:vlan")
|
|
# Apply rules to vlan networks
|
|
for net_type in self.network_subnets:
|
|
if net_type == "oob":
|
|
ip_offset = oob_ip_offset
|
|
else:
|
|
ip_offset = default_ip_offset
|
|
|
|
subnet = self.network_subnets[net_type]
|
|
ips = list(subnet)
|
|
|
|
self.data["network"]["vlan_network_data"][net_type][
|
|
"gateway"
|
|
] = str(ips[gateway_ip_offset])
|
|
|
|
self.data["network"]["vlan_network_data"][net_type][
|
|
"reserved_start"
|
|
] = str(ips[1])
|
|
self.data["network"]["vlan_network_data"][net_type][
|
|
"reserved_end"
|
|
] = str(ips[ip_offset])
|
|
|
|
static_start = str(ips[ip_offset + 1])
|
|
static_end = str(ips[static_ip_end_offset])
|
|
|
|
if net_type == "pxe":
|
|
mid = len(ips) // 2
|
|
static_end = str(ips[mid - 1])
|
|
dhcp_start = str(ips[mid])
|
|
dhcp_end = str(ips[dhcp_ip_end_offset])
|
|
|
|
self.data["network"]["vlan_network_data"][net_type][
|
|
"dhcp_start"
|
|
] = dhcp_start
|
|
self.data["network"]["vlan_network_data"][net_type][
|
|
"dhcp_end"
|
|
] = dhcp_end
|
|
|
|
self.data["network"]["vlan_network_data"][net_type][
|
|
"static_start"
|
|
] = static_start
|
|
self.data["network"]["vlan_network_data"][net_type][
|
|
"static_end"
|
|
] = static_end
|
|
|
|
# There is no vlan for oob network
|
|
if net_type != "oob":
|
|
self.data["network"]["vlan_network_data"][net_type][
|
|
"vlan"
|
|
] = self.data["network"]["vlan_network_data"][net_type]["vlan"]
|
|
|
|
# OAM have default routes. Only for cruiser. TBD
|
|
if net_type == "oam":
|
|
routes = ["0.0.0.0/0"]
|
|
else:
|
|
routes = []
|
|
self.data["network"]["vlan_network_data"][net_type][
|
|
"routes"
|
|
] = routes
|
|
|
|
LOG.debug(
|
|
"Updated vlan network data:\n{}".format(
|
|
pprint.pformat(self.data["network"]["vlan_network_data"])
|
|
)
|
|
)
|
|
|
|
def load_extracted_data_from_data_source(self, extracted_data):
|
|
"""Function called from spyglass.py to pass extracted data
|
|
|
|
from input data source
|
|
"""
|
|
|
|
# TBR(pg710r): for internal testing
|
|
"""
|
|
raw_data = self._read_file('extracted_data.yaml')
|
|
extracted_data = yaml.safe_load(raw_data)
|
|
"""
|
|
|
|
LOG.info("Loading plugin data source")
|
|
self.data = extracted_data
|
|
LOG.debug(
|
|
"Extracted data from plugin:\n{}".format(
|
|
pprint.pformat(extracted_data)
|
|
)
|
|
)
|
|
# Uncommeent following segment for debugging purpose.
|
|
# extracted_file = "extracted_file.yaml"
|
|
# yaml_file = yaml.dump(extracted_data, default_flow_style=False)
|
|
# with open(extracted_file, 'w') as f:
|
|
# f.write(yaml_file)
|
|
# f.close()
|
|
|
|
# Append region_data supplied from CLI to self.data
|
|
self.data["region_name"] = self.region_name
|
|
|
|
def dump_intermediary_file(self, intermediary_dir):
|
|
"""Writing intermediary yaml"""
|
|
|
|
LOG.info("Writing intermediary yaml")
|
|
intermediary_file = "{}_intermediary.yaml".format(
|
|
self.data["region_name"]
|
|
)
|
|
# Check of if output dir = intermediary_dir exists
|
|
if intermediary_dir is not None:
|
|
outfile = "{}/{}".format(intermediary_dir, intermediary_file)
|
|
else:
|
|
outfile = intermediary_file
|
|
LOG.info("Intermediary file:{}".format(outfile))
|
|
yaml_file = yaml.dump(self.data, default_flow_style=False)
|
|
with open(outfile, "w") as f:
|
|
f.write(yaml_file)
|
|
f.close()
|
|
|
|
def generate_intermediary_yaml(self):
|
|
"""Generating intermediary yaml"""
|
|
|
|
LOG.info("Start: Generate Intermediary")
|
|
self._apply_design_rules()
|
|
self._get_genesis_node_details()
|
|
# This will validate the extracted data from different sources.
|
|
self._validate_intermediary_data(self.data)
|
|
return self.data
|