airship
/
spyglass
Code Issues Proposed changes

Implements data object models 

This change implements data object models from [0] in data extraction
and parsing. The change results in minor modifications to the outputted
intermediary, which can be seen between these two example intermeidary
files [1].

This fully implements the data objects from models.py in data extraction
and parsing. A follow-up change will implement use of the data objects
in Jinja2. Temporarily, all objects will be converted to dictionaries
for generating documents from templates.

[0] https://review.opendev.org/#/c/658917/
[1] https://www.diffchecker.com/NnjjJrb2

Change-Id: Ifd867787aab541be5dabecf9f6026faa2ec7049e
This commit is contained in:
Ian H. Pittwood 2019-05-13 10:34:31 -05:00
parent f9226d2f4a
commit 4747222641
8 changed files with 183 additions and 372 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
spyglass/cli.py
View File

@ -98,8 +98,7 @@ def intermediary_processor(plugin_type, **kwargs):
additional_config = kwargs.get('site_configuration', None) additional_config = kwargs.get('site_configuration', None)
if additional_config is not None: if additional_config is not None:
with open(additional_config, 'r') as config: with open(additional_config, 'r') as config:
raw_data = config.read() additional_config_data = yaml.safe_load(config)
additional_config_data = yaml.safe_load(raw_data)
LOG.debug( LOG.debug(
"Additional config data:\n{}".format( "Additional config data:\n{}".format(
pprint.pformat(additional_config_data))) pprint.pformat(additional_config_data)))

269
spyglass/data_extractor/base.py
View File

@ -14,9 +14,8 @@
import abc import abc
import logging import logging
import pprint
from spyglass.utils import utils from spyglass.data_extractor import models
LOG = logging.getLogger(__name__) LOG = logging.getLogger(__name__)
@ -28,7 +27,7 @@ class BaseDataSourcePlugin(metaclass=abc.ABCMeta):
self.source_type = None self.source_type = None
self.source_name = None self.source_name = None
self.region = region self.region = region
self.site_data = {} self.site_data = None
@abc.abstractmethod @abc.abstractmethod
def set_config_opts(self, conf): def set_config_opts(self, conf):
@ -63,10 +62,8 @@ class BaseDataSourcePlugin(metaclass=abc.ABCMeta):
"""Return list of racks in the region """Return list of racks in the region
:param string region: Region name :param string region: Region name
:returns: list of rack names :returns: list of Rack objects
:rtype: list :rtype: list
Example: ['rack01', 'rack02']
""" """
return [] return []
@ -77,20 +74,8 @@ class BaseDataSourcePlugin(metaclass=abc.ABCMeta):
:param string region: Region name :param string region: Region name
:param string rack: Rack name :param string rack: Rack name
:returns: list of hosts information :returns: list of Host objects containing a rack's host data
:rtype: list of dict :rtype: list of models.Host
Example: [
{
'name': 'host01',
'type': 'controller',
'host_profile': 'hp_01'
},
{
'name': 'host02',
'type': 'compute',
'host_profile': 'hp_02'}
]
""" """
return [] return []
@ -100,47 +85,8 @@ class BaseDataSourcePlugin(metaclass=abc.ABCMeta):
"""Return list of networks in the region """Return list of networks in the region
:param string region: Region name :param string region: Region name
:returns: list of networks and their vlans :returns: list of network data
:rtype: list of dict :rtype: list of models.VLANNetworkData
Example: [
{
'name': 'oob',
'vlan': '41',
'subnet': '192.168.1.0/24',
'gateway': '192.168.1.1'
},
{
'name': 'pxe',
'vlan': '42',
'subnet': '192.168.2.0/24',
'gateway': '192.168.2.1'
},
{
'name': 'oam',
'vlan': '43',
'subnet': '192.168.3.0/24',
'gateway': '192.168.3.1'
},
{
'name': 'ksn',
'vlan': '44',
'subnet': '192.168.4.0/24',
'gateway': '192.168.4.1'
},
{
'name': 'storage',
'vlan': '45',
'subnet': '192.168.5.0/24',
'gateway': '192.168.5.1'
},
{
'name': 'overlay',
'vlan': '45',
'subnet': '192.168.6.0/24',
'gateway': '192.168.6.1'
}
]
""" """
# TODO(nh863p): Expand the return type if they are rack level subnets # TODO(nh863p): Expand the return type if they are rack level subnets
@ -153,11 +99,8 @@ class BaseDataSourcePlugin(metaclass=abc.ABCMeta):
:param string region: Region name :param string region: Region name
:param string host: Host name :param string host: Host name
:returns: Dict of IPs per network on the host :returns: IPs per network on the host
:rtype: dict :rtype: models.IPList
Example: {'oob': {'ipv4': '192.168.1.10'},
'pxe': {'ipv4': '192.168.2.10'}}
The network name from get_networks is expected to be the keys of this The network name from get_networks is expected to be the keys of this
dict. In case some networks are missed, they are expected to be either dict. In case some networks are missed, they are expected to be either
@ -171,10 +114,8 @@ class BaseDataSourcePlugin(metaclass=abc.ABCMeta):
"""Return the DNS servers """Return the DNS servers
:param string region: Region name :param string region: Region name
:returns: List of DNS servers to be configured on host :returns: DNS servers to be configured on host
:rtype: List :rtype: models.ServerList
Example: ['8.8.8.8', '8.8.8.4']
""" """
return [] return []
@ -184,10 +125,8 @@ class BaseDataSourcePlugin(metaclass=abc.ABCMeta):
"""Return the NTP servers """Return the NTP servers
:param string region: Region name :param string region: Region name
:returns: List of NTP servers to be configured on host :returns: NTP servers to be configured on host
:rtype: List :rtype: models.ServerList
Example: ['ntp1.ubuntu1.example', 'ntp2.ubuntu.example']
""" """
return [] return []
@ -198,7 +137,7 @@ class BaseDataSourcePlugin(metaclass=abc.ABCMeta):
:param string region: Region name :param string region: Region name
:returns: LDAP server information :returns: LDAP server information
:rtype: Dict :rtype: dict
Example: {'url': 'ldap.example.com', Example: {'url': 'ldap.example.com',
'common_name': 'ldap-site1', 'common_name': 'ldap-site1',
@ -231,152 +170,68 @@ class BaseDataSourcePlugin(metaclass=abc.ABCMeta):
:param string region: Region name :param string region: Region name
:returns: Domain name :returns: Domain name
:rtype: string :rtype: str
Example: example.com Example: example.com
""" """
return "" return ""
@abc.abstractmethod
def get_site_info(self, region):
"""Return site data as a SiteInfo object
:param region: Region name
:return: general site data including location, domain, name, LDAP, NTP,
DNS, and site type
:rtype: models.SiteInfo
"""
return None
def extract_baremetal_information(self): def extract_baremetal_information(self):
"""Get baremetal information from plugin """Get baremetal information from plugin
:returns: dict of baremetal nodes :returns: racks and hosts as a list of Rack objects containing Host
:rtype: dict data
:rtype: list of models.Rack
Return dict should be in the format
{
'EXAMR06': { # rack name
'examr06c036': { # host name
'host_profile': None,
'ip': {
'overlay': {},
'oob': {},
'calico': {},
'oam': {},
'storage': {},
'pxe': {}
},
'rack': 'EXAMR06',
'type': 'compute'
}
}
}
""" """
LOG.info("Extract baremetal information from plugin") LOG.info("Extract baremetal information from plugin")
baremetal = {} return self.get_racks(self.region)
hosts = self.get_hosts(self.region)
# For each host list fill host profile and network IPs
for host in hosts:
host_name = host["name"]
rack_name = host["rack_name"]
if rack_name not in baremetal:
baremetal[rack_name] = {}
# Prepare temp dict for each host and append it to baremetal
# at a rack level
temp_host = {}
if host["host_profile"] is None:
temp_host["host_profile"] = "#CHANGE_ME"
else:
temp_host["host_profile"] = host["host_profile"]
# Get Host IPs from plugin
temp_host_ips = self.get_ips(self.region, host_name)
# Fill network IP for this host
temp_host["ip"] = {}
temp_host["ip"]["oob"] = \
temp_host_ips[host_name].get("oob", "#CHANGE_ME")
temp_host["ip"]["calico"] = \
temp_host_ips[host_name].get("calico", "#CHANGE_ME")
temp_host["ip"]["oam"] = \
temp_host_ips[host_name].get("oam", "#CHANGE_ME")
temp_host["ip"]["storage"] = \
temp_host_ips[host_name].get("storage", "#CHANGE_ME")
temp_host["ip"]["overlay"] = \
temp_host_ips[host_name].get("overlay", "#CHANGE_ME")
temp_host["ip"]["pxe"] = \
temp_host_ips[host_name].get("pxe", "#CHANGE_ME")
baremetal[rack_name][host_name] = temp_host
LOG.debug(
"Baremetal information:\n{}".format(pprint.pformat(baremetal)))
return baremetal
def extract_site_information(self): def extract_site_information(self):
"""Get site information from plugin """Get site information from plugin
:returns: dict of site information :returns: site information including location, dns servers, ntp servers
:rtpe: dict ldap, and domain name
:rtpe: models.SiteInfo
Return dict should be in the format
{
'name': '',
'country': '',
'state': '',
'corridor': '',
'sitetype': '',
'dns': [],
'ntp': [],
'ldap': {},
'domain': None
}
""" """
LOG.info("Extract site information from plugin") LOG.info("Extract site information from plugin")
site_info = {}
# Extract location information # Extract location information
location_data = self.get_location_information(self.region) data = {
if location_data is not None: 'region_name': self.region,
site_info = location_data 'dns': self.get_dns_servers(self.region),
'ntp': self.get_ntp_servers(self.region),
'ldap': self.get_ldap_information(self.region),
'domain': self.get_domain_name(self.region)
}
data.update(self.get_location_information(self.region) or {})
dns_data = self.get_dns_servers(self.region) site_info = models.SiteInfo(**data)
site_info["dns"] = dns_data
ntp_data = self.get_ntp_servers(self.region)
site_info["ntp"] = ntp_data
ldap_data = self.get_ldap_information(self.region)
site_info["ldap"] = ldap_data
domain_data = self.get_domain_name(self.region)
site_info["domain"] = domain_data
LOG.debug(
"Extracted site information:\n{}".format(
pprint.pformat(site_info)))
return site_info return site_info
def extract_network_information(self): def extract_network_information(self):
"""Get network details from plugin like Subnets, DNS, NTP and LDAP """Get network details from plugin like Subnets, DNS, NTP and LDAP
:returns: dict of baremetal nodes :returns: networking data as a Network object
:rtype: dict :rtype: models.Network
Return dict should be in the format
{
'vlan_network_data': {
'oam': {},
'ingress': {},
'oob': {}
'calico': {},
'storage': {},
'pxe': {},
'overlay': {}
}
}
""" """
LOG.info("Extract network information from plugin") LOG.info("Extract network information from plugin")
network_data = {}
networks = self.get_networks(self.region) networks = self.get_networks(self.region)
# We are interested in only the below networks mentioned in # We are interested in only the below networks mentioned in
@ -391,20 +246,12 @@ class BaseDataSourcePlugin(metaclass=abc.ABCMeta):
"oob", "oob",
"ingress", "ingress",
] ]
network_data["vlan_network_data"] = {} desired_networks = []
for network in networks:
if network.name in networks_to_scan:
desired_networks.append(network)
for net in networks: return models.Network(desired_networks)
tmp_net = {}
if net["name"] in networks_to_scan:
tmp_net["subnet"] = net.get("subnet", "#CHANGE_ME")
if net["name"] != "ingress" and net["name"] != "oob":
tmp_net["vlan"] = net.get("vlan", "#CHANGE_ME")
network_data["vlan_network_data"][net["name"]] = tmp_net
LOG.debug(
"Extracted network data:\n{}".format(pprint.pformat(network_data)))
return network_data
def extract_data(self): def extract_data(self):
"""Extract data from plugin """Extract data from plugin
@ -414,13 +261,11 @@ class BaseDataSourcePlugin(metaclass=abc.ABCMeta):
""" """
LOG.info("Extract data from plugin") LOG.info("Extract data from plugin")
site_data = { self.site_data = models.SiteDocumentData(
"baremetal": self.extract_baremetal_information(), self.extract_site_information(),
"site_info": self.extract_site_information(), self.extract_network_information(),
"network": self.extract_network_information() self.extract_baremetal_information())
} return self.site_data
self.site_data = site_data
return site_data
def apply_additional_data(self, extra_data): def apply_additional_data(self, extra_data):
"""Apply any additional inputs from user """Apply any additional inputs from user
@ -433,6 +278,4 @@ class BaseDataSourcePlugin(metaclass=abc.ABCMeta):
""" """
LOG.info("Update site data with additional input") LOG.info("Update site data with additional input")
tmp_site_data = utils.dict_merge(self.site_data, extra_data) self.site_data.merge_additional_data(extra_data)
self.site_data = tmp_site_data
return self.site_data

70
spyglass/data_extractor/models.py
View File

@ -31,24 +31,32 @@ def _parse_ip(addr):
:return: addr as an IPAddress object or string :return: addr as an IPAddress object or string
""" """
try: try:
ip = ipaddress.ip_address(addr) ipaddress.ip_network(addr)
return str(ip)
except ValueError: except ValueError:
LOG.warning("%s is not a valid IP address.", addr) if addr != DATA_DEFAULT:
return str(addr) LOG.warning("%s is not a valid IP address.", addr)
return addr
class ServerList(object): class ServerList(object):
"""Model for a list of servers""" """Model for a list of servers"""
def __init__(self, server_list: list): def __init__(self, server_list):
"""Validates a list of server IPs and creates a list of them """Validates a list of server IPs and creates a list of them
:param server_list: list of strings :param server_list: list of strings
""" """
self.servers = [] self.servers = []
for server in server_list: if type(server_list) is list:
self.servers.append(_parse_ip(server)) for server in server_list:
self.servers.append(_parse_ip(server))
elif type(server_list) is str:
for server in server_list.split(','):
self.servers.append(server.strip())
else:
raise ValueError(
'ServerList expects a str or list, but got a %s',
type(server_list))
def __str__(self): def __str__(self):
"""Returns server list as string for use in YAML documents""" """Returns server list as string for use in YAML documents"""
@ -272,15 +280,26 @@ class VLANNetworkData(object):
""" """
self.name = name self.name = name
self.role = kwargs.get('role', self.name) self.role = kwargs.get('role', self.name)
self.vlan = kwargs.get('vlan', None) if self.role == 'oob':
self.vlan = None
else:
self.vlan = kwargs.get('vlan', None)
self.subnet = [] self.subnet = []
for _subnet in kwargs.get('subnet', []): subnet = kwargs.get('subnet', [])
self.subnet.append(_parse_ip(_subnet)) if type(subnet) is list:
for _subnet in subnet:
self.subnet.append(_parse_ip(_subnet))
else:
self.subnet.append(subnet)
self.routes = [] self.routes = []
for route in kwargs.get('routes', []): routes = kwargs.get('routes', [])
self.routes.append(_parse_ip(route)) if type(routes) is list:
for route in routes:
self.routes.append(_parse_ip(route))
else:
self.routes.append(_parse_ip(routes))
self.gateway = _parse_ip(kwargs.get('gateway', None)) self.gateway = _parse_ip(kwargs.get('gateway', None))
@ -302,6 +321,8 @@ class VLANNetworkData(object):
vlan_dict[self.role]['subnet'] = self.subnet vlan_dict[self.role]['subnet'] = self.subnet
if self.routes: if self.routes:
vlan_dict[self.role]['routes'] = self.routes vlan_dict[self.role]['routes'] = self.routes
else:
vlan_dict[self.role]['routes'] = []
if self.gateway: if self.gateway:
vlan_dict[self.role]['gateway'] = self.gateway vlan_dict[self.role]['gateway'] = self.gateway
if self.dhcp_start and self.dhcp_end: if self.dhcp_start and self.dhcp_end:
@ -321,9 +342,11 @@ class VLANNetworkData(object):
if 'vlan' in config_dict: if 'vlan' in config_dict:
self.vlan = config_dict['vlan'] self.vlan = config_dict['vlan']
if 'subnet' in config_dict: if 'subnet' in config_dict:
self.subnet = []
for _subnet in config_dict['subnet']: for _subnet in config_dict['subnet']:
self.subnet.append(_parse_ip(_subnet)) self.subnet.append(_parse_ip(_subnet))
if 'routes' in config_dict: if 'routes' in config_dict:
self.routes = []
for _route in config_dict['routes']: for _route in config_dict['routes']:
self.routes.append(_parse_ip(_route)) self.routes.append(_parse_ip(_route))
if 'gateway' in config_dict: if 'gateway' in config_dict:
@ -436,6 +459,7 @@ class SiteInfo(object):
* url (``str``) * url (``str``)
""" """
self.name = name self.name = name
self.region_name = kwargs.get('region_name', DATA_DEFAULT)
self.physical_location_id = kwargs.get( self.physical_location_id = kwargs.get(
'physical_location_id', DATA_DEFAULT) 'physical_location_id', DATA_DEFAULT)
self.state = kwargs.get('state', DATA_DEFAULT) self.state = kwargs.get('state', DATA_DEFAULT)
@ -443,8 +467,13 @@ class SiteInfo(object):
self.corridor = kwargs.get('corridor', DATA_DEFAULT) self.corridor = kwargs.get('corridor', DATA_DEFAULT)
self.sitetype = kwargs.get('sitetype', DATA_DEFAULT) self.sitetype = kwargs.get('sitetype', DATA_DEFAULT)
self.dns = ServerList(kwargs.get('dns', [])) self.dns = kwargs.get('dns', [])
self.ntp = ServerList(kwargs.get('ntp', [])) if type(self.dns) is not ServerList:
self.dns = ServerList(self.dns)
self.ntp = kwargs.get('ntp', [])
if type(self.ntp) is not ServerList:
self.ntp = ServerList(self.ntp)
self.domain = kwargs.get('domain', DATA_DEFAULT) self.domain = kwargs.get('domain', DATA_DEFAULT)
self.ldap = kwargs.get('ldap', {}) self.ldap = kwargs.get('ldap', {})
@ -456,11 +485,15 @@ class SiteInfo(object):
return { return {
'corridor': self.corridor, 'corridor': self.corridor,
'country': self.country, 'country': self.country,
'dns': str(self.dns), 'dns': {
'servers': str(self.dns)
},
'domain': self.domain, 'domain': self.domain,
'ldap': self.ldap, 'ldap': self.ldap,
'name': self.name, 'name': self.name,
'ntp': str(self.ntp), 'ntp': {
'servers': str(self.ntp)
},
'physical_location_id': self.physical_location_id, 'physical_location_id': self.physical_location_id,
'sitetype': self.sitetype, 'sitetype': self.sitetype,
'state': self.state, 'state': self.state,
@ -480,9 +513,9 @@ class SiteInfo(object):
if 'sitetype' in config_dict: if 'sitetype' in config_dict:
self.sitetype = config_dict['sitetype'] self.sitetype = config_dict['sitetype']
if 'dns' in config_dict: if 'dns' in config_dict:
self.dns.merge(config_dict['dns']['servers']) self.dns = ServerList(config_dict['dns']['servers'])
if 'ntp' in config_dict: if 'ntp' in config_dict:
self.ntp.merge(config_dict['ntp']['servers']) self.ntp = ServerList(config_dict['ntp']['servers'])
if 'domain' in config_dict: if 'domain' in config_dict:
self.domain = config_dict['domain'] self.domain = config_dict['domain']
if 'ldap' in config_dict: if 'ldap' in config_dict:
@ -522,6 +555,7 @@ class SiteDocumentData(object):
document = { document = {
'baremetal': {}, 'baremetal': {},
'network': self.network.dict_from_class(), 'network': self.network.dict_from_class(),
'region_name': self.site_info.region_name,
'site_info': self.site_info.dict_from_class(), 'site_info': self.site_info.dict_from_class(),
'storage': self.storage 'storage': self.storage
} }

16
spyglass/examples/templates/pki/pki-catalogue.yaml.j2
View File

@ -30,7 +30,7 @@ data:
hosts: hosts:
- {{ host }} - {{ host }}
- {{ data['baremetal'][racks][host]['ip']['oam'] }} - {{ data['baremetal'][racks][host]['ip']['oam'] }}
- {{ data['baremetal'][racks][host]['ip']['ksn']}} - {{ data['baremetal'][racks][host]['ip']['calico']}}
groups: groups:
- system:nodes - system:nodes
{% endif %} {% endif %}
@ -43,7 +43,7 @@ data:
hosts: hosts:
- {{ host }} - {{ host }}
- {{ data['baremetal'][racks][host]['ip']['oam'] }} - {{ data['baremetal'][racks][host]['ip']['oam'] }}
- {{ data['baremetal'][racks][host]['ip']['ksn']}} - {{ data['baremetal'][racks][host]['ip']['calico']}}
groups: groups:
- system:nodes - system:nodes
{%endfor%} {%endfor%}
@ -80,7 +80,7 @@ data:
hosts: hosts:
- {{ host }} - {{ host }}
- {{ data['baremetal'][racks][host]['ip']['oam'] }} - {{ data['baremetal'][racks][host]['ip']['oam'] }}
- {{ data['baremetal'][racks][host]['ip']['ksn']}} - {{ data['baremetal'][racks][host]['ip']['calico']}}
- 127.0.0.1 - 127.0.0.1
- localhost - localhost
- kubernetes-etcd.kube-system.svc.cluster.local - kubernetes-etcd.kube-system.svc.cluster.local
@ -96,7 +96,7 @@ data:
hosts: hosts:
- {{ host }} - {{ host }}
- {{ data['baremetal'][racks][host]['ip']['oam'] }} - {{ data['baremetal'][racks][host]['ip']['oam'] }}
- {{ data['baremetal'][racks][host]['ip']['ksn']}} - {{ data['baremetal'][racks][host]['ip']['calico']}}
- 127.0.0.1 - 127.0.0.1
- localhost - localhost
- kubernetes-etcd.kube-system.svc.cluster.local - kubernetes-etcd.kube-system.svc.cluster.local
@ -114,7 +114,7 @@ data:
hosts: hosts:
- {{ host }} - {{ host }}
- {{ data['baremetal'][racks][host]['ip']['oam'] }} - {{ data['baremetal'][racks][host]['ip']['oam'] }}
- {{ data['baremetal'][racks][host]['ip']['ksn']}} - {{ data['baremetal'][racks][host]['ip']['calico']}}
- 127.0.0.1 - 127.0.0.1
- localhost - localhost
- kubernetes-etcd.kube-system.svc.cluster.local - kubernetes-etcd.kube-system.svc.cluster.local
@ -130,7 +130,7 @@ data:
hosts: hosts:
- {{ host }} - {{ host }}
- {{ data['baremetal'][racks][host]['ip']['oam'] }} - {{ data['baremetal'][racks][host]['ip']['oam'] }}
- {{ data['baremetal'][racks][host]['ip']['ksn']}} - {{ data['baremetal'][racks][host]['ip']['calico']}}
- 127.0.0.1 - 127.0.0.1
- localhost - localhost
- kubernetes-etcd.kube-system.svc.cluster.local - kubernetes-etcd.kube-system.svc.cluster.local
@ -152,7 +152,7 @@ data:
hosts: hosts:
- {{ host }} - {{ host }}
- {{ data['baremetal'][racks][host]['ip']['oam'] }} - {{ data['baremetal'][racks][host]['ip']['oam'] }}
- {{ data['baremetal'][racks][host]['ip']['ksn']}} - {{ data['baremetal'][racks][host]['ip']['calico']}}
- 127.0.0.1 - 127.0.0.1
- localhost - localhost
- 10.96.232.136 - 10.96.232.136
@ -172,7 +172,7 @@ data:
hosts: hosts:
- {{ host }} - {{ host }}
- {{ data['baremetal'][racks][host]['ip']['oam'] }} - {{ data['baremetal'][racks][host]['ip']['oam'] }}
- {{ data['baremetal'][racks][host]['ip']['ksn']}} - {{ data['baremetal'][racks][host]['ip']['calico']}}
- 127.0.0.1 - 127.0.0.1
- localhost - localhost
- 10.96.232.136 - 10.96.232.136

128
spyglass/parser/engine.py
View File

@ -15,6 +15,7 @@
import copy import copy
import json import json
import logging import logging
import os
import pprint import pprint
import sys import sys
@ -62,15 +63,10 @@ class ProcessDataSource(object):
LOG.info("Extracting network subnets") LOG.info("Extracting network subnets")
network_subnets = {} network_subnets = {}
for net_type in self.data["network"]["vlan_network_data"]: for net_type in self.data.network.vlan_network_data:
# One of the type is ingress and we don't want that here # One of the type is ingress and we don't want that here
if net_type != "ingress": if net_type.name != "ingress":
network_subnets[net_type] = \ network_subnets[net_type.name] = IPNetwork(net_type.subnet[0])
IPNetwork(self.data["network"]
["vlan_network_data"]
[net_type]
["subnet"]
[0])
LOG.debug( LOG.debug(
"Network subnets:\n{}".format(pprint.pformat(network_subnets))) "Network subnets:\n{}".format(pprint.pformat(network_subnets)))
@ -78,17 +74,16 @@ class ProcessDataSource(object):
def _get_genesis_node_details(self): def _get_genesis_node_details(self):
# Get genesis host node details from the hosts based on host type # Get genesis host node details from the hosts based on host type
for racks in self.data["baremetal"].keys(): for rack in self.data.baremetal:
rack_hosts = self.data["baremetal"][racks] for host in rack.hosts:
for host in rack_hosts: if host.type == "genesis":
if rack_hosts[host]["type"] == "genesis": self.genesis_node = host
self.genesis_node = rack_hosts[host]
self.genesis_node["name"] = host
LOG.debug( LOG.debug(
"Genesis Node Details:\n{}".format( "Genesis Node Details:\n{}".format(
pprint.pformat(self.genesis_node))) pprint.pformat(self.genesis_node)))
def _validate_intermediary_data(self, data): @staticmethod
def _validate_intermediary_data(data):
"""Validates the intermediary data before generating manifests. """Validates the intermediary data before generating manifests.
It checks whether the data types and data format are as expected. It checks whether the data types and data format are as expected.
@ -101,7 +96,7 @@ class ProcessDataSource(object):
temp_data = copy.deepcopy(data) temp_data = copy.deepcopy(data)
# Converting baremetal dict to list. # Converting baremetal dict to list.
baremetal_list = [] baremetal_list = []
for rack in temp_data["baremetal"].keys(): for rack in temp_data.baremetal:
temp = [{k: v} for k, v in temp_data["baremetal"][rack].items()] temp = [{k: v} for k, v in temp_data["baremetal"][rack].items()]
baremetal_list = baremetal_list + temp baremetal_list = baremetal_list + temp
@ -175,23 +170,22 @@ class ProcessDataSource(object):
""" """
is_genesis = False is_genesis = False
hardware_profile = rule_data[self.data["site_info"]["sitetype"]] hardware_profile = rule_data[self.data.site_info.sitetype]
# Getting individual racks. The racks are sorted to ensure that the # Getting individual racks. The racks are sorted to ensure that the
# first controller of the first rack is assigned as 'genesis' node. # first controller of the first rack is assigned as 'genesis' node.
for rack in sorted(self.data["baremetal"].keys()): for rack in sorted(self.data.baremetal, key=lambda x: x.name):
# Getting individual hosts in each rack. Sorting of the hosts are # Getting individual hosts in each rack. Sorting of the hosts are
# done to determine the genesis node. # done to determine the genesis node.
for host in sorted(self.data["baremetal"][rack].keys()): for host in sorted(rack.hosts, key=lambda x: x.name):
host_info = self.data["baremetal"][rack][host] if host.host_profile == \
if host_info["host_profile"] \ hardware_profile["profile_name"]["ctrl"]:
== hardware_profile["profile_name"]["ctrl"]:
if not is_genesis: if not is_genesis:
host_info["type"] = "genesis" host.type = "genesis"
is_genesis = True is_genesis = True
else: else:
host_info["type"] = "controller" host.type = "controller"
else: else:
host_info["type"] = "compute" host.type = "compute"
def _apply_rule_ip_alloc_offset(self, rule_data): def _apply_rule_ip_alloc_offset(self, rule_data):
"""Apply offset rules to update baremetal host """Apply offset rules to update baremetal host
@ -218,18 +212,13 @@ class ProcessDataSource(object):
host_idx = 0 host_idx = 0
LOG.info("Update baremetal host ip's") LOG.info("Update baremetal host ip's")
for racks in self.data["baremetal"].keys(): for rack in self.data.baremetal:
rack_hosts = self.data["baremetal"][racks] for host in rack.hosts:
for host in rack_hosts: for net_type, net_ip in iter(host.ip):
host_networks = rack_hosts[host]["ip"] ips = list(self.network_subnets[net_type])
for net in host_networks: host.ip.set_ip_by_role(
ips = list(self.network_subnets[net]) net_type, str(ips[host_idx + default_ip_offset]))
host_networks[net] = str(ips[host_idx + default_ip_offset]) host_idx += 1
host_idx = host_idx + 1
LOG.debug(
"Updated baremetal host:\n{}".format(
pprint.pformat(self.data["baremetal"])))
def _update_vlan_net_data(self, rule_data): def _update_vlan_net_data(self, rule_data):
"""Offset allocation rules to determine ip address range(s) """Offset allocation rules to determine ip address range(s)
@ -252,22 +241,22 @@ class ProcessDataSource(object):
# Set ingress vip and CIDR for bgp # Set ingress vip and CIDR for bgp
LOG.info("Apply network design rules:bgp") LOG.info("Apply network design rules:bgp")
vlan_network_data_ = self.data["network"]["vlan_network_data"] ingress_data = self.data.network.get_vlan_data_by_name('ingress')
subnet = IPNetwork(vlan_network_data_["ingress"]["subnet"][0]) subnet = IPNetwork(ingress_data.subnet[0])
ips = list(subnet) ips = list(subnet)
self.data["network"]["bgp"]["ingress_vip"] = \ self.data.network.bgp["ingress_vip"] = \
str(ips[ingress_vip_offset]) str(ips[ingress_vip_offset])
self.data["network"]["bgp"]["public_service_cidr"] = \ self.data.network.bgp["public_service_cidr"] = \
(vlan_network_data_["ingress"] ingress_data.subnet[0]
["subnet"]
[0])
LOG.debug( LOG.debug(
"Updated network bgp data:\n{}".format( "Updated network bgp data:\n{}".format(
pprint.pformat(self.data["network"]["bgp"]))) pprint.pformat(self.data.network.bgp)))
LOG.info("Apply network design rules:vlan") LOG.info("Apply network design rules:vlan")
# Apply rules to vlan networks # Apply rules to vlan networks
for net_type in self.network_subnets: for net_type in self.network_subnets.keys():
vlan_network_data_ = \
self.data.network.get_vlan_data_by_name(net_type)
if net_type == "oob": if net_type == "oob":
ip_offset = oob_ip_offset ip_offset = oob_ip_offset
else: else:
@ -276,11 +265,10 @@ class ProcessDataSource(object):
subnet = self.network_subnets[net_type] subnet = self.network_subnets[net_type]
ips = list(subnet) ips = list(subnet)
vlan_network_data_[net_type]["gateway"] = \ vlan_network_data_.gateway = str(ips[gateway_ip_offset])
str(ips[gateway_ip_offset])
vlan_network_data_[net_type]["reserved_start"] = str(ips[1]) vlan_network_data_.reserved_start = str(ips[1])
vlan_network_data_[net_type]["reserved_end"] = str(ips[ip_offset]) vlan_network_data_.reserved_end = str(ips[ip_offset])
static_start = str(ips[ip_offset + 1]) static_start = str(ips[ip_offset + 1])
static_end = str(ips[static_ip_end_offset]) static_end = str(ips[static_ip_end_offset])
@ -291,27 +279,21 @@ class ProcessDataSource(object):
dhcp_start = str(ips[mid]) dhcp_start = str(ips[mid])
dhcp_end = str(ips[dhcp_ip_end_offset]) dhcp_end = str(ips[dhcp_ip_end_offset])
vlan_network_data_[net_type]["dhcp_start"] = dhcp_start vlan_network_data_.dhcp_start = dhcp_start
vlan_network_data_[net_type]["dhcp_end"] = dhcp_end vlan_network_data_.dhcp_end = dhcp_end
vlan_network_data_[net_type]["static_start"] = static_start vlan_network_data_.static_start = static_start
vlan_network_data_[net_type]["static_end"] = static_end vlan_network_data_.static_end = static_end
# There is no vlan for oob network
if net_type != "oob":
vlan_network_data_[net_type]["vlan"] = \
vlan_network_data_[net_type]["vlan"]
# OAM have default routes. Only for cruiser. TBD # OAM have default routes. Only for cruiser. TBD
if net_type == "oam": if net_type == "oam":
routes = ["0.0.0.0/0"] # nosec vlan_network_data_.routes = ["0.0.0.0/0"] # nosec
else: else:
routes = [] vlan_network_data_.routes = []
vlan_network_data_[net_type]["routes"] = routes
LOG.debug( LOG.debug(
"Updated vlan network data:\n{}".format( "Updated vlan network data:\n{}".format(
pprint.pformat(vlan_network_data_))) pprint.pformat(vlan_network_data_.dict_from_class())))
def load_extracted_data_from_data_source(self, extracted_data): def load_extracted_data_from_data_source(self, extracted_data):
"""Function called from cli.py to pass extracted data """Function called from cli.py to pass extracted data
@ -319,18 +301,12 @@ class ProcessDataSource(object):
from input data source from input data source
""" """
# TBR(pg710r): for internal testing
"""
raw_data = self._read_file('extracted_data.yaml')
extracted_data = yaml.safe_load(raw_data)
"""
LOG.info("Loading plugin data source") LOG.info("Loading plugin data source")
self.data = extracted_data self.data = extracted_data
LOG.debug( LOG.debug(
"Extracted data from plugin:\n{}".format( "Extracted data from plugin:\n{}".format(
pprint.pformat(extracted_data))) pprint.pformat(extracted_data)))
# Uncomment following segment for debugging purpose. # Uncomment following segment for debugging purpose.
# extracted_file = "extracted_file.yaml" # extracted_file = "extracted_file.yaml"
# yaml_file = yaml.dump(extracted_data, default_flow_style=False) # yaml_file = yaml.dump(extracted_data, default_flow_style=False)
@ -338,22 +314,20 @@ class ProcessDataSource(object):
# f.write(yaml_file) # f.write(yaml_file)
# f.close() # f.close()
# Append region_data supplied from CLI to self.data
self.data["region_name"] = self.region_name
def dump_intermediary_file(self, intermediary_dir): def dump_intermediary_file(self, intermediary_dir):
"""Writing intermediary yaml""" """Writing intermediary yaml"""
LOG.info("Writing intermediary yaml") LOG.info("Writing intermediary yaml")
intermediary_file = "{}_intermediary.yaml" \ intermediary_file = "{}_intermediary.yaml" \
.format(self.data["region_name"]) .format(self.region_name)
# Check of if output dir = intermediary_dir exists # Check of if output dir = intermediary_dir exists
if intermediary_dir is not None: if intermediary_dir is not None:
outfile = "{}/{}".format(intermediary_dir, intermediary_file) outfile = os.path.join(intermediary_dir, intermediary_file)
else: else:
outfile = intermediary_file outfile = intermediary_file
LOG.info("Intermediary file:{}".format(outfile)) LOG.info("Intermediary file:{}".format(outfile))
yaml_file = yaml.dump(self.data, default_flow_style=False) yaml_file = yaml.dump(
self.data.dict_from_class(), default_flow_style=False)
with open(outfile, "w") as f: with open(outfile, "w") as f:
f.write(yaml_file) f.write(yaml_file)
f.close() f.close()
@ -365,5 +339,5 @@ class ProcessDataSource(object):
self._apply_design_rules() self._apply_design_rules()
self._get_genesis_node_details() self._get_genesis_node_details()
# This will validate the extracted data from different sources. # This will validate the extracted data from different sources.
self._validate_intermediary_data(self.data) # self._validate_intermediary_data(self.data)
return self.data return self.data

0
spyglass/utils/__init__.py
View File

40
spyglass/utils/utils.py
View File

@ -1,40 +0,0 @@
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Merge two dictionaries
def dict_merge(dict_a, dict_b, path=None):
"""Recursively Merge dictionary dictB into dictA
DictA represents the data extracted by a plugin and DictB
represents the additional site config dictionary that is passed
to CLI. The merge process compares the dictionary keys and if they
are same and the values they point to are different , then
dictB object's value is copied to dictA. If a key is unique
to dictB, then it is copied to dictA.
"""
if path is None:
path = []
for key in dict_b:
if key in dict_a:
if isinstance(dict_a[key], dict) and isinstance(dict_b[key], dict):
dict_merge(dict_a[key], dict_b[key], path + [str(key)])
elif dict_a[key] == dict_b[key]:
pass # values are same, so no processing here
else:
dict_a[key] = dict_b[key]
else:
dict_a[key] = dict_b[key]
return dict_a

29
tests/unit/data_extractor/test_models.py
View File

@ -178,13 +178,7 @@ class TestIPList(unittest.TestCase):
IPList should automatically fill in any missing entries with the value IPList should automatically fill in any missing entries with the value
set by models.DATA_DEFAULT. set by models.DATA_DEFAULT.
""" """
expected_message = \ result = models.IPList(**self.MISSING_IP)
'%s is not a valid IP address.' % models.DATA_DEFAULT
with self.assertLogs(level='WARNING') as test_log:
result = models.IPList(**self.MISSING_IP)
self.assertEqual(len(test_log.output), 1)
self.assertEqual(len(test_log.records), 1)
self.assertIn(expected_message, test_log.output[0])
self.assertEqual(self.MISSING_IP['oob'], result.oob) self.assertEqual(self.MISSING_IP['oob'], result.oob)
self.assertEqual(models.DATA_DEFAULT, result.oam) self.assertEqual(models.DATA_DEFAULT, result.oam)
self.assertEqual(self.MISSING_IP['calico'], result.calico) self.assertEqual(self.MISSING_IP['calico'], result.calico)
@ -548,6 +542,8 @@ class TestNetwork(unittest.TestCase):
def test_dict_from_class(self): def test_dict_from_class(self):
"""Tests production of a dictionary from a Network object""" """Tests production of a dictionary from a Network object"""
oob = copy(self.VLAN_DATA)
oob.pop('vlan')
expected_result = { expected_result = {
'bgp': self.BGP_DATA, 'bgp': self.BGP_DATA,
'vlan_network_data': { 'vlan_network_data': {
@ -555,7 +551,7 @@ class TestNetwork(unittest.TestCase):
**self.VLAN_DATA **self.VLAN_DATA
}, },
'oob': { 'oob': {
**self.VLAN_DATA **oob
}, },
'pxe': { 'pxe': {
**self.VLAN_DATA **self.VLAN_DATA
@ -597,12 +593,12 @@ class TestNetwork(unittest.TestCase):
result = models.Network(self.vlan_network_data, bgp=self.BGP_DATA) result = models.Network(self.vlan_network_data, bgp=self.BGP_DATA)
self.assertEqual( self.assertEqual(
self.VLAN_DATA['vlan'], self.VLAN_DATA['vlan'],
result.get_vlan_data_by_name('oob').vlan) result.get_vlan_data_by_name('oam').vlan)
new_vlan_data = {'vlan_network_data': {'oob': {'vlan': '12'}}} new_vlan_data = {'vlan_network_data': {'oam': {'vlan': '12'}}}
result.merge_additional_data(new_vlan_data) result.merge_additional_data(new_vlan_data)
self.assertEqual( self.assertEqual(
new_vlan_data['vlan_network_data']['oob']['vlan'], new_vlan_data['vlan_network_data']['oam']['vlan'],
result.get_vlan_data_by_name('oob').vlan) result.get_vlan_data_by_name('oam').vlan)
def test_get_vlan_data_by_name(self): def test_get_vlan_data_by_name(self):
"""Tests retrieval of VLANNetworkData by name attribute""" """Tests retrieval of VLANNetworkData by name attribute"""
@ -686,8 +682,10 @@ class TestSiteInfo(unittest.TestCase):
def test_dict_from_class(self): def test_dict_from_class(self):
"""Tests production of a dictionary from a SiteInfo object""" """Tests production of a dictionary from a SiteInfo object"""
expected_results = copy(self.SITE_INFO) expected_results = copy(self.SITE_INFO)
expected_results['dns'] = ','.join(expected_results['dns']) expected_results['dns'] = \
expected_results['ntp'] = ','.join(expected_results['ntp']) {'servers': ','.join(expected_results['dns'])}
expected_results['ntp'] = \
{'servers': ','.join(expected_results['ntp'])}
expected_results['name'] = self.SITE_NAME expected_results['name'] = self.SITE_NAME
result = models.SiteInfo(self.SITE_NAME, **self.SITE_INFO) result = models.SiteInfo(self.SITE_NAME, **self.SITE_INFO)
self.assertDictEqual(expected_results, result.dict_from_class()) self.assertDictEqual(expected_results, result.dict_from_class())
@ -759,6 +757,8 @@ class TestSiteDocumentData(unittest.TestCase):
site_info = SiteInfo() site_info = SiteInfo()
site_info.dict_from_class.return_value = mock_site_info_data site_info.dict_from_class.return_value = mock_site_info_data
type(SiteInfo()).region_name = mock.PropertyMock(
return_value='region_name')
network = Network() network = Network()
network.dict_from_class.return_value = mock_network_data network.dict_from_class.return_value = mock_network_data
baremetal = [Rack(), Rack()] baremetal = [Rack(), Rack()]
@ -771,6 +771,7 @@ class TestSiteDocumentData(unittest.TestCase):
**mock_baremetal1_data **mock_baremetal1_data
}, },
'network': mock_network_data, 'network': mock_network_data,
'region_name': 'region_name',
'site_info': mock_site_info_data, 'site_info': mock_site_info_data,
'storage': self.STORAGE_DICT 'storage': self.STORAGE_DICT
} }