diff --git a/.zuul.yaml b/.zuul.yaml
index 115d77c..922d40b 100644
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -9,7 +9,10 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
- project:
- templates:
- - noop-jobs
+ check:
+ jobs:
+ - openstack-tox-pep8
+ gate:
+ jobs:
+ - openstack-tox-pep8
diff --git a/spyglass/data_extractor/base.py b/spyglass/data_extractor/base.py
index 550b9e6..955e7f8 100644
--- a/spyglass/data_extractor/base.py
+++ b/spyglass/data_extractor/base.py
@@ -281,8 +281,8 @@ class BaseDataSourcePlugin(object):
# For each host list fill host profile and network IPs
for host in hosts:
- host_name = host['name']
- rack_name = host['rack_name']
+ host_name = host["name"]
+ rack_name = host["rack_name"]
if rack_name not in baremetal:
baremetal[rack_name] = {}
@@ -290,32 +290,39 @@ class BaseDataSourcePlugin(object):
# Prepare temp dict for each host and append it to baremetal
# at a rack level
temp_host = {}
- if host['host_profile'] is None:
- temp_host['host_profile'] = "#CHANGE_ME"
+ if host["host_profile"] is None:
+ temp_host["host_profile"] = "#CHANGE_ME"
else:
- temp_host['host_profile'] = host['host_profile']
+ temp_host["host_profile"] = host["host_profile"]
# Get Host IPs from plugin
temp_host_ips = self.get_ips(self.region, host_name)
# Fill network IP for this host
- temp_host['ip'] = {}
- temp_host['ip']['oob'] = temp_host_ips[host_name].get(
- 'oob', "#CHANGE_ME")
- temp_host['ip']['calico'] = temp_host_ips[host_name].get(
- 'calico', "#CHANGE_ME")
- temp_host['ip']['oam'] = temp_host_ips[host_name].get(
- 'oam', "#CHANGE_ME")
- temp_host['ip']['storage'] = temp_host_ips[host_name].get(
- 'storage', "#CHANGE_ME")
- temp_host['ip']['overlay'] = temp_host_ips[host_name].get(
- 'overlay', "#CHANGE_ME")
- temp_host['ip']['pxe'] = temp_host_ips[host_name].get(
- 'pxe', "#CHANGE_ME")
+ temp_host["ip"] = {}
+ temp_host["ip"]["oob"] = temp_host_ips[host_name].get(
+ "oob", "#CHANGE_ME"
+ )
+ temp_host["ip"]["calico"] = temp_host_ips[host_name].get(
+ "calico", "#CHANGE_ME"
+ )
+ temp_host["ip"]["oam"] = temp_host_ips[host_name].get(
+ "oam", "#CHANGE_ME"
+ )
+ temp_host["ip"]["storage"] = temp_host_ips[host_name].get(
+ "storage", "#CHANGE_ME"
+ )
+ temp_host["ip"]["overlay"] = temp_host_ips[host_name].get(
+ "overlay", "#CHANGE_ME"
+ )
+ temp_host["ip"]["pxe"] = temp_host_ips[host_name].get(
+ "pxe", "#CHANGE_ME"
+ )
baremetal[rack_name][host_name] = temp_host
- LOG.debug("Baremetal information:\n{}".format(
- pprint.pformat(baremetal)))
+ LOG.debug(
+ "Baremetal information:\n{}".format(pprint.pformat(baremetal))
+ )
return baremetal
@@ -348,19 +355,20 @@ class BaseDataSourcePlugin(object):
site_info = location_data
dns_data = self.get_dns_servers(self.region)
- site_info['dns'] = dns_data
+ site_info["dns"] = dns_data
ntp_data = self.get_ntp_servers(self.region)
- site_info['ntp'] = ntp_data
+ site_info["ntp"] = ntp_data
ldap_data = self.get_ldap_information(self.region)
- site_info['ldap'] = ldap_data
+ site_info["ldap"] = ldap_data
domain_data = self.get_domain_name(self.region)
- site_info['domain'] = domain_data
+ site_info["domain"] = domain_data
- LOG.debug("Extracted site information:\n{}".format(
- pprint.pformat(site_info)))
+ LOG.debug(
+ "Extracted site information:\n{}".format(pprint.pformat(site_info))
+ )
return site_info
@@ -393,21 +401,28 @@ class BaseDataSourcePlugin(object):
# networks_to_scan, so look for these networks from the data
# returned by plugin
networks_to_scan = [
- 'calico', 'overlay', 'pxe', 'storage', 'oam', 'oob', 'ingress'
+ "calico",
+ "overlay",
+ "pxe",
+ "storage",
+ "oam",
+ "oob",
+ "ingress",
]
- network_data['vlan_network_data'] = {}
+ network_data["vlan_network_data"] = {}
for net in networks:
tmp_net = {}
- if net['name'] in networks_to_scan:
- tmp_net['subnet'] = net.get('subnet', '#CHANGE_ME')
- if ((net['name'] != 'ingress') and (net['name'] != 'oob')):
- tmp_net['vlan'] = net.get('vlan', '#CHANGE_ME')
+ if net["name"] in networks_to_scan:
+ tmp_net["subnet"] = net.get("subnet", "#CHANGE_ME")
+ if (net["name"] != "ingress") and (net["name"] != "oob"):
+ tmp_net["vlan"] = net.get("vlan", "#CHANGE_ME")
- network_data['vlan_network_data'][net['name']] = tmp_net
+ network_data["vlan_network_data"][net["name"]] = tmp_net
- LOG.debug("Extracted network data:\n{}".format(
- pprint.pformat(network_data)))
+ LOG.debug(
+ "Extracted network data:\n{}".format(pprint.pformat(network_data))
+ )
return network_data
def extract_data(self):
@@ -418,9 +433,9 @@ class BaseDataSourcePlugin(object):
"""
LOG.info("Extract data from plugin")
site_data = {}
- site_data['baremetal'] = self.extract_baremetal_information()
- site_data['site_info'] = self.extract_site_information()
- site_data['network'] = self.extract_network_information()
+ site_data["baremetal"] = self.extract_baremetal_information()
+ site_data["site_info"] = self.extract_site_information()
+ site_data["network"] = self.extract_network_information()
self.site_data = site_data
return site_data
diff --git a/spyglass/data_extractor/custom_exceptions.py b/spyglass/data_extractor/custom_exceptions.py
index 40f67d6..44b0536 100644
--- a/spyglass/data_extractor/custom_exceptions.py
+++ b/spyglass/data_extractor/custom_exceptions.py
@@ -31,8 +31,11 @@ class NoSpecMatched(BaseError):
self.specs = excel_specs
def display_error(self):
- print('No spec matched. Following are the available specs:\n'.format(
- self.specs))
+ print(
+ "No spec matched. Following are the available specs:\n".format(
+ self.specs
+ )
+ )
sys.exit(1)
diff --git a/spyglass/data_extractor/plugins/formation.py b/spyglass/data_extractor/plugins/formation.py
index c91a8fb..d149757 100644
--- a/spyglass/data_extractor/plugins/formation.py
+++ b/spyglass/data_extractor/plugins/formation.py
@@ -22,8 +22,11 @@ import urllib3
from spyglass.data_extractor.base import BaseDataSourcePlugin
from spyglass.data_extractor.custom_exceptions import (
- ApiClientError, ConnectionError, MissingAttributeError,
- TokenGenerationError)
+ ApiClientError,
+ ConnectionError,
+ MissingAttributeError,
+ TokenGenerationError,
+)
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
@@ -41,8 +44,8 @@ class FormationPlugin(BaseDataSourcePlugin):
LOG.info("Check spyglass --help for details")
exit()
- self.source_type = 'rest'
- self.source_name = 'formation'
+ self.source_type = "rest"
+ self.source_name = "formation"
# Configuration parameters
self.formation_api_url = None
@@ -67,10 +70,10 @@ class FormationPlugin(BaseDataSourcePlugin):
""" Sets the config params passed by CLI"""
LOG.info("Plugin params passed:\n{}".format(pprint.pformat(conf)))
self._validate_config_options(conf)
- self.formation_api_url = conf['url']
- self.user = conf['user']
- self.password = conf['password']
- self.token = conf.get('token', None)
+ self.formation_api_url = conf["url"]
+ self.user = conf["user"]
+ self.password = conf["password"]
+ self.token = conf.get("token", None)
self._get_formation_client()
self._update_site_and_zone(self.region)
@@ -78,21 +81,24 @@ class FormationPlugin(BaseDataSourcePlugin):
def get_plugin_conf(self, kwargs):
""" Validates the plugin param and return if success"""
try:
- assert (kwargs['formation_url']
- ) is not None, "formation_url is Not Specified"
- url = kwargs['formation_url']
- assert (kwargs['formation_user']
- ) is not None, "formation_user is Not Specified"
- user = kwargs['formation_user']
- assert (kwargs['formation_password']
- ) is not None, "formation_password is Not Specified"
- password = kwargs['formation_password']
+ assert (
+ kwargs["formation_url"]
+ ) is not None, "formation_url is Not Specified"
+ url = kwargs["formation_url"]
+ assert (
+ kwargs["formation_user"]
+ ) is not None, "formation_user is Not Specified"
+ user = kwargs["formation_user"]
+ assert (
+ kwargs["formation_password"]
+ ) is not None, "formation_password is Not Specified"
+ password = kwargs["formation_password"]
except AssertionError:
LOG.error("Insufficient plugin parameter! Spyglass exited!")
raise
exit()
- plugin_conf = {'url': url, 'user': user, 'password': password}
+ plugin_conf = {"url": url, "user": user, "password": password}
return plugin_conf
def _validate_config_options(self, conf):
@@ -129,21 +135,24 @@ class FormationPlugin(BaseDataSourcePlugin):
if self.token:
return self.token
- url = self.formation_api_url + '/zones'
+ url = self.formation_api_url + "/zones"
try:
token_response = requests.get(
url,
auth=(self.user, self.password),
- verify=self.client_config.verify_ssl)
+ verify=self.client_config.verify_ssl,
+ )
except requests.exceptions.ConnectionError:
- raise ConnectionError('Incorrect URL: {}'.format(url))
+ raise ConnectionError("Incorrect URL: {}".format(url))
if token_response.status_code == 200:
- self.token = token_response.json().get('X-Subject-Token', None)
+ self.token = token_response.json().get("X-Subject-Token", None)
else:
raise TokenGenerationError(
- 'Unable to generate token because {}'.format(
- token_response.reason))
+ "Unable to generate token because {}".format(
+ token_response.reason
+ )
+ )
return self.token
@@ -155,9 +164,10 @@ class FormationPlugin(BaseDataSourcePlugin):
Generate the token and add it formation config object.
"""
token = self._generate_token()
- self.client_config.api_key = {'X-Auth-Token': self.user + '|' + token}
+ self.client_config.api_key = {"X-Auth-Token": self.user + "|" + token}
self.formation_api_client = formation_client.ApiClient(
- self.client_config)
+ self.client_config
+ )
def _update_site_and_zone(self, region):
"""Get Zone name and Site name from region"""
@@ -169,8 +179,8 @@ class FormationPlugin(BaseDataSourcePlugin):
# site = zone[:-1]
self.region_zone_map[region] = {}
- self.region_zone_map[region]['zone'] = zone
- self.region_zone_map[region]['site'] = site
+ self.region_zone_map[region]["zone"] = zone
+ self.region_zone_map[region]["site"] = site
def _get_zone_by_region_name(self, region_name):
zone_api = formation_client.ZonesApi(self.formation_api_client)
@@ -248,7 +258,7 @@ class FormationPlugin(BaseDataSourcePlugin):
return self.device_name_id_mapping.get(device_name, None)
- def _get_racks(self, zone, rack_type='compute'):
+ def _get_racks(self, zone, rack_type="compute"):
zone_id = self._get_zone_id_by_name(zone)
rack_api = formation_client.RacksApi(self.formation_api_client)
racks = rack_api.zones_zone_id_racks_get(zone_id)
@@ -296,35 +306,40 @@ class FormationPlugin(BaseDataSourcePlugin):
# Implement Abstract functions
def get_racks(self, region):
- zone = self.region_zone_map[region]['zone']
- return self._get_racks(zone, rack_type='compute')
+ zone = self.region_zone_map[region]["zone"]
+ return self._get_racks(zone, rack_type="compute")
def get_hosts(self, region, rack=None):
- zone = self.region_zone_map[region]['zone']
+ zone = self.region_zone_map[region]["zone"]
zone_id = self._get_zone_id_by_name(zone)
device_api = formation_client.DevicesApi(self.formation_api_client)
control_hosts = device_api.zones_zone_id_control_nodes_get(zone_id)
compute_hosts = device_api.zones_zone_id_devices_get(
- zone_id, type='KVM')
+ zone_id, type="KVM"
+ )
hosts_list = []
for host in control_hosts:
self.device_name_id_mapping[host.aic_standard_name] = host.id
- hosts_list.append({
- 'name': host.aic_standard_name,
- 'type': 'controller',
- 'rack_name': host.rack_name,
- 'host_profile': host.host_profile_name
- })
+ hosts_list.append(
+ {
+ "name": host.aic_standard_name,
+ "type": "controller",
+ "rack_name": host.rack_name,
+ "host_profile": host.host_profile_name,
+ }
+ )
for host in compute_hosts:
self.device_name_id_mapping[host.aic_standard_name] = host.id
- hosts_list.append({
- 'name': host.aic_standard_name,
- 'type': 'compute',
- 'rack_name': host.rack_name,
- 'host_profile': host.host_profile_name
- })
+ hosts_list.append(
+ {
+ "name": host.aic_standard_name,
+ "type": "compute",
+ "rack_name": host.rack_name,
+ "host_profile": host.host_profile_name,
+ }
+ )
"""
for host in itertools.chain(control_hosts, compute_hosts):
self.device_name_id_mapping[host.aic_standard_name] = host.id
@@ -339,40 +354,43 @@ class FormationPlugin(BaseDataSourcePlugin):
return hosts_list
def get_networks(self, region):
- zone = self.region_zone_map[region]['zone']
+ zone = self.region_zone_map[region]["zone"]
zone_id = self._get_zone_id_by_name(zone)
region_id = self._get_region_id_by_name(region)
vlan_api = formation_client.VlansApi(self.formation_api_client)
vlans = vlan_api.zones_zone_id_regions_region_id_vlans_get(
- zone_id, region_id)
+ zone_id, region_id
+ )
# Case when vlans list is empty from
# zones_zone_id_regions_region_id_vlans_get
- if len(vlans) is 0:
+ if len(vlans) == 0:
# get device-id from the first host and get the network details
hosts = self.get_hosts(self.region)
- host = hosts[0]['name']
+ host = hosts[0]["name"]
device_id = self._get_device_id_by_name(host)
vlans = vlan_api.zones_zone_id_devices_device_id_vlans_get(
- zone_id, device_id)
+ zone_id, device_id
+ )
LOG.debug("Extracted region network information\n{}".format(vlans))
vlans_list = []
for vlan_ in vlans:
- if len(vlan_.vlan.ipv4) is not 0:
+ if len(vlan_.vlan.ipv4) != 0:
tmp_vlan = {}
- tmp_vlan['name'] = self._get_network_name_from_vlan_name(
- vlan_.vlan.name)
- tmp_vlan['vlan'] = vlan_.vlan.vlan_id
- tmp_vlan['subnet'] = vlan_.vlan.subnet_range
- tmp_vlan['gateway'] = vlan_.ipv4_gateway
- tmp_vlan['subnet_level'] = vlan_.vlan.subnet_level
+ tmp_vlan["name"] = self._get_network_name_from_vlan_name(
+ vlan_.vlan.name
+ )
+ tmp_vlan["vlan"] = vlan_.vlan.vlan_id
+ tmp_vlan["subnet"] = vlan_.vlan.subnet_range
+ tmp_vlan["gateway"] = vlan_.ipv4_gateway
+ tmp_vlan["subnet_level"] = vlan_.vlan.subnet_level
vlans_list.append(tmp_vlan)
return vlans_list
def get_ips(self, region, host=None):
- zone = self.region_zone_map[region]['zone']
+ zone = self.region_zone_map[region]["zone"]
zone_id = self._get_zone_id_by_name(zone)
if host:
@@ -381,7 +399,7 @@ class FormationPlugin(BaseDataSourcePlugin):
hosts = []
hosts_dict = self.get_hosts(zone)
for host in hosts_dict:
- hosts.append(host['name'])
+ hosts.append(host["name"])
vlan_api = formation_client.VlansApi(self.formation_api_client)
ip_ = {}
@@ -389,18 +407,23 @@ class FormationPlugin(BaseDataSourcePlugin):
for host in hosts:
device_id = self._get_device_id_by_name(host)
vlans = vlan_api.zones_zone_id_devices_device_id_vlans_get(
- zone_id, device_id)
+ zone_id, device_id
+ )
LOG.debug("Received VLAN Network Information\n{}".format(vlans))
ip_[host] = {}
for vlan_ in vlans:
# TODO(pg710r) We need to handle the case when incoming ipv4
# list is empty
- if len(vlan_.vlan.ipv4) is not 0:
+ if len(vlan_.vlan.ipv4) != 0:
name = self._get_network_name_from_vlan_name(
- vlan_.vlan.name)
+ vlan_.vlan.name
+ )
ipv4 = vlan_.vlan.ipv4[0].ip
- LOG.debug("vlan:{},name:{},ip:{},vlan_name:{}".format(
- vlan_.vlan.vlan_id, name, ipv4, vlan_.vlan.name))
+ LOG.debug(
+ "vlan:{},name:{},ip:{},vlan_name:{}".format(
+ vlan_.vlan.vlan_id, name, ipv4, vlan_.vlan.name
+ )
+ )
# TODD(pg710r) This code needs to extended to support ipv4
# and ipv6
# ip_[host][name] = {'ipv4': ipv4}
@@ -419,12 +442,12 @@ class FormationPlugin(BaseDataSourcePlugin):
vlan_name contains "ILO" the network name is "oob"
"""
network_names = {
- 'ksn': 'calico',
- 'storage': 'storage',
- 'server': 'oam',
- 'ovs': 'overlay',
- 'ILO': 'oob',
- 'pxe': 'pxe'
+ "ksn": "calico",
+ "storage": "storage",
+ "server": "oam",
+ "ovs": "overlay",
+ "ILO": "oob",
+ "pxe": "pxe",
}
for name in network_names:
@@ -438,7 +461,7 @@ class FormationPlugin(BaseDataSourcePlugin):
def get_dns_servers(self, region):
try:
- zone = self.region_zone_map[region]['zone']
+ zone = self.region_zone_map[region]["zone"]
zone_id = self._get_zone_id_by_name(zone)
zone_api = formation_client.ZonesApi(self.formation_api_client)
zone_ = zone_api.zones_zone_id_get(zone_id)
@@ -463,7 +486,7 @@ class FormationPlugin(BaseDataSourcePlugin):
def get_location_information(self, region):
""" get location information for a zone and return """
- site = self.region_zone_map[region]['site']
+ site = self.region_zone_map[region]["site"]
site_id = self._get_site_id_by_name(site)
site_api = formation_client.SitesApi(self.formation_api_client)
site_info = site_api.sites_site_id_get(site_id)
@@ -471,18 +494,19 @@ class FormationPlugin(BaseDataSourcePlugin):
try:
return {
# 'corridor': site_info.corridor,
- 'name': site_info.city,
- 'state': site_info.state,
- 'country': site_info.country,
- 'physical_location_id': site_info.clli,
+ "name": site_info.city,
+ "state": site_info.state,
+ "country": site_info.country,
+ "physical_location_id": site_info.clli,
}
except AttributeError as e:
- raise MissingAttributeError('Missing {} information in {}'.format(
- e, site_info.city))
+ raise MissingAttributeError(
+ "Missing {} information in {}".format(e, site_info.city)
+ )
def get_domain_name(self, region):
try:
- zone = self.region_zone_map[region]['zone']
+ zone = self.region_zone_map[region]["zone"]
zone_id = self._get_zone_id_by_name(zone)
zone_api = formation_client.ZonesApi(self.formation_api_client)
zone_ = zone_api.zones_zone_id_get(zone_id)
@@ -490,7 +514,7 @@ class FormationPlugin(BaseDataSourcePlugin):
raise ApiClientError(e.msg)
if not zone_.dns:
- LOG.warn('Got None while running get domain name')
+ LOG.warn("Got None while running get domain name")
return None
return zone_.dns
diff --git a/spyglass/data_extractor/plugins/tugboat/check_exceptions.py b/spyglass/data_extractor/plugins/tugboat/check_exceptions.py
index d11d58a..a077af4 100644
--- a/spyglass/data_extractor/plugins/tugboat/check_exceptions.py
+++ b/spyglass/data_extractor/plugins/tugboat/check_exceptions.py
@@ -23,7 +23,7 @@ class NotEnoughIp(BaseError):
self.total_nodes = total_nodes
def display_error(self):
- print('{} can not handle {} nodes'.format(self.cidr, self.total_nodes))
+ print("{} can not handle {} nodes".format(self.cidr, self.total_nodes))
class NoSpecMatched(BaseError):
@@ -31,5 +31,8 @@ class NoSpecMatched(BaseError):
self.specs = excel_specs
def display_error(self):
- print('No spec matched. Following are the available specs:\n'.format(
- self.specs))
+ print(
+ "No spec matched. Following are the available specs:\n".format(
+ self.specs
+ )
+ )
diff --git a/spyglass/data_extractor/plugins/tugboat/excel_parser.py b/spyglass/data_extractor/plugins/tugboat/excel_parser.py
index f7b78c1..47559f5 100644
--- a/spyglass/data_extractor/plugins/tugboat/excel_parser.py
+++ b/spyglass/data_extractor/plugins/tugboat/excel_parser.py
@@ -20,17 +20,18 @@ import yaml
from openpyxl import load_workbook
from openpyxl import Workbook
from spyglass.data_extractor.custom_exceptions import NoSpecMatched
+
# from spyglass.data_extractor.custom_exceptions
LOG = logging.getLogger(__name__)
-class ExcelParser():
+class ExcelParser:
""" Parse data from excel into a dict """
def __init__(self, file_name, excel_specs):
self.file_name = file_name
- with open(excel_specs, 'r') as f:
+ with open(excel_specs, "r") as f:
spec_raw_data = f.read()
self.excel_specs = yaml.safe_load(spec_raw_data)
# A combined design spec, returns a workbok object after combining
@@ -38,12 +39,12 @@ class ExcelParser():
combined_design_spec = self.combine_excel_design_specs(file_name)
self.wb_combined = combined_design_spec
self.filenames = file_name
- self.spec = 'xl_spec'
+ self.spec = "xl_spec"
@staticmethod
def sanitize(string):
""" Remove extra spaces and convert string to lower case """
- return string.replace(' ', '').lower()
+ return string.replace(" ", "").lower()
def compare(self, string1, string2):
""" Compare the strings """
@@ -52,19 +53,19 @@ class ExcelParser():
def validate_sheet(self, spec, sheet):
""" Check if the sheet is correct or not """
ws = self.wb_combined[sheet]
- header_row = self.excel_specs['specs'][spec]['header_row']
- ipmi_header = self.excel_specs['specs'][spec]['ipmi_address_header']
- ipmi_column = self.excel_specs['specs'][spec]['ipmi_address_col']
+ header_row = self.excel_specs["specs"][spec]["header_row"]
+ ipmi_header = self.excel_specs["specs"][spec]["ipmi_address_header"]
+ ipmi_column = self.excel_specs["specs"][spec]["ipmi_address_col"]
header_value = ws.cell(row=header_row, column=ipmi_column).value
return bool(self.compare(ipmi_header, header_value))
def find_correct_spec(self):
""" Find the correct spec """
- for spec in self.excel_specs['specs']:
- sheet_name = self.excel_specs['specs'][spec]['ipmi_sheet_name']
+ for spec in self.excel_specs["specs"]:
+ sheet_name = self.excel_specs["specs"][spec]["ipmi_sheet_name"]
for sheet in self.wb_combined.sheetnames:
if self.compare(sheet_name, sheet):
- self.excel_specs['specs'][spec]['ipmi_sheet_name'] = sheet
+ self.excel_specs["specs"][spec]["ipmi_sheet_name"] = sheet
if self.validate_sheet(spec, sheet):
return spec
raise NoSpecMatched(self.excel_specs)
@@ -73,31 +74,37 @@ class ExcelParser():
""" Read IPMI data from the sheet """
ipmi_data = {}
hosts = []
- provided_sheetname = self.excel_specs['specs'][self.
- spec]['ipmi_sheet_name']
+ provided_sheetname = self.excel_specs["specs"][self.spec][
+ "ipmi_sheet_name"
+ ]
workbook_object, extracted_sheetname = self.get_xl_obj_and_sheetname(
- provided_sheetname)
+ provided_sheetname
+ )
if workbook_object is not None:
ws = workbook_object[extracted_sheetname]
else:
ws = self.wb_combined[provided_sheetname]
- row = self.excel_specs['specs'][self.spec]['start_row']
- end_row = self.excel_specs['specs'][self.spec]['end_row']
- hostname_col = self.excel_specs['specs'][self.spec]['hostname_col']
- ipmi_address_col = self.excel_specs['specs'][self.
- spec]['ipmi_address_col']
- host_profile_col = self.excel_specs['specs'][self.
- spec]['host_profile_col']
- ipmi_gateway_col = self.excel_specs['specs'][self.
- spec]['ipmi_gateway_col']
+ row = self.excel_specs["specs"][self.spec]["start_row"]
+ end_row = self.excel_specs["specs"][self.spec]["end_row"]
+ hostname_col = self.excel_specs["specs"][self.spec]["hostname_col"]
+ ipmi_address_col = self.excel_specs["specs"][self.spec][
+ "ipmi_address_col"
+ ]
+ host_profile_col = self.excel_specs["specs"][self.spec][
+ "host_profile_col"
+ ]
+ ipmi_gateway_col = self.excel_specs["specs"][self.spec][
+ "ipmi_gateway_col"
+ ]
previous_server_gateway = None
while row <= end_row:
hostname = self.sanitize(
- ws.cell(row=row, column=hostname_col).value)
+ ws.cell(row=row, column=hostname_col).value
+ )
hosts.append(hostname)
ipmi_address = ws.cell(row=row, column=ipmi_address_col).value
- if '/' in ipmi_address:
- ipmi_address = ipmi_address.split('/')[0]
+ if "/" in ipmi_address:
+ ipmi_address = ipmi_address.split("/")[0]
ipmi_gateway = ws.cell(row=row, column=ipmi_gateway_col).value
if ipmi_gateway:
previous_server_gateway = ipmi_gateway
@@ -106,32 +113,39 @@ class ExcelParser():
host_profile = ws.cell(row=row, column=host_profile_col).value
try:
if host_profile is None:
- raise RuntimeError("No value read from {} ".format(
- self.file_name) + "sheet:{} row:{}, col:{}".format(
- self.spec, row, host_profile_col))
+ raise RuntimeError(
+ "No value read from {} ".format(self.file_name)
+ + "sheet:{} row:{}, col:{}".format(
+ self.spec, row, host_profile_col
+ )
+ )
except RuntimeError as rerror:
LOG.critical(rerror)
sys.exit("Tugboat exited!!")
ipmi_data[hostname] = {
- 'ipmi_address': ipmi_address,
- 'ipmi_gateway': ipmi_gateway,
- 'host_profile': host_profile,
- 'type': type,
+ "ipmi_address": ipmi_address,
+ "ipmi_gateway": ipmi_gateway,
+ "host_profile": host_profile,
+ "type": type,
}
row += 1
- LOG.debug("ipmi data extracted from excel:\n{}".format(
- pprint.pformat(ipmi_data)))
- LOG.debug("host data extracted from excel:\n{}".format(
- pprint.pformat(hosts)))
+ LOG.debug(
+ "ipmi data extracted from excel:\n{}".format(
+ pprint.pformat(ipmi_data)
+ )
+ )
+ LOG.debug(
+ "host data extracted from excel:\n{}".format(pprint.pformat(hosts))
+ )
return [ipmi_data, hosts]
def get_private_vlan_data(self, ws):
""" Get private vlan data from private IP sheet """
vlan_data = {}
- row = self.excel_specs['specs'][self.spec]['vlan_start_row']
- end_row = self.excel_specs['specs'][self.spec]['vlan_end_row']
- type_col = self.excel_specs['specs'][self.spec]['net_type_col']
- vlan_col = self.excel_specs['specs'][self.spec]['vlan_col']
+ row = self.excel_specs["specs"][self.spec]["vlan_start_row"]
+ end_row = self.excel_specs["specs"][self.spec]["vlan_end_row"]
+ type_col = self.excel_specs["specs"][self.spec]["net_type_col"]
+ vlan_col = self.excel_specs["specs"][self.spec]["vlan_col"]
while row <= end_row:
cell_value = ws.cell(row=row, column=type_col).value
if cell_value:
@@ -140,27 +154,30 @@ class ExcelParser():
vlan = vlan.lower()
vlan_data[vlan] = cell_value
row += 1
- LOG.debug("vlan data extracted from excel:\n%s",
- pprint.pformat(vlan_data))
+ LOG.debug(
+ "vlan data extracted from excel:\n%s", pprint.pformat(vlan_data)
+ )
return vlan_data
def get_private_network_data(self):
""" Read network data from the private ip sheet """
- provided_sheetname = self.excel_specs['specs'][
- self.spec]['private_ip_sheet']
+ provided_sheetname = self.excel_specs["specs"][self.spec][
+ "private_ip_sheet"
+ ]
workbook_object, extracted_sheetname = self.get_xl_obj_and_sheetname(
- provided_sheetname)
+ provided_sheetname
+ )
if workbook_object is not None:
ws = workbook_object[extracted_sheetname]
else:
ws = self.wb_combined[provided_sheetname]
vlan_data = self.get_private_vlan_data(ws)
network_data = {}
- row = self.excel_specs['specs'][self.spec]['net_start_row']
- end_row = self.excel_specs['specs'][self.spec]['net_end_row']
- col = self.excel_specs['specs'][self.spec]['net_col']
- vlan_col = self.excel_specs['specs'][self.spec]['net_vlan_col']
- old_vlan = ''
+ row = self.excel_specs["specs"][self.spec]["net_start_row"]
+ end_row = self.excel_specs["specs"][self.spec]["net_end_row"]
+ col = self.excel_specs["specs"][self.spec]["net_col"]
+ vlan_col = self.excel_specs["specs"][self.spec]["net_vlan_col"]
+ old_vlan = ""
while row <= end_row:
vlan = ws.cell(row=row, column=vlan_col).value
if vlan:
@@ -168,11 +185,8 @@ class ExcelParser():
network = ws.cell(row=row, column=col).value
if vlan and network:
net_type = vlan_data[vlan]
- if 'vlan' not in network_data:
- network_data[net_type] = {
- 'vlan': vlan,
- 'subnet': [],
- }
+ if "vlan" not in network_data:
+ network_data[net_type] = {"vlan": vlan, "subnet": []}
elif not vlan and network:
# If vlan is not present then assign old vlan to vlan as vlan
# value is spread over several rows
@@ -180,11 +194,11 @@ class ExcelParser():
else:
row += 1
continue
- network_data[vlan_data[vlan]]['subnet'].append(network)
+ network_data[vlan_data[vlan]]["subnet"].append(network)
old_vlan = vlan
row += 1
for network in network_data:
- network_data[network]['is_common'] = True
+ network_data[network]["is_common"] = True
"""
if len(network_data[network]['subnet']) > 1:
network_data[network]['is_common'] = False
@@ -199,153 +213,167 @@ class ExcelParser():
def get_public_network_data(self):
""" Read public network data from public ip data """
network_data = {}
- provided_sheetname = self.excel_specs['specs'][self.
- spec]['public_ip_sheet']
+ provided_sheetname = self.excel_specs["specs"][self.spec][
+ "public_ip_sheet"
+ ]
workbook_object, extracted_sheetname = self.get_xl_obj_and_sheetname(
- provided_sheetname)
+ provided_sheetname
+ )
if workbook_object is not None:
ws = workbook_object[extracted_sheetname]
else:
ws = self.wb_combined[provided_sheetname]
- oam_row = self.excel_specs['specs'][self.spec]['oam_ip_row']
- oam_col = self.excel_specs['specs'][self.spec]['oam_ip_col']
- oam_vlan_col = self.excel_specs['specs'][self.spec]['oam_vlan_col']
- ingress_row = self.excel_specs['specs'][self.spec]['ingress_ip_row']
- oob_row = self.excel_specs['specs'][self.spec]['oob_net_row']
- col = self.excel_specs['specs'][self.spec]['oob_net_start_col']
- end_col = self.excel_specs['specs'][self.spec]['oob_net_end_col']
+ oam_row = self.excel_specs["specs"][self.spec]["oam_ip_row"]
+ oam_col = self.excel_specs["specs"][self.spec]["oam_ip_col"]
+ oam_vlan_col = self.excel_specs["specs"][self.spec]["oam_vlan_col"]
+ ingress_row = self.excel_specs["specs"][self.spec]["ingress_ip_row"]
+ oob_row = self.excel_specs["specs"][self.spec]["oob_net_row"]
+ col = self.excel_specs["specs"][self.spec]["oob_net_start_col"]
+ end_col = self.excel_specs["specs"][self.spec]["oob_net_end_col"]
network_data = {
- 'oam': {
- 'subnet': [ws.cell(row=oam_row, column=oam_col).value],
- 'vlan': ws.cell(row=oam_row, column=oam_vlan_col).value,
+ "oam": {
+ "subnet": [ws.cell(row=oam_row, column=oam_col).value],
+ "vlan": ws.cell(row=oam_row, column=oam_vlan_col).value,
},
- 'ingress': ws.cell(row=ingress_row, column=oam_col).value,
- }
- network_data['oob'] = {
- 'subnet': [],
+ "ingress": ws.cell(row=ingress_row, column=oam_col).value,
}
+ network_data["oob"] = {"subnet": []}
while col <= end_col:
cell_value = ws.cell(row=oob_row, column=col).value
if cell_value:
- network_data['oob']['subnet'].append(self.sanitize(cell_value))
+ network_data["oob"]["subnet"].append(self.sanitize(cell_value))
col += 1
LOG.debug(
"public network data extracted from\
- excel:\n%s", pprint.pformat(network_data))
+ excel:\n%s",
+ pprint.pformat(network_data),
+ )
return network_data
def get_site_info(self):
""" Read location, dns, ntp and ldap data"""
site_info = {}
- provided_sheetname = self.excel_specs['specs'][
- self.spec]['dns_ntp_ldap_sheet']
+ provided_sheetname = self.excel_specs["specs"][self.spec][
+ "dns_ntp_ldap_sheet"
+ ]
workbook_object, extracted_sheetname = self.get_xl_obj_and_sheetname(
- provided_sheetname)
+ provided_sheetname
+ )
if workbook_object is not None:
ws = workbook_object[extracted_sheetname]
else:
ws = self.wb_combined[provided_sheetname]
- dns_row = self.excel_specs['specs'][self.spec]['dns_row']
- dns_col = self.excel_specs['specs'][self.spec]['dns_col']
- ntp_row = self.excel_specs['specs'][self.spec]['ntp_row']
- ntp_col = self.excel_specs['specs'][self.spec]['ntp_col']
- domain_row = self.excel_specs['specs'][self.spec]['domain_row']
- domain_col = self.excel_specs['specs'][self.spec]['domain_col']
- login_domain_row = self.excel_specs['specs'][self.
- spec]['login_domain_row']
- ldap_col = self.excel_specs['specs'][self.spec]['ldap_col']
- global_group = self.excel_specs['specs'][self.spec]['global_group']
- ldap_search_url_row = self.excel_specs['specs'][
- self.spec]['ldap_search_url_row']
+ dns_row = self.excel_specs["specs"][self.spec]["dns_row"]
+ dns_col = self.excel_specs["specs"][self.spec]["dns_col"]
+ ntp_row = self.excel_specs["specs"][self.spec]["ntp_row"]
+ ntp_col = self.excel_specs["specs"][self.spec]["ntp_col"]
+ domain_row = self.excel_specs["specs"][self.spec]["domain_row"]
+ domain_col = self.excel_specs["specs"][self.spec]["domain_col"]
+ login_domain_row = self.excel_specs["specs"][self.spec][
+ "login_domain_row"
+ ]
+ ldap_col = self.excel_specs["specs"][self.spec]["ldap_col"]
+ global_group = self.excel_specs["specs"][self.spec]["global_group"]
+ ldap_search_url_row = self.excel_specs["specs"][self.spec][
+ "ldap_search_url_row"
+ ]
dns_servers = ws.cell(row=dns_row, column=dns_col).value
ntp_servers = ws.cell(row=ntp_row, column=ntp_col).value
try:
if dns_servers is None:
raise RuntimeError(
- "No value for dns_server from:{} Sheet:'{}' Row:{} Col:{}".
- format(self.file_name, provided_sheetname, dns_row,
- dns_col))
- raise RuntimeError(
- "No value for ntp_server frome:{} Sheet:'{}' Row:{} Col:{}"
- .format(self.file_name, provided_sheetname, ntp_row,
- ntp_col))
+ (
+ "No value for dns_server from:{} Sheet:'{}' ",
+ "Row:{} Col:{}",
+ ).format(
+ self.file_name, provided_sheetname, dns_row, dns_col
+ )
+ )
except RuntimeError as rerror:
LOG.critical(rerror)
sys.exit("Tugboat exited!!")
- dns_servers = dns_servers.replace('\n', ' ')
- ntp_servers = ntp_servers.replace('\n', ' ')
- if ',' in dns_servers:
- dns_servers = dns_servers.split(',')
+ dns_servers = dns_servers.replace("\n", " ")
+ ntp_servers = ntp_servers.replace("\n", " ")
+ if "," in dns_servers:
+ dns_servers = dns_servers.split(",")
else:
dns_servers = dns_servers.split()
- if ',' in ntp_servers:
- ntp_servers = ntp_servers.split(',')
+ if "," in ntp_servers:
+ ntp_servers = ntp_servers.split(",")
else:
ntp_servers = ntp_servers.split()
site_info = {
- 'location': self.get_location_data(),
- 'dns': dns_servers,
- 'ntp': ntp_servers,
- 'domain': ws.cell(row=domain_row, column=domain_col).value,
- 'ldap': {
- 'subdomain': ws.cell(row=login_domain_row,
- column=ldap_col).value,
- 'common_name': ws.cell(row=global_group,
- column=ldap_col).value,
- 'url': ws.cell(row=ldap_search_url_row, column=ldap_col).value,
- }
+ "location": self.get_location_data(),
+ "dns": dns_servers,
+ "ntp": ntp_servers,
+ "domain": ws.cell(row=domain_row, column=domain_col).value,
+ "ldap": {
+ "subdomain": ws.cell(
+ row=login_domain_row, column=ldap_col
+ ).value,
+ "common_name": ws.cell(
+ row=global_group, column=ldap_col
+ ).value,
+ "url": ws.cell(row=ldap_search_url_row, column=ldap_col).value,
+ },
}
LOG.debug(
"Site Info extracted from\
- excel:\n%s", pprint.pformat(site_info))
+ excel:\n%s",
+ pprint.pformat(site_info),
+ )
return site_info
def get_location_data(self):
""" Read location data from the site and zone sheet """
- provided_sheetname = self.excel_specs['specs'][self.
- spec]['location_sheet']
+ provided_sheetname = self.excel_specs["specs"][self.spec][
+ "location_sheet"
+ ]
workbook_object, extracted_sheetname = self.get_xl_obj_and_sheetname(
- provided_sheetname)
+ provided_sheetname
+ )
if workbook_object is not None:
ws = workbook_object[extracted_sheetname]
else:
ws = self.wb_combined[provided_sheetname]
- corridor_row = self.excel_specs['specs'][self.spec]['corridor_row']
- column = self.excel_specs['specs'][self.spec]['column']
- site_name_row = self.excel_specs['specs'][self.spec]['site_name_row']
- state_name_row = self.excel_specs['specs'][self.spec]['state_name_row']
- country_name_row = self.excel_specs['specs'][self.
- spec]['country_name_row']
- clli_name_row = self.excel_specs['specs'][self.spec]['clli_name_row']
+ corridor_row = self.excel_specs["specs"][self.spec]["corridor_row"]
+ column = self.excel_specs["specs"][self.spec]["column"]
+ site_name_row = self.excel_specs["specs"][self.spec]["site_name_row"]
+ state_name_row = self.excel_specs["specs"][self.spec]["state_name_row"]
+ country_name_row = self.excel_specs["specs"][self.spec][
+ "country_name_row"
+ ]
+ clli_name_row = self.excel_specs["specs"][self.spec]["clli_name_row"]
return {
- 'corridor': ws.cell(row=corridor_row, column=column).value,
- 'name': ws.cell(row=site_name_row, column=column).value,
- 'state': ws.cell(row=state_name_row, column=column).value,
- 'country': ws.cell(row=country_name_row, column=column).value,
- 'physical_location': ws.cell(row=clli_name_row,
- column=column).value,
+ "corridor": ws.cell(row=corridor_row, column=column).value,
+ "name": ws.cell(row=site_name_row, column=column).value,
+ "state": ws.cell(row=state_name_row, column=column).value,
+ "country": ws.cell(row=country_name_row, column=column).value,
+ "physical_location": ws.cell(
+ row=clli_name_row, column=column
+ ).value,
}
def validate_sheet_names_with_spec(self):
""" Checks is sheet name in spec file matches with excel file"""
- spec = list(self.excel_specs['specs'].keys())[0]
- spec_item = self.excel_specs['specs'][spec]
+ spec = list(self.excel_specs["specs"].keys())[0]
+ spec_item = self.excel_specs["specs"][spec]
sheet_name_list = []
- ipmi_header_sheet_name = spec_item['ipmi_sheet_name']
+ ipmi_header_sheet_name = spec_item["ipmi_sheet_name"]
sheet_name_list.append(ipmi_header_sheet_name)
- private_ip_sheet_name = spec_item['private_ip_sheet']
+ private_ip_sheet_name = spec_item["private_ip_sheet"]
sheet_name_list.append(private_ip_sheet_name)
- public_ip_sheet_name = spec_item['public_ip_sheet']
+ public_ip_sheet_name = spec_item["public_ip_sheet"]
sheet_name_list.append(public_ip_sheet_name)
- dns_ntp_ldap_sheet_name = spec_item['dns_ntp_ldap_sheet']
+ dns_ntp_ldap_sheet_name = spec_item["dns_ntp_ldap_sheet"]
sheet_name_list.append(dns_ntp_ldap_sheet_name)
- location_sheet_name = spec_item['location_sheet']
+ location_sheet_name = spec_item["location_sheet"]
sheet_name_list.append(location_sheet_name)
try:
for sheetname in sheet_name_list:
- workbook_object, extracted_sheetname = \
- self.get_xl_obj_and_sheetname(sheetname)
+ workbook_object, extracted_sheetname = (
+ self.get_xl_obj_and_sheetname(sheetname))
if workbook_object is not None:
wb = workbook_object
sheetname = extracted_sheetname
@@ -354,7 +382,8 @@ class ExcelParser():
if sheetname not in wb.sheetnames:
raise RuntimeError(
- "SheetName '{}' not found ".format(sheetname))
+ "SheetName '{}' not found ".format(sheetname)
+ )
except RuntimeError as rerror:
LOG.critical(rerror)
sys.exit("Tugboat exited!!")
@@ -369,16 +398,18 @@ class ExcelParser():
public_network_data = self.get_public_network_data()
site_info_data = self.get_site_info()
data = {
- 'ipmi_data': ipmi_data,
- 'network_data': {
- 'private': network_data,
- 'public': public_network_data,
+ "ipmi_data": ipmi_data,
+ "network_data": {
+ "private": network_data,
+ "public": public_network_data,
},
- 'site_info': site_info_data,
+ "site_info": site_info_data,
}
LOG.debug(
"Location data extracted from\
- excel:\n%s", pprint.pformat(data))
+ excel:\n%s",
+ pprint.pformat(data),
+ )
return data
def combine_excel_design_specs(self, filenames):
@@ -391,8 +422,9 @@ class ExcelParser():
loaded_workbook_ws = loaded_workbook[names]
for row in loaded_workbook_ws:
for cell in row:
- design_spec_worksheet[cell.
- coordinate].value = cell.value
+ design_spec_worksheet[
+ cell.coordinate
+ ].value = cell.value
return design_spec
def get_xl_obj_and_sheetname(self, sheetname):
@@ -400,10 +432,10 @@ class ExcelParser():
The logic confirms if the sheetname is specified for example as:
"MTN57a_AEC_Network_Design_v1.6.xlsx:Public IPs"
"""
- if (re.search('.xlsx', sheetname) or re.search('.xls', sheetname)):
+ if re.search(".xlsx", sheetname) or re.search(".xls", sheetname):
""" Extract file name """
- source_xl_file = sheetname.split(':')[0]
+ source_xl_file = sheetname.split(":")[0]
wb = load_workbook(source_xl_file, data_only=True)
- return [wb, sheetname.split(':')[1]]
+ return [wb, sheetname.split(":")[1]]
else:
return [None, sheetname]
diff --git a/spyglass/data_extractor/plugins/tugboat/tugboat.py b/spyglass/data_extractor/plugins/tugboat/tugboat.py
index b2829e1..c928cce 100644
--- a/spyglass/data_extractor/plugins/tugboat/tugboat.py
+++ b/spyglass/data_extractor/plugins/tugboat/tugboat.py
@@ -25,8 +25,8 @@ LOG = logging.getLogger(__name__)
class TugboatPlugin(BaseDataSourcePlugin):
def __init__(self, region):
LOG.info("Tugboat Initializing")
- self.source_type = 'excel'
- self.source_name = 'tugboat'
+ self.source_type = "excel"
+ self.source_name = "tugboat"
# Configuration parameters
self.excel_path = None
@@ -52,8 +52,8 @@ class TugboatPlugin(BaseDataSourcePlugin):
Each plugin will have their own config opts.
"""
- self.excel_path = conf['excel_path']
- self.excel_spec = conf['excel_spec']
+ self.excel_path = conf["excel_path"]
+ self.excel_spec = conf["excel_spec"]
# Extract raw data from excel sheets
self._get_excel_obj()
@@ -69,18 +69,18 @@ class TugboatPlugin(BaseDataSourcePlugin):
written as an additional safeguard.
"""
try:
- assert (len(
- kwargs['excel'])), "Engineering Spec file not specified"
- excel_file_info = kwargs['excel']
- assert (kwargs['excel_spec']
- ) is not None, "Excel Spec file not specified"
- excel_spec_info = kwargs['excel_spec']
+ assert len(kwargs["excel"]), "Engineering Spec file not specified"
+ excel_file_info = kwargs["excel"]
+ assert (
+ kwargs["excel_spec"]
+ ) is not None, "Excel Spec file not specified"
+ excel_spec_info = kwargs["excel_spec"]
except AssertionError as e:
LOG.error("{}:Spyglass exited!".format(e))
exit()
plugin_conf = {
- 'excel_path': excel_file_info,
- 'excel_spec': excel_spec_info
+ "excel_path": excel_file_info,
+ "excel_spec": excel_spec_info,
}
return plugin_conf
@@ -103,19 +103,18 @@ class TugboatPlugin(BaseDataSourcePlugin):
]
"""
LOG.info("Get Host Information")
- ipmi_data = self.parsed_xl_data['ipmi_data'][0]
+ ipmi_data = self.parsed_xl_data["ipmi_data"][0]
rackwise_hosts = self._get_rackwise_hosts()
host_list = []
for rack in rackwise_hosts.keys():
for host in rackwise_hosts[rack]:
- host_list.append({
- 'rack_name':
- rack,
- 'name':
- host,
- 'host_profile':
- ipmi_data[host]['host_profile']
- })
+ host_list.append(
+ {
+ "rack_name": rack,
+ "name": host,
+ "host_profile": ipmi_data[host]["host_profile"],
+ }
+ )
return host_list
def get_networks(self, region):
@@ -123,39 +122,44 @@ class TugboatPlugin(BaseDataSourcePlugin):
vlan_list = []
# Network data extracted from xl is formatted to have a predictable
# data type. For e.g VlAN 45 extracted from xl is formatted as 45
- vlan_pattern = r'\d+'
- private_net = self.parsed_xl_data['network_data']['private']
- public_net = self.parsed_xl_data['network_data']['public']
+ vlan_pattern = r"\d+"
+ private_net = self.parsed_xl_data["network_data"]["private"]
+ public_net = self.parsed_xl_data["network_data"]["public"]
# Extract network information from private and public network data
- for net_type, net_val in itertools.chain(private_net.items(),
- public_net.items()):
+ for net_type, net_val in itertools.chain(
+ private_net.items(), public_net.items()
+ ):
tmp_vlan = {}
# Ingress is special network that has no vlan, only a subnet string
# So treatment for ingress is different
- if net_type is not 'ingress':
+ if net_type != "ingress":
# standardize the network name as net_type may ne different.
# For e.g insteas of pxe it may be PXE or instead of calico
# it may be ksn. Valid network names are pxe, calico, oob, oam,
# overlay, storage, ingress
- tmp_vlan['name'] = self._get_network_name_from_vlan_name(
- net_type)
+ tmp_vlan["name"] = self._get_network_name_from_vlan_name(
+ net_type
+ )
# extract vlan tag. It was extracted from xl file as 'VlAN 45'
# The code below extracts the numeric data fron net_val['vlan']
- if net_val.get('vlan', "") is not "":
- value = re.findall(vlan_pattern, net_val['vlan'])
- tmp_vlan['vlan'] = value[0]
+ if net_val.get("vlan", "") != "":
+ value = re.findall(vlan_pattern, net_val["vlan"])
+ tmp_vlan["vlan"] = value[0]
else:
- tmp_vlan['vlan'] = "#CHANGE_ME"
+ tmp_vlan["vlan"] = "#CHANGE_ME"
- tmp_vlan['subnet'] = net_val.get('subnet', "#CHANGE_ME")
- tmp_vlan['gateway'] = net_val.get('gateway', "#CHANGE_ME")
+ tmp_vlan["subnet"] = net_val.get("subnet", "#CHANGE_ME")
+ tmp_vlan["gateway"] = net_val.get("gateway", "#CHANGE_ME")
else:
- tmp_vlan['name'] = 'ingress'
- tmp_vlan['subnet'] = net_val
+ tmp_vlan["name"] = "ingress"
+ tmp_vlan["subnet"] = net_val
vlan_list.append(tmp_vlan)
- LOG.debug("vlan list extracted from tugboat:\n{}".format(
- pprint.pformat(vlan_list)))
+ LOG.debug(
+ "vlan list extracted from tugboat:\n{}".format(
+ pprint.pformat(vlan_list)
+ )
+ )
return vlan_list
def get_ips(self, region, host=None):
@@ -172,33 +176,34 @@ class TugboatPlugin(BaseDataSourcePlugin):
"""
ip_ = {}
- ipmi_data = self.parsed_xl_data['ipmi_data'][0]
+ ipmi_data = self.parsed_xl_data["ipmi_data"][0]
ip_[host] = {
- 'oob': ipmi_data[host].get('ipmi_address', '#CHANGE_ME'),
- 'oam': ipmi_data[host].get('oam', '#CHANGE_ME'),
- 'calico': ipmi_data[host].get('calico', '#CHANGE_ME'),
- 'overlay': ipmi_data[host].get('overlay', '#CHANGE_ME'),
- 'pxe': ipmi_data[host].get('pxe', '#CHANGE_ME'),
- 'storage': ipmi_data[host].get('storage', '#CHANGE_ME')
+ "oob": ipmi_data[host].get("ipmi_address", "#CHANGE_ME"),
+ "oam": ipmi_data[host].get("oam", "#CHANGE_ME"),
+ "calico": ipmi_data[host].get("calico", "#CHANGE_ME"),
+ "overlay": ipmi_data[host].get("overlay", "#CHANGE_ME"),
+ "pxe": ipmi_data[host].get("pxe", "#CHANGE_ME"),
+ "storage": ipmi_data[host].get("storage", "#CHANGE_ME"),
}
return ip_
def get_ldap_information(self, region):
""" Extract ldap information from excel"""
- ldap_raw_data = self.parsed_xl_data['site_info']['ldap']
+ ldap_raw_data = self.parsed_xl_data["site_info"]["ldap"]
ldap_info = {}
# raw url is 'url: ldap://example.com' so we are converting to
# 'ldap://example.com'
- url = ldap_raw_data.get('url', '#CHANGE_ME')
+ url = ldap_raw_data.get("url", "#CHANGE_ME")
try:
- ldap_info['url'] = url.split(' ')[1]
- ldap_info['domain'] = url.split('.')[1]
+ ldap_info["url"] = url.split(" ")[1]
+ ldap_info["domain"] = url.split(".")[1]
except IndexError as e:
LOG.error("url.split:{}".format(e))
- ldap_info['common_name'] = ldap_raw_data.get('common_name',
- '#CHANGE_ME')
- ldap_info['subdomain'] = ldap_raw_data.get('subdomain', '#CHANGE_ME')
+ ldap_info["common_name"] = ldap_raw_data.get(
+ "common_name", "#CHANGE_ME"
+ )
+ ldap_info["subdomain"] = ldap_raw_data.get("subdomain", "#CHANGE_ME")
return ldap_info
@@ -206,41 +211,44 @@ class TugboatPlugin(BaseDataSourcePlugin):
""" Returns a comma separated list of ntp ip addresses"""
ntp_server_list = self._get_formatted_server_list(
- self.parsed_xl_data['site_info']['ntp'])
+ self.parsed_xl_data["site_info"]["ntp"]
+ )
return ntp_server_list
def get_dns_servers(self, region):
""" Returns a comma separated list of dns ip addresses"""
dns_server_list = self._get_formatted_server_list(
- self.parsed_xl_data['site_info']['dns'])
+ self.parsed_xl_data["site_info"]["dns"]
+ )
return dns_server_list
def get_domain_name(self, region):
""" Returns domain name extracted from excel file"""
- return self.parsed_xl_data['site_info']['domain']
+ return self.parsed_xl_data["site_info"]["domain"]
def get_location_information(self, region):
"""
Prepare location data from information extracted
by ExcelParser(i.e raw data)
"""
- location_data = self.parsed_xl_data['site_info']['location']
+ location_data = self.parsed_xl_data["site_info"]["location"]
- corridor_pattern = r'\d+'
- corridor_number = re.findall(corridor_pattern,
- location_data['corridor'])[0]
- name = location_data.get('name', '#CHANGE_ME')
- state = location_data.get('state', '#CHANGE_ME')
- country = location_data.get('country', '#CHANGE_ME')
- physical_location_id = location_data.get('physical_location', '')
+ corridor_pattern = r"\d+"
+ corridor_number = re.findall(
+ corridor_pattern, location_data["corridor"]
+ )[0]
+ name = location_data.get("name", "#CHANGE_ME")
+ state = location_data.get("state", "#CHANGE_ME")
+ country = location_data.get("country", "#CHANGE_ME")
+ physical_location_id = location_data.get("physical_location", "")
return {
- 'name': name,
- 'physical_location_id': physical_location_id,
- 'state': state,
- 'country': country,
- 'corridor': 'c{}'.format(corridor_number),
+ "name": name,
+ "physical_location_id": physical_location_id,
+ "state": state,
+ "country": country,
+ "corridor": "c{}".format(corridor_number),
}
def get_racks(self, region):
@@ -277,29 +285,35 @@ class TugboatPlugin(BaseDataSourcePlugin):
vlan_name contains "pxe" the network name is "pxe"
"""
network_names = [
- 'ksn|calico', 'storage', 'oam|server', 'ovs|overlay', 'oob', 'pxe'
+ "ksn|calico",
+ "storage",
+ "oam|server",
+ "ovs|overlay",
+ "oob",
+ "pxe",
]
for name in network_names:
# Make a pattern that would ignore case.
# if name is 'ksn' pattern name is '(?i)(ksn)'
name_pattern = "(?i)({})".format(name)
if re.search(name_pattern, vlan_name):
- if name is 'ksn|calico':
- return 'calico'
- if name is 'storage':
- return 'storage'
- if name is 'oam|server':
- return 'oam'
- if name is 'ovs|overlay':
- return 'overlay'
- if name is 'oob':
- return 'oob'
- if name is 'pxe':
- return 'pxe'
+ if name == "ksn|calico":
+ return "calico"
+ if name == "storage":
+ return "storage"
+ if name == "oam|server":
+ return "oam"
+ if name == "ovs|overlay":
+ return "overlay"
+ if name == "oob":
+ return "oob"
+ if name == "pxe":
+ return "pxe"
# if nothing matches
LOG.error(
- "Unable to recognize VLAN name extracted from Plugin data source")
- return ("")
+ "Unable to recognize VLAN name extracted from Plugin data source"
+ )
+ return ""
def _get_formatted_server_list(self, server_list):
""" Format dns and ntp server list as comma separated string """
@@ -309,9 +323,9 @@ class TugboatPlugin(BaseDataSourcePlugin):
# The function returns a list of comma separated dns ip addresses
servers = []
for data in server_list:
- if '(' not in data:
+ if "(" not in data:
servers.append(data)
- formatted_server_list = ','.join(servers)
+ formatted_server_list = ",".join(servers)
return formatted_server_list
def _get_rack(self, host):
@@ -319,7 +333,7 @@ class TugboatPlugin(BaseDataSourcePlugin):
Get rack id from the rack string extracted
from xl
"""
- rack_pattern = r'\w.*(r\d+)\w.*'
+ rack_pattern = r"\w.*(r\d+)\w.*"
rack = re.findall(rack_pattern, host)[0]
if not self.region:
self.region = host.split(rack)[0]
@@ -328,7 +342,7 @@ class TugboatPlugin(BaseDataSourcePlugin):
def _get_rackwise_hosts(self):
""" Mapping hosts with rack ids """
rackwise_hosts = {}
- hostnames = self.parsed_xl_data['ipmi_data'][1]
+ hostnames = self.parsed_xl_data["ipmi_data"][1]
racks = self._get_rack_data()
for rack in racks:
if rack not in rackwise_hosts:
@@ -343,8 +357,8 @@ class TugboatPlugin(BaseDataSourcePlugin):
""" Format rack name """
LOG.info("Getting rack data")
racks = {}
- hostnames = self.parsed_xl_data['ipmi_data'][1]
+ hostnames = self.parsed_xl_data["ipmi_data"][1]
for host in hostnames:
rack = self._get_rack(host)
- racks[rack] = rack.replace('r', 'rack')
+ racks[rack] = rack.replace("r", "rack")
return racks
diff --git a/spyglass/parser/engine.py b/spyglass/parser/engine.py
index 6e92a89..f3ba441 100644
--- a/spyglass/parser/engine.py
+++ b/spyglass/parser/engine.py
@@ -28,7 +28,7 @@ import yaml
LOG = logging.getLogger(__name__)
-class ProcessDataSource():
+class ProcessDataSource:
def __init__(self, sitetype):
# Initialize intermediary and save site type
self._initialize_intermediary()
@@ -36,18 +36,18 @@ class ProcessDataSource():
@staticmethod
def _read_file(file_name):
- with open(file_name, 'r') as f:
+ with open(file_name, "r") as f:
raw_data = f.read()
return raw_data
def _initialize_intermediary(self):
self.host_type = {}
self.data = {
- 'network': {},
- 'baremetal': {},
- 'region_name': '',
- 'storage': {},
- 'site_info': {},
+ "network": {},
+ "baremetal": {},
+ "region_name": "",
+ "storage": {},
+ "site_info": {},
}
self.sitetype = None
self.genesis_node = None
@@ -62,37 +62,43 @@ class ProcessDataSource():
we assign only the first subnet """
LOG.info("Extracting network subnets")
network_subnets = {}
- for net_type in self.data['network']['vlan_network_data']:
+ for net_type in self.data["network"]["vlan_network_data"]:
# One of the type is ingress and we don't want that here
- if (net_type != 'ingress'):
+ if net_type != "ingress":
network_subnets[net_type] = netaddr.IPNetwork(
- self.data['network']['vlan_network_data'][net_type]
- ['subnet'][0])
+ self.data["network"]["vlan_network_data"][net_type][
+ "subnet"
+ ][0]
+ )
- LOG.debug("Network subnets:\n{}".format(
- pprint.pformat(network_subnets)))
+ LOG.debug(
+ "Network subnets:\n{}".format(pprint.pformat(network_subnets))
+ )
return network_subnets
def _get_genesis_node_details(self):
# Get genesis host node details from the hosts based on host type
- for racks in self.data['baremetal'].keys():
- rack_hosts = self.data['baremetal'][racks]
+ for racks in self.data["baremetal"].keys():
+ rack_hosts = self.data["baremetal"][racks]
for host in rack_hosts:
- if rack_hosts[host]['type'] == 'genesis':
+ if rack_hosts[host]["type"] == "genesis":
self.genesis_node = rack_hosts[host]
- self.genesis_node['name'] = host
- LOG.debug("Genesis Node Details:\n{}".format(
- pprint.pformat(self.genesis_node)))
+ self.genesis_node["name"] = host
+ LOG.debug(
+ "Genesis Node Details:\n{}".format(
+ pprint.pformat(self.genesis_node)
+ )
+ )
def _get_genesis_node_ip(self):
""" Returns the genesis node ip """
- ip = '0.0.0.0'
+ ip = "0.0.0.0"
LOG.info("Getting Genesis Node IP")
if not self.genesis_node:
self._get_genesis_node_details()
- ips = self.genesis_node.get('ip', '')
+ ips = self.genesis_node.get("ip", "")
if ips:
- ip = ips.get('oam', '0.0.0.0')
+ ip = ips.get("oam", "0.0.0.0")
return ip
def _validate_intermediary_data(self, data):
@@ -103,21 +109,21 @@ class ProcessDataSource():
The method validates this with regex pattern defined for each
data type.
"""
- LOG.info('Validating Intermediary data')
+ LOG.info("Validating Intermediary data")
temp_data = {}
# Peforming a deep copy
temp_data = copy.deepcopy(data)
# Converting baremetal dict to list.
baremetal_list = []
- for rack in temp_data['baremetal'].keys():
- temp = [{k: v} for k, v in temp_data['baremetal'][rack].items()]
+ for rack in temp_data["baremetal"].keys():
+ temp = [{k: v} for k, v in temp_data["baremetal"][rack].items()]
baremetal_list = baremetal_list + temp
- temp_data['baremetal'] = baremetal_list
- schema_dir = pkg_resources.resource_filename('spyglass', 'schemas/')
+ temp_data["baremetal"] = baremetal_list
+ schema_dir = pkg_resources.resource_filename("spyglass", "schemas/")
schema_file = schema_dir + "data_schema.json"
json_data = json.loads(json.dumps(temp_data))
- with open(schema_file, 'r') as f:
+ with open(schema_file, "r") as f:
json_schema = json.load(f)
try:
# Suppressing writing of data2.json. Can use it for debugging
@@ -152,14 +158,14 @@ class ProcessDataSource():
based on rule name and applies them to appropriate data objects.
"""
LOG.info("Apply design rules")
- rules_dir = pkg_resources.resource_filename('spyglass', 'config/')
- rules_file = rules_dir + 'rules.yaml'
+ rules_dir = pkg_resources.resource_filename("spyglass", "config/")
+ rules_file = rules_dir + "rules.yaml"
rules_data_raw = self._read_file(rules_file)
rules_yaml = yaml.safe_load(rules_data_raw)
rules_data = {}
rules_data.update(rules_yaml)
for rule in rules_data.keys():
- rule_name = rules_data[rule]['name']
+ rule_name = rules_data[rule]["name"]
function_str = "_apply_rule_" + rule_name
rule_data_name = rules_data[rule][rule_name]
function = getattr(self, function_str)
@@ -182,23 +188,25 @@ class ProcessDataSource():
compute or controller based on host_profile. For defining 'genesis'
the first controller host is defined as genesis."""
is_genesis = False
- hardware_profile = rule_data[self.data['site_info']['sitetype']]
+ hardware_profile = rule_data[self.data["site_info"]["sitetype"]]
# Getting individual racks. The racks are sorted to ensure that the
# first controller of the first rack is assigned as 'genesis' node.
- for rack in sorted(self.data['baremetal'].keys()):
+ for rack in sorted(self.data["baremetal"].keys()):
# Getting individual hosts in each rack. Sorting of the hosts are
# done to determine the genesis node.
- for host in sorted(self.data['baremetal'][rack].keys()):
- host_info = self.data['baremetal'][rack][host]
- if (host_info['host_profile'] == hardware_profile[
- 'profile_name']['ctrl']):
+ for host in sorted(self.data["baremetal"][rack].keys()):
+ host_info = self.data["baremetal"][rack][host]
+ if (
+ host_info["host_profile"]
+ == hardware_profile["profile_name"]["ctrl"]
+ ):
if not is_genesis:
- host_info['type'] = 'genesis'
+ host_info["type"] = "genesis"
is_genesis = True
else:
- host_info['type'] = 'controller'
+ host_info["type"] = "controller"
else:
- host_info['type'] = 'compute'
+ host_info["type"] = "compute"
def _apply_rule_ip_alloc_offset(self, rule_data):
""" Apply offset rules to update baremetal host ip's and vlan network
@@ -219,21 +227,24 @@ class ProcessDataSource():
If a particular ip exists it is overridden."""
# Ger defult ip offset
- default_ip_offset = rule_data['default']
+ default_ip_offset = rule_data["default"]
host_idx = 0
LOG.info("Update baremetal host ip's")
- for racks in self.data['baremetal'].keys():
- rack_hosts = self.data['baremetal'][racks]
+ for racks in self.data["baremetal"].keys():
+ rack_hosts = self.data["baremetal"][racks]
for host in rack_hosts:
- host_networks = rack_hosts[host]['ip']
+ host_networks = rack_hosts[host]["ip"]
for net in host_networks:
ips = list(self.network_subnets[net])
host_networks[net] = str(ips[host_idx + default_ip_offset])
host_idx = host_idx + 1
- LOG.debug("Updated baremetal host:\n{}".format(
- pprint.pformat(self.data['baremetal'])))
+ LOG.debug(
+ "Updated baremetal host:\n{}".format(
+ pprint.pformat(self.data["baremetal"])
+ )
+ )
def _update_vlan_net_data(self, rule_data):
""" Offset allocation rules to determine ip address range(s)
@@ -245,31 +256,37 @@ class ProcessDataSource():
LOG.info("Apply network design rules")
# Collect Rules
- default_ip_offset = rule_data['default']
- oob_ip_offset = rule_data['oob']
- gateway_ip_offset = rule_data['gateway']
- ingress_vip_offset = rule_data['ingress_vip']
+ default_ip_offset = rule_data["default"]
+ oob_ip_offset = rule_data["oob"]
+ gateway_ip_offset = rule_data["gateway"]
+ ingress_vip_offset = rule_data["ingress_vip"]
# static_ip_end_offset for non pxe network
- static_ip_end_offset = rule_data['static_ip_end']
+ static_ip_end_offset = rule_data["static_ip_end"]
# dhcp_ip_end_offset for pxe network
- dhcp_ip_end_offset = rule_data['dhcp_ip_end']
+ dhcp_ip_end_offset = rule_data["dhcp_ip_end"]
# Set ingress vip and CIDR for bgp
LOG.info("Apply network design rules:bgp")
subnet = netaddr.IPNetwork(
- self.data['network']['vlan_network_data']['ingress']['subnet'][0])
+ self.data["network"]["vlan_network_data"]["ingress"]["subnet"][0]
+ )
ips = list(subnet)
- self.data['network']['bgp']['ingress_vip'] = str(
- ips[ingress_vip_offset])
- self.data['network']['bgp']['public_service_cidr'] = self.data[
- 'network']['vlan_network_data']['ingress']['subnet'][0]
- LOG.debug("Updated network bgp data:\n{}".format(
- pprint.pformat(self.data['network']['bgp'])))
+ self.data["network"]["bgp"]["ingress_vip"] = str(
+ ips[ingress_vip_offset]
+ )
+ self.data["network"]["bgp"]["public_service_cidr"] = self.data[
+ "network"
+ ]["vlan_network_data"]["ingress"]["subnet"][0]
+ LOG.debug(
+ "Updated network bgp data:\n{}".format(
+ pprint.pformat(self.data["network"]["bgp"])
+ )
+ )
LOG.info("Apply network design rules:vlan")
# Apply rules to vlan networks
for net_type in self.network_subnets:
- if net_type == 'oob':
+ if net_type == "oob":
ip_offset = oob_ip_offset
else:
ip_offset = default_ip_offset
@@ -277,49 +294,60 @@ class ProcessDataSource():
subnet = self.network_subnets[net_type]
ips = list(subnet)
- self.data['network']['vlan_network_data'][net_type][
- 'gateway'] = str(ips[gateway_ip_offset])
+ self.data["network"]["vlan_network_data"][net_type][
+ "gateway"
+ ] = str(ips[gateway_ip_offset])
- self.data['network']['vlan_network_data'][net_type][
- 'reserved_start'] = str(ips[1])
- self.data['network']['vlan_network_data'][net_type][
- 'reserved_end'] = str(ips[ip_offset])
+ self.data["network"]["vlan_network_data"][net_type][
+ "reserved_start"
+ ] = str(ips[1])
+ self.data["network"]["vlan_network_data"][net_type][
+ "reserved_end"
+ ] = str(ips[ip_offset])
static_start = str(ips[ip_offset + 1])
static_end = str(ips[static_ip_end_offset])
- if net_type == 'pxe':
+ if net_type == "pxe":
mid = len(ips) // 2
static_end = str(ips[mid - 1])
dhcp_start = str(ips[mid])
dhcp_end = str(ips[dhcp_ip_end_offset])
- self.data['network']['vlan_network_data'][net_type][
- 'dhcp_start'] = dhcp_start
- self.data['network']['vlan_network_data'][net_type][
- 'dhcp_end'] = dhcp_end
+ self.data["network"]["vlan_network_data"][net_type][
+ "dhcp_start"
+ ] = dhcp_start
+ self.data["network"]["vlan_network_data"][net_type][
+ "dhcp_end"
+ ] = dhcp_end
- self.data['network']['vlan_network_data'][net_type][
- 'static_start'] = static_start
- self.data['network']['vlan_network_data'][net_type][
- 'static_end'] = static_end
+ self.data["network"]["vlan_network_data"][net_type][
+ "static_start"
+ ] = static_start
+ self.data["network"]["vlan_network_data"][net_type][
+ "static_end"
+ ] = static_end
# There is no vlan for oob network
- if (net_type != 'oob'):
- self.data['network']['vlan_network_data'][net_type][
- 'vlan'] = self.data['network']['vlan_network_data'][
- net_type]['vlan']
+ if net_type != "oob":
+ self.data["network"]["vlan_network_data"][net_type][
+ "vlan"
+ ] = self.data["network"]["vlan_network_data"][net_type]["vlan"]
# OAM have default routes. Only for cruiser. TBD
- if (net_type == 'oam'):
+ if net_type == "oam":
routes = ["0.0.0.0/0"]
else:
routes = []
- self.data['network']['vlan_network_data'][net_type][
- 'routes'] = routes
+ self.data["network"]["vlan_network_data"][net_type][
+ "routes"
+ ] = routes
- LOG.debug("Updated vlan network data:\n{}".format(
- pprint.pformat(self.data['network']['vlan_network_data'])))
+ LOG.debug(
+ "Updated vlan network data:\n{}".format(
+ pprint.pformat(self.data["network"]["vlan_network_data"])
+ )
+ )
def load_extracted_data_from_data_source(self, extracted_data):
"""
@@ -334,8 +362,11 @@ class ProcessDataSource():
LOG.info("Loading plugin data source")
self.data = extracted_data
- LOG.debug("Extracted data from plugin:\n{}".format(
- pprint.pformat(extracted_data)))
+ LOG.debug(
+ "Extracted data from plugin:\n{}".format(
+ pprint.pformat(extracted_data)
+ )
+ )
# Uncommeent following segment for debugging purpose.
# extracted_file = "extracted_file.yaml"
# yaml_file = yaml.dump(extracted_data, default_flow_style=False)
@@ -344,13 +375,14 @@ class ProcessDataSource():
# f.close()
# Append region_data supplied from CLI to self.data
- self.data['region_name'] = self.region_name
+ self.data["region_name"] = self.region_name
def dump_intermediary_file(self, intermediary_dir):
""" Writing intermediary yaml """
LOG.info("Writing intermediary yaml")
intermediary_file = "{}_intermediary.yaml".format(
- self.data['region_name'])
+ self.data["region_name"]
+ )
# Check of if output dir = intermediary_dir exists
if intermediary_dir is not None:
outfile = "{}/{}".format(intermediary_dir, intermediary_file)
@@ -358,7 +390,7 @@ class ProcessDataSource():
outfile = intermediary_file
LOG.info("Intermediary file:{}".format(outfile))
yaml_file = yaml.dump(self.data, default_flow_style=False)
- with open(outfile, 'w') as f:
+ with open(outfile, "w") as f:
f.write(yaml_file)
f.close()
@@ -379,10 +411,11 @@ class ProcessDataSource():
def edit_intermediary_yaml(self):
""" Edit generated data using on browser """
LOG.info(
- "edit_intermediary_yaml: Invoking web server for yaml editing")
- with tempfile.NamedTemporaryFile(mode='r+') as file_obj:
+ "edit_intermediary_yaml: Invoking web server for yaml editing"
+ )
+ with tempfile.NamedTemporaryFile(mode="r+") as file_obj:
yaml.safe_dump(self.data, file_obj, default_flow_style=False)
host = self._get_genesis_node_ip()
- os.system('yaml-editor -f {0} -h {1}'.format(file_obj.name, host))
+ os.system("yaml-editor -f {0} -h {1}".format(file_obj.name, host))
file_obj.seek(0)
self.data = yaml.safe_load(file_obj)
diff --git a/spyglass/site_processors/base.py b/spyglass/site_processors/base.py
index ea61db7..bfc048a 100644
--- a/spyglass/site_processors/base.py
+++ b/spyglass/site_processors/base.py
@@ -22,23 +22,20 @@ class BaseProcessor:
@staticmethod
def get_role_wise_nodes(yaml_data):
- hosts = {
- 'genesis': {},
- 'masters': [],
- 'workers': [],
- }
+ hosts = {"genesis": {}, "masters": [], "workers": []}
- for rack in yaml_data['baremetal']:
- for host in yaml_data['baremetal'][rack]:
- if yaml_data['baremetal'][rack][host]['type'] == 'genesis':
- hosts['genesis'] = {
- 'name': host,
- 'pxe': yaml_data['baremetal'][rack][host]['ip']['pxe'],
- 'oam': yaml_data['baremetal'][rack][host]['ip']['oam'],
+ for rack in yaml_data["baremetal"]:
+ for host in yaml_data["baremetal"][rack]:
+ if yaml_data["baremetal"][rack][host]["type"] == "genesis":
+ hosts["genesis"] = {
+ "name": host,
+ "pxe": yaml_data["baremetal"][rack][host]["ip"]["pxe"],
+ "oam": yaml_data["baremetal"][rack][host]["ip"]["oam"],
}
- elif yaml_data['baremetal'][rack][host][
- 'type'] == 'controller':
- hosts['masters'].append(host)
+ elif (
+ yaml_data["baremetal"][rack][host]["type"] == "controller"
+ ):
+ hosts["masters"].append(host)
else:
- hosts['workers'].append(host)
+ hosts["workers"].append(host)
return hosts
diff --git a/spyglass/site_processors/site_processor.py b/spyglass/site_processors/site_processor.py
index 9f65e2d..bd4605d 100644
--- a/spyglass/site_processors/site_processor.py
+++ b/spyglass/site_processors/site_processor.py
@@ -36,12 +36,12 @@ class SiteProcessor(BaseProcessor):
"""
# Check of manifest_dir exists
if self.manifest_dir is not None:
- site_manifest_dir = self.manifest_dir + '/pegleg_manifests/site/'
+ site_manifest_dir = self.manifest_dir + "/pegleg_manifests/site/"
else:
- site_manifest_dir = 'pegleg_manifests/site/'
+ site_manifest_dir = "pegleg_manifests/site/"
LOG.info("Site manifest output dir:{}".format(site_manifest_dir))
- template_software_dir = template_dir + '/'
+ template_software_dir = template_dir + "/"
template_dir_abspath = os.path.dirname(template_software_dir)
LOG.debug("Template Path:%s", template_dir_abspath)
@@ -50,16 +50,19 @@ class SiteProcessor(BaseProcessor):
j2_env = Environment(
autoescape=False,
loader=FileSystemLoader(dirpath),
- trim_blocks=True)
+ trim_blocks=True,
+ )
j2_env.filters[
- 'get_role_wise_nodes'] = self.get_role_wise_nodes
+ "get_role_wise_nodes"
+ ] = self.get_role_wise_nodes
templatefile = os.path.join(dirpath, filename)
- outdirs = dirpath.split('templates')[1]
+ outdirs = dirpath.split("templates")[1]
- outfile_path = '{}{}{}'.format(
- site_manifest_dir, self.yaml_data['region_name'], outdirs)
- outfile_yaml = templatefile.split('.j2')[0].split('/')[-1]
- outfile = outfile_path + '/' + outfile_yaml
+ outfile_path = "{}{}{}".format(
+ site_manifest_dir, self.yaml_data["region_name"], outdirs
+ )
+ outfile_yaml = templatefile.split(".j2")[0].split("/")[-1]
+ outfile = outfile_path + "/" + outfile_yaml
outfile_dir = os.path.dirname(outfile)
if not os.path.exists(outfile_dir):
os.makedirs(outfile_dir)
@@ -71,7 +74,10 @@ class SiteProcessor(BaseProcessor):
out.close()
except IOError as ioe:
LOG.error(
- "IOError during rendering:{}".format(outfile_yaml))
+ "IOError during rendering:{}".format(outfile_yaml)
+ )
raise SystemExit(
"Error when generating {:s}:\n{:s}".format(
- outfile, ioe.strerror))
+ outfile, ioe.strerror
+ )
+ )
diff --git a/spyglass/spyglass.py b/spyglass/spyglass.py
index 0927eaf..9d53a3a 100644
--- a/spyglass/spyglass.py
+++ b/spyglass/spyglass.py
@@ -22,98 +22,115 @@ import yaml
from spyglass.parser.engine import ProcessDataSource
from spyglass.site_processors.site_processor import SiteProcessor
-LOG = logging.getLogger('spyglass')
+LOG = logging.getLogger("spyglass")
@click.command()
@click.option(
- '--site',
- '-s',
- help='Specify the site for which manifests to be generated')
+ "--site", "-s", help="Specify the site for which manifests to be generated"
+)
@click.option(
- '--type', '-t', help='Specify the plugin type formation or tugboat')
-@click.option('--formation_url', '-f', help='Specify the formation url')
-@click.option('--formation_user', '-u', help='Specify the formation user id')
+ "--type", "-t", help="Specify the plugin type formation or tugboat"
+)
+@click.option("--formation_url", "-f", help="Specify the formation url")
+@click.option("--formation_user", "-u", help="Specify the formation user id")
@click.option(
- '--formation_password', '-p', help='Specify the formation user password')
+ "--formation_password", "-p", help="Specify the formation user password"
+)
@click.option(
- '--intermediary',
- '-i',
+ "--intermediary",
+ "-i",
type=click.Path(exists=True),
- help=
- 'Intermediary file path generate manifests, use -m also with this option')
+ help=(
+ "Intermediary file path generate manifests, "
+ "use -m also with this option"
+ ),
+)
@click.option(
- '--additional_config',
- '-d',
+ "--additional_config",
+ "-d",
type=click.Path(exists=True),
- help='Site specific configuraton details')
+ help="Site specific configuraton details",
+)
@click.option(
- '--generate_intermediary',
- '-g',
+ "--generate_intermediary",
+ "-g",
is_flag=True,
- help='Dump intermediary file from passed excel and excel spec')
+ help="Dump intermediary file from passed excel and excel spec",
+)
@click.option(
- '--intermediary_dir',
- '-idir',
+ "--intermediary_dir",
+ "-idir",
type=click.Path(exists=True),
- help='The path where intermediary file needs to be generated')
+ help="The path where intermediary file needs to be generated",
+)
@click.option(
- '--edit_intermediary/--no_edit_intermediary',
- '-e/-nedit',
+ "--edit_intermediary/--no_edit_intermediary",
+ "-e/-nedit",
default=True,
- help='Flag to let user edit intermediary')
+ help="Flag to let user edit intermediary",
+)
@click.option(
- '--generate_manifests',
- '-m',
+ "--generate_manifests",
+ "-m",
is_flag=True,
- help='Generate manifests from the generated intermediary file')
+ help="Generate manifests from the generated intermediary file",
+)
@click.option(
- '--manifest_dir',
- '-mdir',
+ "--manifest_dir",
+ "-mdir",
type=click.Path(exists=True),
- help='The path where manifest files needs to be generated')
+ help="The path where manifest files needs to be generated",
+)
@click.option(
- '--template_dir',
- '-tdir',
+ "--template_dir",
+ "-tdir",
type=click.Path(exists=True),
- help='The path where J2 templates are available')
+ help="The path where J2 templates are available",
+)
@click.option(
- '--excel',
- '-x',
+ "--excel",
+ "-x",
multiple=True,
type=click.Path(exists=True),
- help=
- 'Path to engineering excel file, to be passed with generate_intermediary')
+ help=(
+ "Path to engineering excel file, to be passed with "
+ "generate_intermediary"
+ ),
+)
@click.option(
- '--excel_spec',
- '-e',
+ "--excel_spec",
+ "-e",
type=click.Path(exists=True),
- help='Path to excel spec, to be passed with generate_intermediary')
+ help="Path to excel spec, to be passed with generate_intermediary",
+)
@click.option(
- '--loglevel',
- '-l',
+ "--loglevel",
+ "-l",
default=20,
multiple=False,
show_default=True,
- help='Loglevel NOTSET:0 ,DEBUG:10, \
- INFO:20, WARNING:30, ERROR:40, CRITICAL:50')
+ help="Loglevel NOTSET:0 ,DEBUG:10, \
+ INFO:20, WARNING:30, ERROR:40, CRITICAL:50",
+)
def main(*args, **kwargs):
# Extract user provided inputs
- generate_intermediary = kwargs['generate_intermediary']
- intermediary_dir = kwargs['intermediary_dir']
- edit_intermediary = kwargs['edit_intermediary']
- generate_manifests = kwargs['generate_manifests']
- manifest_dir = kwargs['manifest_dir']
- intermediary = kwargs['intermediary']
- site = kwargs['site']
- template_dir = kwargs['template_dir']
- loglevel = kwargs['loglevel']
+ generate_intermediary = kwargs["generate_intermediary"]
+ intermediary_dir = kwargs["intermediary_dir"]
+ edit_intermediary = kwargs["edit_intermediary"]
+ generate_manifests = kwargs["generate_manifests"]
+ manifest_dir = kwargs["manifest_dir"]
+ intermediary = kwargs["intermediary"]
+ site = kwargs["site"]
+ template_dir = kwargs["template_dir"]
+ loglevel = kwargs["loglevel"]
# Set Logging format
LOG.setLevel(loglevel)
stream_handle = logging.StreamHandler()
formatter = logging.Formatter(
- '(%(name)s): %(asctime)s %(levelname)s %(message)s')
+ "(%(name)s): %(asctime)s %(levelname)s %(message)s"
+ )
stream_handle.setFormatter(formatter)
LOG.addHandler(stream_handle)
@@ -139,19 +156,21 @@ def main(*args, **kwargs):
intermediary_yaml = {}
if intermediary is None:
LOG.info("Generating Intermediary yaml")
- plugin_type = kwargs.get('type', None)
+ plugin_type = kwargs.get("type", None)
plugin_class = None
# Discover the plugin and load the plugin class
LOG.info("Load the plugin class")
for entry_point in pkg_resources.iter_entry_points(
- 'data_extractor_plugins'):
+ "data_extractor_plugins"
+ ):
if entry_point.name == plugin_type:
plugin_class = entry_point.load()
if plugin_class is None:
LOG.error(
- "Unsupported Plugin type. Plugin type:{}".format(plugin_type))
+ "Unsupported Plugin type. Plugin type:{}".format(plugin_type)
+ )
exit()
# Extract data from plugin data source
@@ -162,16 +181,22 @@ def main(*args, **kwargs):
data_extractor.extract_data()
# Apply any additional_config provided by user
- additional_config = kwargs.get('additional_config', None)
+ additional_config = kwargs.get("additional_config", None)
if additional_config is not None:
- with open(additional_config, 'r') as config:
+ with open(additional_config, "r") as config:
raw_data = config.read()
additional_config_data = yaml.safe_load(raw_data)
- LOG.debug("Additional config data:\n{}".format(
- pprint.pformat(additional_config_data)))
+ LOG.debug(
+ "Additional config data:\n{}".format(
+ pprint.pformat(additional_config_data)
+ )
+ )
- LOG.info("Apply additional configuration from:{}".format(
- additional_config))
+ LOG.info(
+ "Apply additional configuration from:{}".format(
+ additional_config
+ )
+ )
data_extractor.apply_additional_data(additional_config_data)
LOG.debug(pprint.pformat(data_extractor.site_data))
@@ -179,14 +204,16 @@ def main(*args, **kwargs):
LOG.info("Apply design rules to the extracted data")
process_input_ob = ProcessDataSource(site)
process_input_ob.load_extracted_data_from_data_source(
- data_extractor.site_data)
+ data_extractor.site_data
+ )
LOG.info("Generate intermediary yaml")
intermediary_yaml = process_input_ob.generate_intermediary_yaml(
- edit_intermediary)
+ edit_intermediary
+ )
else:
LOG.info("Loading intermediary from user provided input")
- with open(intermediary, 'r') as intermediary_file:
+ with open(intermediary, "r") as intermediary_file:
raw_data = intermediary_file.read()
intermediary_yaml = yaml.safe_load(raw_data)
@@ -201,5 +228,5 @@ def main(*args, **kwargs):
LOG.info("Spyglass Execution Completed")
-if __name__ == '__main__':
+if __name__ == "__main__":
main()
diff --git a/spyglass/utils/editor/editor.py b/spyglass/utils/editor/editor.py
index d2b9d04..726e0e3 100644
--- a/spyglass/utils/editor/editor.py
+++ b/spyglass/utils/editor/editor.py
@@ -26,49 +26,55 @@ from flask_bootstrap import Bootstrap
app_path = os.path.dirname(os.path.abspath(__file__))
-app = Flask('Yaml Editor!',
- template_folder=os.path.join(app_path, 'templates'),
- static_folder=os.path.join(app_path, 'static'))
+app = Flask(
+ "Yaml Editor!",
+ template_folder=os.path.join(app_path, "templates"),
+ static_folder=os.path.join(app_path, "static"),
+)
Bootstrap(app)
-logging.getLogger('werkzeug').setLevel(logging.ERROR)
+logging.getLogger("werkzeug").setLevel(logging.ERROR)
LOG = app.logger
-@app.route('/favicon.ico')
+@app.route("/favicon.ico")
def favicon():
- return send_from_directory(app.static_folder, 'favicon.ico')
+ return send_from_directory(app.static_folder, "favicon.ico")
-@app.route('/', methods=['GET', 'POST'])
+@app.route("/", methods=["GET", "POST"])
def index():
"""Renders index page to edit provided yaml file."""
- LOG.info('Rendering yaml file for editing')
- with open(app.config['YAML_FILE']) as file_obj:
+ LOG.info("Rendering yaml file for editing")
+ with open(app.config["YAML_FILE"]) as file_obj:
data = yaml.safe_load(file_obj)
- return render_template('yaml.html',
- data=json.dumps(data),
- change_str=app.config['STRING_TO_CHANGE'])
+ return render_template(
+ "yaml.html",
+ data=json.dumps(data),
+ change_str=app.config["STRING_TO_CHANGE"],
+ )
-@app.route('/save', methods=['POST'])
+@app.route("/save", methods=["POST"])
def save():
"""Save current progress on file."""
- LOG.info('Saving edited inputs from user to yaml file')
- out = request.json.get('yaml_data')
- with open(app.config['YAML_FILE'], 'w') as file_obj:
+ LOG.info("Saving edited inputs from user to yaml file")
+ out = request.json.get("yaml_data")
+ with open(app.config["YAML_FILE"], "w") as file_obj:
yaml.safe_dump(out, file_obj, default_flow_style=False)
return "Data saved successfully!"
-@app.route('/saveExit', methods=['POST'])
+@app.route("/saveExit", methods=["POST"])
def save_exit():
"""Save current progress on file and shuts down the server."""
- LOG.info('Saving edited inputs from user to yaml file and shutting'
- ' down server')
- out = request.json.get('yaml_data')
- with open(app.config['YAML_FILE'], 'w') as file_obj:
+ LOG.info(
+ "Saving edited inputs from user to yaml file and shutting"
+ " down server"
+ )
+ out = request.json.get("yaml_data")
+ with open(app.config["YAML_FILE"], "w") as file_obj:
yaml.safe_dump(out, file_obj, default_flow_style=False)
- func = request.environ.get('werkzeug.server.shutdown')
+ func = request.environ.get("werkzeug.server.shutdown")
if func:
func()
return "Saved successfully, Shutting down app! You may close the tab!"
@@ -77,68 +83,72 @@ def save_exit():
@app.errorhandler(404)
def page_not_found(e):
"""Serves 404 error."""
- LOG.info('User tried to access unavailable page.')
- return '404: Page not Found!
'
+ LOG.info("User tried to access unavailable page.")
+ return "404: Page not Found!
"
def run(*args, **kwargs):
"""Starts the server."""
- LOG.info('Initiating web server for yaml editing')
- port = kwargs.get('port', None)
+ LOG.info("Initiating web server for yaml editing")
+ port = kwargs.get("port", None)
if not port:
port = 8161
- app.run(host='0.0.0.0', port=port, debug=False)
+ app.run(host="0.0.0.0", port=port, debug=False)
@click.command()
@click.option(
- '--file',
- '-f',
+ "--file",
+ "-f",
required=True,
type=click.File(),
multiple=False,
- help="Path with file name to the intermediary yaml file."
+ help="Path with file name to the intermediary yaml file.",
)
@click.option(
- '--host',
- '-h',
- default='0.0.0.0',
+ "--host",
+ "-h",
+ default="0.0.0.0",
type=click.STRING,
multiple=False,
- help="Optional host parameter to run Flask on."
+ help="Optional host parameter to run Flask on.",
)
@click.option(
- '--port',
- '-p',
+ "--port",
+ "-p",
default=8161,
type=click.INT,
multiple=False,
- help="Optional port parameter to run Flask on."
+ help="Optional port parameter to run Flask on.",
)
@click.option(
- '--string',
- '-s',
- default='#CHANGE_ME',
+ "--string",
+ "-s",
+ default="#CHANGE_ME",
type=click.STRING,
multiple=False,
- help="Text which is required to be changed on yaml file."
+ help="Text which is required to be changed on yaml file.",
)
def main(*args, **kwargs):
LOG.setLevel(logging.INFO)
- LOG.info('Initiating yaml-editor')
+ LOG.info("Initiating yaml-editor")
try:
- yaml.safe_load(kwargs['file'])
+ yaml.safe_load(kwargs["file"])
except yaml.YAMLError as e:
- LOG.error('EXITTING - Please provide a valid yaml file.')
- if hasattr(e, 'problem_mark'):
+ LOG.error("EXITTING - Please provide a valid yaml file.")
+ if hasattr(e, "problem_mark"):
mark = e.problem_mark
- LOG.error("Error position: ({0}:{1})".format(
- mark.line + 1, mark.column + 1))
+ LOG.error(
+ "Error position: ({0}:{1})".format(
+ mark.line + 1, mark.column + 1
+ )
+ )
sys.exit(2)
except Exception:
- LOG.error('EXITTING - Please provide a valid yaml file.')
+ LOG.error("EXITTING - Please provide a valid yaml file.")
sys.exit(2)
- LOG.info("""
+ LOG.info(
+ """
##############################################################################
@@ -146,12 +156,15 @@ Please go to http://{0}:{1}/ to edit your yaml file.
##############################################################################
- """.format(kwargs['host'], kwargs['port']))
- app.config['YAML_FILE'] = kwargs['file'].name
- app.config['STRING_TO_CHANGE'] = kwargs['string']
+ """.format(
+ kwargs["host"], kwargs["port"]
+ )
+ )
+ app.config["YAML_FILE"] = kwargs["file"].name
+ app.config["STRING_TO_CHANGE"] = kwargs["string"]
run(*args, **kwargs)
-if __name__ == '__main__':
+if __name__ == "__main__":
"""Invoked when used as a script."""
main()