airship
/
spyglass
Code Issues Proposed changes

Various fixes to Spyglass to resolve tox errors 

The tox configuration needs to be updated in order for it to be helpful
for Zuul. This change makes some minor modifications to the tox config
to improve its usability and uniformity with other Open Stack configs.

Makes bare minimum changes for pep8 tests compliance.

Adds gate scripts for white space linting.

In the future, additional changes will need to be made to add tests to
Spyglass. These changes will need to enable the 'py36' and 'cover' tox
env and set coverage levels. Additional linting should also be
performed to improve Spyglass code readability and organization.

Change-Id: I73f946619786e661d02c69bd8e197453f049e0c7
This commit is contained in:
Alexander Hughes 2019-03-29 14:58:13 -05:00
parent 8dd891aced
commit 03a058bfcf
15 changed files with 201 additions and 152 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
spyglass/data_extractor/base.py
View File

@ -1,21 +1,21 @@
# Copyright 2018 AT&T Intellectual Property. All other rights reserved. # Copyright 2018 AT&T Intellectual Property. All other rights reserved.
# #
# Licensed under the Apache License, Version 2.0 (the 'License'); # Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License. # you may not use this file except in compliance with the License.
# You may obtain a copy of the License at # You may obtain a copy of the License at
# #
# http://www.apache.org/licenses/LICENSE-2.0 # http://www.apache.org/licenses/LICENSE-2.0
# #
# Unless required by applicable law or agreed to in writing, software # Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an 'AS IS' BASIS, # distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import abc import abc
import logging
import pprint import pprint
import six import six
import logging
from spyglass.utils import utils from spyglass.utils import utils
@ -34,8 +34,7 @@ class BaseDataSourcePlugin(object):
@abc.abstractmethod @abc.abstractmethod
def set_config_opts(self, conf): def set_config_opts(self, conf):
"""Placeholder to set configuration options """Placeholder to set configuration options specific to each plugin.
specific to each plugin.
:param dict conf: Configuration options as dict :param dict conf: Configuration options as dict
@ -44,19 +43,21 @@ class BaseDataSourcePlugin(object):
Each plugin will have their own config opts. Each plugin will have their own config opts.
""" """
return return
@abc.abstractmethod @abc.abstractmethod
def get_plugin_conf(self, kwargs): def get_plugin_conf(self, kwargs):
""" Validate and returns the plugin config parameters. """Validate and returns the plugin config parameters.
If validation fails, Spyglass exits. If validation fails, Spyglass exits.
:param char pointer: Spyglass CLI parameters. :param char pointer: Spyglass CLI parameters.
:returns plugin conf if successfully validated. :returns plugin conf if successfully validated.
Each plugin implements their own validaton mechanism. Each plugin implements their own validaton mechanism.
""" """
return {} return {}
@abc.abstractmethod @abc.abstractmethod
@ -64,13 +65,12 @@ class BaseDataSourcePlugin(object):
"""Return list of racks in the region """Return list of racks in the region
:param string region: Region name :param string region: Region name
:returns: list of rack names :returns: list of rack names
:rtype: list :rtype: list
Example: ['rack01', 'rack02'] Example: ['rack01', 'rack02']
""" """
return [] return []
@abc.abstractmethod @abc.abstractmethod
@ -79,9 +79,7 @@ class BaseDataSourcePlugin(object):
:param string region: Region name :param string region: Region name
:param string rack: Rack name :param string rack: Rack name
:returns: list of hosts information :returns: list of hosts information
:rtype: list of dict :rtype: list of dict
Example: [ Example: [
@ -96,6 +94,7 @@ class BaseDataSourcePlugin(object):
'host_profile': 'hp_02'} 'host_profile': 'hp_02'}
] ]
""" """
return [] return []
@abc.abstractmethod @abc.abstractmethod
@ -103,9 +102,7 @@ class BaseDataSourcePlugin(object):
"""Return list of networks in the region """Return list of networks in the region
:param string region: Region name :param string region: Region name
:returns: list of networks and their vlans :returns: list of networks and their vlans
:rtype: list of dict :rtype: list of dict
Example: [ Example: [
@ -158,9 +155,7 @@ class BaseDataSourcePlugin(object):
:param string region: Region name :param string region: Region name
:param string host: Host name :param string host: Host name
:returns: Dict of IPs per network on the host :returns: Dict of IPs per network on the host
:rtype: dict :rtype: dict
Example: {'oob': {'ipv4': '192.168.1.10'}, Example: {'oob': {'ipv4': '192.168.1.10'},
@ -170,6 +165,7 @@ class BaseDataSourcePlugin(object):
dict. In case some networks are missed, they are expected to be either dict. In case some networks are missed, they are expected to be either
DHCP or internally generated n the next steps by the design rules. DHCP or internally generated n the next steps by the design rules.
""" """
return {} return {}
@abc.abstractmethod @abc.abstractmethod
@ -177,13 +173,12 @@ class BaseDataSourcePlugin(object):
"""Return the DNS servers """Return the DNS servers
:param string region: Region name :param string region: Region name
:returns: List of DNS servers to be configured on host :returns: List of DNS servers to be configured on host
:rtype: List :rtype: List
Example: ['8.8.8.8', '8.8.8.4'] Example: ['8.8.8.8', '8.8.8.4']
""" """
return [] return []
@abc.abstractmethod @abc.abstractmethod
@ -191,13 +186,12 @@ class BaseDataSourcePlugin(object):
"""Return the NTP servers """Return the NTP servers
:param string region: Region name :param string region: Region name
:returns: List of NTP servers to be configured on host :returns: List of NTP servers to be configured on host
:rtype: List :rtype: List
Example: ['ntp1.ubuntu1.example', 'ntp2.ubuntu.example'] Example: ['ntp1.ubuntu1.example', 'ntp2.ubuntu.example']
""" """
return [] return []
@abc.abstractmethod @abc.abstractmethod
@ -205,9 +199,7 @@ class BaseDataSourcePlugin(object):
"""Return the LDAP server information """Return the LDAP server information
:param string region: Region name :param string region: Region name
:returns: LDAP server information :returns: LDAP server information
:rtype: Dict :rtype: Dict
Example: {'url': 'ldap.example.com', Example: {'url': 'ldap.example.com',
@ -215,6 +207,7 @@ class BaseDataSourcePlugin(object):
'domain': 'test', 'domain': 'test',
'subdomain': 'test_sub1'} 'subdomain': 'test_sub1'}
""" """
return {} return {}
@abc.abstractmethod @abc.abstractmethod
@ -222,9 +215,7 @@ class BaseDataSourcePlugin(object):
"""Return location information """Return location information
:param string region: Region name :param string region: Region name
:returns: Dict of location information :returns: Dict of location information
:rtype: dict :rtype: dict
Example: {'name': 'Dallas', Example: {'name': 'Dallas',
@ -233,6 +224,7 @@ class BaseDataSourcePlugin(object):
'country': 'US', 'country': 'US',
'corridor': 'CR1'} 'corridor': 'CR1'}
""" """
return {} return {}
@abc.abstractmethod @abc.abstractmethod
@ -240,20 +232,18 @@ class BaseDataSourcePlugin(object):
"""Return the Domain name """Return the Domain name
:param string region: Region name :param string region: Region name
:returns: Domain name :returns: Domain name
:rtype: string :rtype: string
Example: example.com Example: example.com
""" """
return "" return ""
def extract_baremetal_information(self): def extract_baremetal_information(self):
"""Get baremetal information from plugin """Get baremetal information from plugin
:returns: dict of baremetal nodes :returns: dict of baremetal nodes
:rtype: dict :rtype: dict
Return dict should be in the format Return dict should be in the format
@ -275,6 +265,7 @@ class BaseDataSourcePlugin(object):
} }
} }
""" """
LOG.info("Extract baremetal information from plugin") LOG.info("Extract baremetal information from plugin")
baremetal = {} baremetal = {}
hosts = self.get_hosts(self.region) hosts = self.get_hosts(self.region)
@ -330,7 +321,6 @@ class BaseDataSourcePlugin(object):
"""Get site information from plugin """Get site information from plugin
:returns: dict of site information :returns: dict of site information
:rtpe: dict :rtpe: dict
Return dict should be in the format Return dict should be in the format
@ -346,6 +336,7 @@ class BaseDataSourcePlugin(object):
'domain': None 'domain': None
} }
""" """
LOG.info("Extract site information from plugin") LOG.info("Extract site information from plugin")
site_info = {} site_info = {}
@ -373,11 +364,9 @@ class BaseDataSourcePlugin(object):
return site_info return site_info
def extract_network_information(self): def extract_network_information(self):
"""Get network information from plugin """Get network details from plugin like Subnets, DNS, NTP and LDAP
like Subnets, DNS, NTP, LDAP details.
:returns: dict of baremetal nodes :returns: dict of baremetal nodes
:rtype: dict :rtype: dict
Return dict should be in the format Return dict should be in the format
@ -393,6 +382,7 @@ class BaseDataSourcePlugin(object):
} }
} }
""" """
LOG.info("Extract network information from plugin") LOG.info("Extract network information from plugin")
network_data = {} network_data = {}
networks = self.get_networks(self.region) networks = self.get_networks(self.region)
@ -431,6 +421,7 @@ class BaseDataSourcePlugin(object):
Gather data related to baremetal, networks, storage and other site Gather data related to baremetal, networks, storage and other site
related information from plugin related information from plugin
""" """
LOG.info("Extract data from plugin") LOG.info("Extract data from plugin")
site_data = {} site_data = {}
site_data["baremetal"] = self.extract_baremetal_information() site_data["baremetal"] = self.extract_baremetal_information()
@ -448,6 +439,7 @@ class BaseDataSourcePlugin(object):
If there is repetition of data then additional data supplied If there is repetition of data then additional data supplied
shall take precedence. shall take precedence.
""" """
LOG.info("Update site data with additional input") LOG.info("Update site data with additional input")
tmp_site_data = utils.dict_merge(self.site_data, extra_data) tmp_site_data = utils.dict_merge(self.site_data, extra_data)
self.site_data = tmp_site_data self.site_data = tmp_site_data

7
spyglass/data_extractor/custom_exceptions.py
View File

@ -1,16 +1,17 @@
# Copyright 2018 AT&T Intellectual Property. All other rights reserved. # Copyright 2018 AT&T Intellectual Property. All other rights reserved.
# #
# Licensed under the Apache License, Version 2.0 (the 'License'); # Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License. # you may not use this file except in compliance with the License.
# You may obtain a copy of the License at # You may obtain a copy of the License at
# #
# http://www.apache.org/licenses/LICENSE-2.0 # http://www.apache.org/licenses/LICENSE-2.0
# #
# Unless required by applicable law or agreed to in writing, software # Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an 'AS IS' BASIS, # distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import logging import logging
import sys import sys

41
spyglass/data_extractor/plugins/formation.py
View File

@ -1,32 +1,27 @@
# Copyright 2018 AT&T Intellectual Property. All other rights reserved. # Copyright 2018 AT&T Intellectual Property. All other rights reserved.
# #
# Licensed under the Apache License, Version 2.0 (the 'License'); # Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License. # you may not use this file except in compliance with the License.
# You may obtain a copy of the License at # You may obtain a copy of the License at
# #
# http://www.apache.org/licenses/LICENSE-2.0 # http://www.apache.org/licenses/LICENSE-2.0
# #
# Unless required by applicable law or agreed to in writing, software # Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an 'AS IS' BASIS, # distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import formation_client
import logging import logging
import pprint import pprint
import re import re
import requests import requests
import formation_client
import urllib3 import urllib3
from spyglass.data_extractor.base import BaseDataSourcePlugin from spyglass.data_extractor.base import BaseDataSourcePlugin
from spyglass.data_extractor.custom_exceptions import ( import spyglass.data_extractor.custom_exceptions as exceptions
ApiClientError,
ConnectionError,
MissingAttributeError,
TokenGenerationError,
)
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
@ -65,7 +60,8 @@ class FormationPlugin(BaseDataSourcePlugin):
LOG.info("Initiated data extractor plugin:{}".format(self.source_name)) LOG.info("Initiated data extractor plugin:{}".format(self.source_name))
def set_config_opts(self, conf): def set_config_opts(self, conf):
""" Sets the config params passed by CLI""" """Sets the config params passed by CLI"""
LOG.info("Plugin params passed:\n{}".format(pprint.pformat(conf))) LOG.info("Plugin params passed:\n{}".format(pprint.pformat(conf)))
self._validate_config_options(conf) self._validate_config_options(conf)
self.formation_api_url = conf["url"] self.formation_api_url = conf["url"]
@ -77,7 +73,7 @@ class FormationPlugin(BaseDataSourcePlugin):
self._update_site_and_zone(self.region) self._update_site_and_zone(self.region)
def get_plugin_conf(self, kwargs): def get_plugin_conf(self, kwargs):
""" Validates the plugin param and return if success""" """Validates the plugin param and return if success"""
if not kwargs["formation_url"]: if not kwargs["formation_url"]:
LOG.error("formation_url not specified! Spyglass exited!") LOG.error("formation_url not specified! Spyglass exited!")
@ -116,10 +112,12 @@ class FormationPlugin(BaseDataSourcePlugin):
def _generate_token(self): def _generate_token(self):
"""Generate token for Formation """Generate token for Formation
Formation API does not provide separate resource to generate Formation API does not provide separate resource to generate
token. This is a workaround to call directly Formation API token. This is a workaround to call directly Formation API
to get token instead of using Formation client. to get token instead of using Formation client.
""" """
# Create formation client config object # Create formation client config object
self.client_config = formation_client.Configuration() self.client_config = formation_client.Configuration()
self.client_config.host = self.formation_api_url self.client_config.host = self.formation_api_url
@ -138,13 +136,13 @@ class FormationPlugin(BaseDataSourcePlugin):
auth=(self.user, self.password), auth=(self.user, self.password),
verify=self.client_config.verify_ssl, verify=self.client_config.verify_ssl,
) )
except requests.exceptions.ConnectionError: except requests.exceptions.exceptions.ConnectionError:
raise ConnectionError("Incorrect URL: {}".format(url)) raise exceptions.ConnectionError("Incorrect URL: {}".format(url))
if token_response.status_code == 200: if token_response.status_code == 200:
self.token = token_response.json().get("X-Subject-Token", None) self.token = token_response.json().get("X-Subject-Token", None)
else: else:
raise TokenGenerationError( raise exceptions.TokenGenerationError(
"Unable to generate token because {}".format( "Unable to generate token because {}".format(
token_response.reason token_response.reason
) )
@ -159,6 +157,7 @@ class FormationPlugin(BaseDataSourcePlugin):
format "user|token". format "user|token".
Generate the token and add it formation config object. Generate the token and add it formation config object.
""" """
token = self._generate_token() token = self._generate_token()
self.client_config.api_key = {"X-Auth-Token": self.user + "|" + token} self.client_config.api_key = {"X-Auth-Token": self.user + "|" + token}
self.formation_api_client = formation_client.ApiClient( self.formation_api_client = formation_client.ApiClient(
@ -428,7 +427,7 @@ class FormationPlugin(BaseDataSourcePlugin):
return ip_ return ip_
def _get_network_name_from_vlan_name(self, vlan_name): def _get_network_name_from_vlan_name(self, vlan_name):
""" network names are ksn, oam, oob, overlay, storage, pxe """Network names are ksn, oam, oob, overlay, storage, pxe
The following mapping rules apply: The following mapping rules apply:
vlan_name contains "ksn" the network name is "calico" vlan_name contains "ksn" the network name is "calico"
@ -437,6 +436,7 @@ class FormationPlugin(BaseDataSourcePlugin):
vlan_name contains "ovs" the network name is "overlay" vlan_name contains "ovs" the network name is "overlay"
vlan_name contains "ILO" the network name is "oob" vlan_name contains "ILO" the network name is "oob"
""" """
network_names = { network_names = {
"ksn": "calico", "ksn": "calico",
"storage": "storage", "storage": "storage",
@ -462,7 +462,7 @@ class FormationPlugin(BaseDataSourcePlugin):
zone_api = formation_client.ZonesApi(self.formation_api_client) zone_api = formation_client.ZonesApi(self.formation_api_client)
zone_ = zone_api.zones_zone_id_get(zone_id) zone_ = zone_api.zones_zone_id_get(zone_id)
except formation_client.rest.ApiException as e: except formation_client.rest.ApiException as e:
raise ApiClientError(e.msg) raise exceptions.ApiClientError(e.msg)
if not zone_.ipv4_dns: if not zone_.ipv4_dns:
LOG.warn("No dns server") LOG.warn("No dns server")
@ -481,7 +481,8 @@ class FormationPlugin(BaseDataSourcePlugin):
return {} return {}
def get_location_information(self, region): def get_location_information(self, region):
""" get location information for a zone and return """ """Get location information for a zone and return"""
site = self.region_zone_map[region]["site"] site = self.region_zone_map[region]["site"]
site_id = self._get_site_id_by_name(site) site_id = self._get_site_id_by_name(site)
site_api = formation_client.SitesApi(self.formation_api_client) site_api = formation_client.SitesApi(self.formation_api_client)
@ -496,7 +497,7 @@ class FormationPlugin(BaseDataSourcePlugin):
"physical_location_id": site_info.clli, "physical_location_id": site_info.clli,
} }
except AttributeError as e: except AttributeError as e:
raise MissingAttributeError( raise exceptions.MissingAttributeError(
"Missing {} information in {}".format(e, site_info.city) "Missing {} information in {}".format(e, site_info.city)
) )
@ -507,7 +508,7 @@ class FormationPlugin(BaseDataSourcePlugin):
zone_api = formation_client.ZonesApi(self.formation_api_client) zone_api = formation_client.ZonesApi(self.formation_api_client)
zone_ = zone_api.zones_zone_id_get(zone_id) zone_ = zone_api.zones_zone_id_get(zone_id)
except formation_client.rest.ApiException as e: except formation_client.rest.ApiException as e:
raise ApiClientError(e.msg) raise exceptions.ApiClientError(e.msg)
if not zone_.dns: if not zone_.dns:
LOG.warn("Got None while running get domain name") LOG.warn("Got None while running get domain name")

2
spyglass/data_extractor/plugins/tugboat/check_exceptions.py
View File

@ -4,7 +4,7 @@
# you may not use this file except in compliance with the License. # you may not use this file except in compliance with the License.
# You may obtain a copy of the License at # You may obtain a copy of the License at
# #
# http://www.apache.org/licenses/LICENSE-2.0 # http://www.apache.org/licenses/LICENSE-2.0
# #
# Unless required by applicable law or agreed to in writing, software # Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, # distributed under the License is distributed on an "AS IS" BASIS,

57
spyglass/data_extractor/plugins/tugboat/excel_parser.py
View File

@ -4,7 +4,7 @@
# you may not use this file except in compliance with the License. # you may not use this file except in compliance with the License.
# You may obtain a copy of the License at # You may obtain a copy of the License at
# #
# http://www.apache.org/licenses/LICENSE-2.0 # http://www.apache.org/licenses/LICENSE-2.0
# #
# Unless required by applicable law or agreed to in writing, software # Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, # distributed under the License is distributed on an "AS IS" BASIS,
@ -13,12 +13,13 @@
# limitations under the License. # limitations under the License.
import logging import logging
from openpyxl import load_workbook
from openpyxl import Workbook
import pprint import pprint
import re import re
import sys import sys
import yaml import yaml
from openpyxl import load_workbook
from openpyxl import Workbook
from spyglass.data_extractor.custom_exceptions import NoSpecMatched from spyglass.data_extractor.custom_exceptions import NoSpecMatched
# from spyglass.data_extractor.custom_exceptions # from spyglass.data_extractor.custom_exceptions
@ -26,8 +27,8 @@ from spyglass.data_extractor.custom_exceptions import NoSpecMatched
LOG = logging.getLogger(__name__) LOG = logging.getLogger(__name__)
class ExcelParser: class ExcelParser(object):
""" Parse data from excel into a dict """ """Parse data from excel into a dict"""
def __init__(self, file_name, excel_specs): def __init__(self, file_name, excel_specs):
self.file_name = file_name self.file_name = file_name
@ -43,15 +44,18 @@ class ExcelParser:
@staticmethod @staticmethod
def sanitize(string): def sanitize(string):
""" Remove extra spaces and convert string to lower case """ """Remove extra spaces and convert string to lower case"""
return string.replace(" ", "").lower() return string.replace(" ", "").lower()
def compare(self, string1, string2): def compare(self, string1, string2):
""" Compare the strings """ """Compare the strings"""
return bool(re.search(self.sanitize(string1), self.sanitize(string2))) return bool(re.search(self.sanitize(string1), self.sanitize(string2)))
def validate_sheet(self, spec, sheet): def validate_sheet(self, spec, sheet):
""" Check if the sheet is correct or not """ """Check if the sheet is correct or not"""
ws = self.wb_combined[sheet] ws = self.wb_combined[sheet]
header_row = self.excel_specs["specs"][spec]["header_row"] header_row = self.excel_specs["specs"][spec]["header_row"]
ipmi_header = self.excel_specs["specs"][spec]["ipmi_address_header"] ipmi_header = self.excel_specs["specs"][spec]["ipmi_address_header"]
@ -60,7 +64,8 @@ class ExcelParser:
return bool(self.compare(ipmi_header, header_value)) return bool(self.compare(ipmi_header, header_value))
def find_correct_spec(self): def find_correct_spec(self):
""" Find the correct spec """ """Find the correct spec"""
for spec in self.excel_specs["specs"]: for spec in self.excel_specs["specs"]:
sheet_name = self.excel_specs["specs"][spec]["ipmi_sheet_name"] sheet_name = self.excel_specs["specs"][spec]["ipmi_sheet_name"]
for sheet in self.wb_combined.sheetnames: for sheet in self.wb_combined.sheetnames:
@ -71,7 +76,8 @@ class ExcelParser:
raise NoSpecMatched(self.excel_specs) raise NoSpecMatched(self.excel_specs)
def get_ipmi_data(self): def get_ipmi_data(self):
""" Read IPMI data from the sheet """ """Read IPMI data from the sheet"""
ipmi_data = {} ipmi_data = {}
hosts = [] hosts = []
provided_sheetname = self.excel_specs["specs"][self.spec][ provided_sheetname = self.excel_specs["specs"][self.spec][
@ -140,7 +146,8 @@ class ExcelParser:
return [ipmi_data, hosts] return [ipmi_data, hosts]
def get_private_vlan_data(self, ws): def get_private_vlan_data(self, ws):
""" Get private vlan data from private IP sheet """ """Get private vlan data from private IP sheet"""
vlan_data = {} vlan_data = {}
row = self.excel_specs["specs"][self.spec]["vlan_start_row"] row = self.excel_specs["specs"][self.spec]["vlan_start_row"]
end_row = self.excel_specs["specs"][self.spec]["vlan_end_row"] end_row = self.excel_specs["specs"][self.spec]["vlan_end_row"]
@ -160,7 +167,8 @@ class ExcelParser:
return vlan_data return vlan_data
def get_private_network_data(self): def get_private_network_data(self):
""" Read network data from the private ip sheet """ """Read network data from the private ip sheet"""
provided_sheetname = self.excel_specs["specs"][self.spec][ provided_sheetname = self.excel_specs["specs"][self.spec][
"private_ip_sheet" "private_ip_sheet"
] ]
@ -211,7 +219,8 @@ class ExcelParser:
return network_data return network_data
def get_public_network_data(self): def get_public_network_data(self):
""" Read public network data from public ip data """ """Read public network data from public ip data"""
network_data = {} network_data = {}
provided_sheetname = self.excel_specs["specs"][self.spec][ provided_sheetname = self.excel_specs["specs"][self.spec][
"public_ip_sheet" "public_ip_sheet"
@ -251,7 +260,8 @@ class ExcelParser:
return network_data return network_data
def get_site_info(self): def get_site_info(self):
""" Read location, dns, ntp and ldap data""" """Read location, dns, ntp and ldap data"""
site_info = {} site_info = {}
provided_sheetname = self.excel_specs["specs"][self.spec][ provided_sheetname = self.excel_specs["specs"][self.spec][
"dns_ntp_ldap_sheet" "dns_ntp_ldap_sheet"
@ -326,7 +336,8 @@ class ExcelParser:
return site_info return site_info
def get_location_data(self): def get_location_data(self):
""" Read location data from the site and zone sheet """ """Read location data from the site and zone sheet"""
provided_sheetname = self.excel_specs["specs"][self.spec][ provided_sheetname = self.excel_specs["specs"][self.spec][
"location_sheet" "location_sheet"
] ]
@ -356,7 +367,8 @@ class ExcelParser:
} }
def validate_sheet_names_with_spec(self): def validate_sheet_names_with_spec(self):
""" Checks is sheet name in spec file matches with excel file""" """Checks is sheet name in spec file matches with excel file"""
spec = list(self.excel_specs["specs"].keys())[0] spec = list(self.excel_specs["specs"].keys())[0]
spec_item = self.excel_specs["specs"][spec] spec_item = self.excel_specs["specs"][spec]
sheet_name_list = [] sheet_name_list = []
@ -391,7 +403,8 @@ class ExcelParser:
LOG.info("Sheet names in excel spec validated") LOG.info("Sheet names in excel spec validated")
def get_data(self): def get_data(self):
""" Create a dict with combined data """ """Create a dict with combined data"""
self.validate_sheet_names_with_spec() self.validate_sheet_names_with_spec()
ipmi_data = self.get_ipmi_data() ipmi_data = self.get_ipmi_data()
network_data = self.get_private_network_data() network_data = self.get_private_network_data()
@ -413,7 +426,8 @@ class ExcelParser:
return data return data
def combine_excel_design_specs(self, filenames): def combine_excel_design_specs(self, filenames):
""" Combines multiple excel file to a single design spec""" """Combines multiple excel file to a single design spec"""
design_spec = Workbook() design_spec = Workbook()
for exel_file in filenames: for exel_file in filenames:
loaded_workbook = load_workbook(exel_file, data_only=True) loaded_workbook = load_workbook(exel_file, data_only=True)
@ -428,10 +442,11 @@ class ExcelParser:
return design_spec return design_spec
def get_xl_obj_and_sheetname(self, sheetname): def get_xl_obj_and_sheetname(self, sheetname):
"""The logic confirms if the sheetname is specified for example as:
'MTN57a_AEC_Network_Design_v1.6.xlsx:Public IPs'
""" """
The logic confirms if the sheetname is specified for example as:
"MTN57a_AEC_Network_Design_v1.6.xlsx:Public IPs"
"""
if re.search(".xlsx", sheetname) or re.search(".xls", sheetname): if re.search(".xlsx", sheetname) or re.search(".xls", sheetname):
""" Extract file name """ """ Extract file name """
source_xl_file = sheetname.split(":")[0] source_xl_file = sheetname.split(":")[0]

55
spyglass/data_extractor/plugins/tugboat/tugboat.py
View File

@ -1,13 +1,13 @@
# Copyright 2018 AT&T Intellectual Property. All other rights reserved. # Copyright 2018 AT&T Intellectual Property. All other rights reserved.
# #
# Licensed under the Apache License, Version 2.0 (the 'License'); # Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License. # you may not use this file except in compliance with the License.
# You may obtain a copy of the License at # You may obtain a copy of the License at
# #
# http://www.apache.org/licenses/LICENSE-2.0 # http://www.apache.org/licenses/LICENSE-2.0
# #
# Unless required by applicable law or agreed to in writing, software # Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an 'AS IS' BASIS, # distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
@ -41,9 +41,7 @@ class TugboatPlugin(BaseDataSourcePlugin):
LOG.info("Initiated data extractor plugin:{}".format(self.source_name)) LOG.info("Initiated data extractor plugin:{}".format(self.source_name))
def set_config_opts(self, conf): def set_config_opts(self, conf):
""" """Placeholder to set configuration options specific to each plugin.
Placeholder to set configuration options
specific to each plugin.
:param dict conf: Configuration options as dict :param dict conf: Configuration options as dict
@ -52,6 +50,7 @@ class TugboatPlugin(BaseDataSourcePlugin):
Each plugin will have their own config opts. Each plugin will have their own config opts.
""" """
self.excel_path = conf["excel_path"] self.excel_path = conf["excel_path"]
self.excel_spec = conf["excel_spec"] self.excel_spec = conf["excel_spec"]
@ -61,8 +60,7 @@ class TugboatPlugin(BaseDataSourcePlugin):
return return
def get_plugin_conf(self, kwargs): def get_plugin_conf(self, kwargs):
""" Validates the plugin param from CLI and return if correct """Validates the plugin param from CLI and return if correct
Ideally the CLICK module shall report an error if excel file Ideally the CLICK module shall report an error if excel file
and excel specs are not specified. The below code has been and excel specs are not specified. The below code has been
@ -84,6 +82,7 @@ class TugboatPlugin(BaseDataSourcePlugin):
def get_hosts(self, region, rack=None): def get_hosts(self, region, rack=None):
"""Return list of hosts in the region """Return list of hosts in the region
:param string region: Region name :param string region: Region name
:param string rack: Rack name :param string rack: Rack name
:returns: list of hosts information :returns: list of hosts information
@ -100,6 +99,7 @@ class TugboatPlugin(BaseDataSourcePlugin):
'host_profile': 'hp_02'} 'host_profile': 'hp_02'}
] ]
""" """
LOG.info("Get Host Information") LOG.info("Get Host Information")
ipmi_data = self.parsed_xl_data["ipmi_data"][0] ipmi_data = self.parsed_xl_data["ipmi_data"][0]
rackwise_hosts = self._get_rackwise_hosts() rackwise_hosts = self._get_rackwise_hosts()
@ -116,7 +116,8 @@ class TugboatPlugin(BaseDataSourcePlugin):
return host_list return host_list
def get_networks(self, region): def get_networks(self, region):
""" Extracts vlan network info from raw network data from excel""" """Extracts vlan network info from raw network data from excel"""
vlan_list = [] vlan_list = []
# Network data extracted from xl is formatted to have a predictable # Network data extracted from xl is formatted to have a predictable
# data type. For e.g VlAN 45 extracted from xl is formatted as 45 # data type. For e.g VlAN 45 extracted from xl is formatted as 45
@ -162,6 +163,7 @@ class TugboatPlugin(BaseDataSourcePlugin):
def get_ips(self, region, host=None): def get_ips(self, region, host=None):
"""Return list of IPs on the host """Return list of IPs on the host
:param string region: Region name :param string region: Region name
:param string host: Host name :param string host: Host name
:returns: Dict of IPs per network on the host :returns: Dict of IPs per network on the host
@ -186,7 +188,7 @@ class TugboatPlugin(BaseDataSourcePlugin):
return ip_ return ip_
def get_ldap_information(self, region): def get_ldap_information(self, region):
""" Extract ldap information from excel""" """Extract ldap information from excel"""
ldap_raw_data = self.parsed_xl_data["site_info"]["ldap"] ldap_raw_data = self.parsed_xl_data["site_info"]["ldap"]
ldap_info = {} ldap_info = {}
@ -206,7 +208,7 @@ class TugboatPlugin(BaseDataSourcePlugin):
return ldap_info return ldap_info
def get_ntp_servers(self, region): def get_ntp_servers(self, region):
""" Returns a comma separated list of ntp ip addresses""" """Returns a comma separated list of ntp ip addresses"""
ntp_server_list = self._get_formatted_server_list( ntp_server_list = self._get_formatted_server_list(
self.parsed_xl_data["site_info"]["ntp"] self.parsed_xl_data["site_info"]["ntp"]
@ -214,22 +216,23 @@ class TugboatPlugin(BaseDataSourcePlugin):
return ntp_server_list return ntp_server_list
def get_dns_servers(self, region): def get_dns_servers(self, region):
""" Returns a comma separated list of dns ip addresses""" """Returns a comma separated list of dns ip addresses"""
dns_server_list = self._get_formatted_server_list( dns_server_list = self._get_formatted_server_list(
self.parsed_xl_data["site_info"]["dns"] self.parsed_xl_data["site_info"]["dns"]
) )
return dns_server_list return dns_server_list
def get_domain_name(self, region): def get_domain_name(self, region):
""" Returns domain name extracted from excel file""" """Returns domain name extracted from excel file"""
return self.parsed_xl_data["site_info"]["domain"] return self.parsed_xl_data["site_info"]["domain"]
def get_location_information(self, region): def get_location_information(self, region):
""" """Prepare location data from information extracted
Prepare location data from information extracted
by ExcelParser(i.e raw data) by ExcelParser(i.e raw data)
""" """
location_data = self.parsed_xl_data["site_info"]["location"] location_data = self.parsed_xl_data["site_info"]["location"]
corridor_pattern = r"\d+" corridor_pattern = r"\d+"
@ -255,20 +258,21 @@ class TugboatPlugin(BaseDataSourcePlugin):
pass pass
def _get_excel_obj(self): def _get_excel_obj(self):
""" Creation of an ExcelParser object to store site information. """Creation of an ExcelParser object to store site information.
The information is obtained based on a excel spec yaml file. The information is obtained based on a excel spec yaml file.
This spec contains row, column and sheet information of This spec contains row, column and sheet information of
the excel file from where site specific data can be extracted. the excel file from where site specific data can be extracted.
""" """
self.excel_obj = ExcelParser(self.excel_path, self.excel_spec) self.excel_obj = ExcelParser(self.excel_path, self.excel_spec)
def _extract_raw_data_from_excel(self): def _extract_raw_data_from_excel(self):
""" Extracts raw information from excel file based on excel spec""" """Extracts raw information from excel file based on excel spec"""
self.parsed_xl_data = self.excel_obj.get_data() self.parsed_xl_data = self.excel_obj.get_data()
def _get_network_name_from_vlan_name(self, vlan_name): def _get_network_name_from_vlan_name(self, vlan_name):
""" network names are ksn, oam, oob, overlay, storage, pxe """Network names are ksn, oam, oob, overlay, storage, pxe
This is a utility function to determine the vlan acceptable This is a utility function to determine the vlan acceptable
@ -282,6 +286,7 @@ class TugboatPlugin(BaseDataSourcePlugin):
vlan_name contains "oob" the network name is "oob" vlan_name contains "oob" the network name is "oob"
vlan_name contains "pxe" the network name is "pxe" vlan_name contains "pxe" the network name is "pxe"
""" """
network_names = [ network_names = [
"ksn|calico", "ksn|calico",
"storage", "storage",
@ -314,7 +319,7 @@ class TugboatPlugin(BaseDataSourcePlugin):
return "" return ""
def _get_formatted_server_list(self, server_list): def _get_formatted_server_list(self, server_list):
""" Format dns and ntp server list as comma separated string """ """Format dns and ntp server list as comma separated string"""
# dns/ntp server info from excel is of the format # dns/ntp server info from excel is of the format
# 'xxx.xxx.xxx.xxx, (aaa.bbb.ccc.com)' # 'xxx.xxx.xxx.xxx, (aaa.bbb.ccc.com)'
@ -327,10 +332,8 @@ class TugboatPlugin(BaseDataSourcePlugin):
return formatted_server_list return formatted_server_list
def _get_rack(self, host): def _get_rack(self, host):
""" """Get rack id from the rack string extracted from xl"""
Get rack id from the rack string extracted
from xl
"""
rack_pattern = r"\w.*(r\d+)\w.*" rack_pattern = r"\w.*(r\d+)\w.*"
rack = re.findall(rack_pattern, host)[0] rack = re.findall(rack_pattern, host)[0]
if not self.region: if not self.region:
@ -338,7 +341,8 @@ class TugboatPlugin(BaseDataSourcePlugin):
return rack return rack
def _get_rackwise_hosts(self): def _get_rackwise_hosts(self):
""" Mapping hosts with rack ids """ """Mapping hosts with rack ids"""
rackwise_hosts = {} rackwise_hosts = {}
hostnames = self.parsed_xl_data["ipmi_data"][1] hostnames = self.parsed_xl_data["ipmi_data"][1]
racks = self._get_rack_data() racks = self._get_rack_data()
@ -352,7 +356,8 @@ class TugboatPlugin(BaseDataSourcePlugin):
return rackwise_hosts return rackwise_hosts
def _get_rack_data(self): def _get_rack_data(self):
""" Format rack name """ """Format rack name"""
LOG.info("Getting rack data") LOG.info("Getting rack data")
racks = {} racks = {}
hostnames = self.parsed_xl_data["ipmi_data"][1] hostnames = self.parsed_xl_data["ipmi_data"][1]

4
spyglass/examples/excel_spec.yaml
View File

@ -29,7 +29,7 @@ specs:
private_ip_sheet: 'Site-Information' private_ip_sheet: 'Site-Information'
net_type_col: 1 net_type_col: 1
vlan_col: 2 vlan_col: 2
vlan_start_row: 19 vlan_start_row: 19
vlan_end_row: 30 vlan_end_row: 30
net_start_row: 33 net_start_row: 33
net_end_row: 40 net_end_row: 40
@ -56,7 +56,7 @@ specs:
domain_col: 2 domain_col: 2
location_sheet: 'Site-Information' location_sheet: 'Site-Information'
column: 2 column: 2
corridor_row: 59 corridor_row: 59
site_name_row: 58 site_name_row: 58
state_name_row: 60 state_name_row: 60
country_name_row: 61 country_name_row: 61

56
spyglass/parser/engine.py
View File

@ -4,7 +4,7 @@
# you may not use this file except in compliance with the License. # you may not use this file except in compliance with the License.
# You may obtain a copy of the License at # You may obtain a copy of the License at
# #
# http://www.apache.org/licenses/LICENSE-2.0 # http://www.apache.org/licenses/LICENSE-2.0
# #
# Unless required by applicable law or agreed to in writing, software # Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, # distributed under the License is distributed on an "AS IS" BASIS,
@ -26,7 +26,7 @@ import yaml
LOG = logging.getLogger(__name__) LOG = logging.getLogger(__name__)
class ProcessDataSource: class ProcessDataSource(object):
def __init__(self, sitetype): def __init__(self, sitetype):
# Initialize intermediary and save site type # Initialize intermediary and save site type
self._initialize_intermediary() self._initialize_intermediary()
@ -53,11 +53,12 @@ class ProcessDataSource:
self.network_subnets = None self.network_subnets = None
def _get_network_subnets(self): def _get_network_subnets(self):
""" Extract subnet information for networks. """Extract subnet information for networks.
In some networks, there are multiple subnets, in that case In some networks, there are multiple subnets, in that case
we assign only the first subnet """ we assign only the first subnet
"""
LOG.info("Extracting network subnets") LOG.info("Extracting network subnets")
network_subnets = {} network_subnets = {}
for net_type in self.data["network"]["vlan_network_data"]: for net_type in self.data["network"]["vlan_network_data"]:
@ -89,7 +90,8 @@ class ProcessDataSource:
) )
def _get_genesis_node_ip(self): def _get_genesis_node_ip(self):
""" Returns the genesis node ip """ """Returns the genesis node ip"""
ip = "0.0.0.0" ip = "0.0.0.0"
LOG.info("Getting Genesis Node IP") LOG.info("Getting Genesis Node IP")
if not self.genesis_node: if not self.genesis_node:
@ -100,13 +102,13 @@ class ProcessDataSource:
return ip return ip
def _validate_intermediary_data(self, data): def _validate_intermediary_data(self, data):
""" Validates the intermediary data before generating manifests. """Validates the intermediary data before generating manifests.
It checks wether the data types and data format are as expected. It checks wether the data types and data format are as expected.
The method validates this with regex pattern defined for each The method validates this with regex pattern defined for each
data type. data type.
""" """
LOG.info("Validating Intermediary data") LOG.info("Validating Intermediary data")
temp_data = {} temp_data = {}
# Peforming a deep copy # Peforming a deep copy
@ -147,14 +149,14 @@ class ProcessDataSource:
LOG.info("Data validation Passed!") LOG.info("Data validation Passed!")
def _apply_design_rules(self): def _apply_design_rules(self):
""" Applies design rules from rules.yaml """Applies design rules from rules.yaml
These rules are used to determine ip address allocation ranges, These rules are used to determine ip address allocation ranges,
host profile interfaces and also to create hardware profile host profile interfaces and also to create hardware profile
information. The method calls corresponding rule hander function information. The method calls corresponding rule hander function
based on rule name and applies them to appropriate data objects. based on rule name and applies them to appropriate data objects.
""" """
LOG.info("Apply design rules") LOG.info("Apply design rules")
rules_dir = pkg_resources.resource_filename("spyglass", "config/") rules_dir = pkg_resources.resource_filename("spyglass", "config/")
rules_file = rules_dir + "rules.yaml" rules_file = rules_dir + "rules.yaml"
@ -178,13 +180,14 @@ class ProcessDataSource:
pass pass
def _apply_rule_hardware_profile(self, rule_data): def _apply_rule_hardware_profile(self, rule_data):
""" Apply rules to define host type from hardware profile info. """Apply rules to define host type from hardware profile info.
Host profile will define host types as "controller, compute or Host profile will define host types as "controller, compute or
genesis". The rule_data has pre-defined information to define genesis". The rule_data has pre-defined information to define
compute or controller based on host_profile. For defining 'genesis' compute or controller based on host_profile. For defining 'genesis'
the first controller host is defined as genesis.""" the first controller host is defined as genesis.
"""
is_genesis = False is_genesis = False
hardware_profile = rule_data[self.data["site_info"]["sitetype"]] hardware_profile = rule_data[self.data["site_info"]["sitetype"]]
# Getting individual racks. The racks are sorted to ensure that the # Getting individual racks. The racks are sorted to ensure that the
@ -207,8 +210,10 @@ class ProcessDataSource:
host_info["type"] = "compute" host_info["type"] = "compute"
def _apply_rule_ip_alloc_offset(self, rule_data): def _apply_rule_ip_alloc_offset(self, rule_data):
""" Apply offset rules to update baremetal host ip's and vlan network """Apply offset rules to update baremetal host
data """
ip's and vlan network
"""
# Get network subnets # Get network subnets
self.network_subnets = self._get_network_subnets() self.network_subnets = self._get_network_subnets()
@ -217,12 +222,12 @@ class ProcessDataSource:
self._update_baremetal_host_ip_data(rule_data) self._update_baremetal_host_ip_data(rule_data)
def _update_baremetal_host_ip_data(self, rule_data): def _update_baremetal_host_ip_data(self, rule_data):
""" Update baremetal host ip's for applicable networks. """Update baremetal host ip's for applicable networks.
The applicable networks are oob, oam, ksn, storage and overlay. The applicable networks are oob, oam, ksn, storage and overlay.
These IPs are assigned based on network subnets ranges. These IPs are assigned based on network subnets ranges.
If a particular ip exists it is overridden.""" If a particular ip exists it is overridden.
"""
# Ger defult ip offset # Ger defult ip offset
default_ip_offset = rule_data["default"] default_ip_offset = rule_data["default"]
@ -245,12 +250,12 @@ class ProcessDataSource:
) )
def _update_vlan_net_data(self, rule_data): def _update_vlan_net_data(self, rule_data):
""" Offset allocation rules to determine ip address range(s) """Offset allocation rules to determine ip address range(s)
This rule is applied to incoming network data to determine This rule is applied to incoming network data to determine
network address, gateway ip and other address ranges network address, gateway ip and other address ranges
""" """
LOG.info("Apply network design rules") LOG.info("Apply network design rules")
# Collect Rules # Collect Rules
@ -348,10 +353,11 @@ class ProcessDataSource:
) )
def load_extracted_data_from_data_source(self, extracted_data): def load_extracted_data_from_data_source(self, extracted_data):
""" """Function called from spyglass.py to pass extracted data
Function called from spyglass.py to pass extracted data
from input data source from input data source
""" """
# TBR(pg710r): for internal testing # TBR(pg710r): for internal testing
""" """
raw_data = self._read_file('extracted_data.yaml') raw_data = self._read_file('extracted_data.yaml')
@ -376,7 +382,8 @@ class ProcessDataSource:
self.data["region_name"] = self.region_name self.data["region_name"] = self.region_name
def dump_intermediary_file(self, intermediary_dir): def dump_intermediary_file(self, intermediary_dir):
""" Writing intermediary yaml """ """Writing intermediary yaml"""
LOG.info("Writing intermediary yaml") LOG.info("Writing intermediary yaml")
intermediary_file = "{}_intermediary.yaml".format( intermediary_file = "{}_intermediary.yaml".format(
self.data["region_name"] self.data["region_name"]
@ -393,7 +400,8 @@ class ProcessDataSource:
f.close() f.close()
def generate_intermediary_yaml(self): def generate_intermediary_yaml(self):
""" Generating intermediary yaml """ """Generating intermediary yaml"""
LOG.info("Start: Generate Intermediary") LOG.info("Start: Generate Intermediary")
self._apply_design_rules() self._apply_design_rules()
self._get_genesis_node_details() self._get_genesis_node_details()

4
spyglass/site_processors/base.py
View File

@ -4,7 +4,7 @@
# you may not use this file except in compliance with the License. # you may not use this file except in compliance with the License.
# You may obtain a copy of the License at # You may obtain a copy of the License at
# #
# http://www.apache.org/licenses/LICENSE-2.0 # http://www.apache.org/licenses/LICENSE-2.0
# #
# Unless required by applicable law or agreed to in writing, software # Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, # distributed under the License is distributed on an "AS IS" BASIS,
@ -13,7 +13,7 @@
# limitations under the License. # limitations under the License.
class BaseProcessor: class BaseProcessor(object):
def __init__(self, file_name): def __init__(self, file_name):
pass pass

10
spyglass/site_processors/site_processor.py
View File

@ -4,7 +4,7 @@
# you may not use this file except in compliance with the License. # you may not use this file except in compliance with the License.
# You may obtain a copy of the License at # You may obtain a copy of the License at
# #
# http://www.apache.org/licenses/LICENSE-2.0 # http://www.apache.org/licenses/LICENSE-2.0
# #
# Unless required by applicable law or agreed to in writing, software # Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, # distributed under the License is distributed on an "AS IS" BASIS,
@ -12,11 +12,11 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
import logging
import os
from jinja2 import Environment from jinja2 import Environment
from jinja2 import FileSystemLoader from jinja2 import FileSystemLoader
from .base import BaseProcessor import logging
import os
from spyglass.site_processors.base import BaseProcessor
LOG = logging.getLogger(__name__) LOG = logging.getLogger(__name__)
@ -27,7 +27,7 @@ class SiteProcessor(BaseProcessor):
self.manifest_dir = manifest_dir self.manifest_dir = manifest_dir
def render_template(self, template_dir): def render_template(self, template_dir):
""" The method renders network config yaml from j2 templates. """The method renders network config yaml from j2 templates.
Network configs common to all racks (i.e oam, overlay, storage, Network configs common to all racks (i.e oam, overlay, storage,

6
spyglass/spyglass.py
View File

@ -1,13 +1,13 @@
# Copyright 2018 AT&T Intellectual Property. All other rights reserved. # Copyright 2018 AT&T Intellectual Property. All other rights reserved.
# #
# Licensed under the Apache License, Version 2.0 (the 'License'); # Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License. # you may not use this file except in compliance with the License.
# You may obtain a copy of the License at # You may obtain a copy of the License at
# #
# http://www.apache.org/licenses/LICENSE-2.0 # http://www.apache.org/licenses/LICENSE-2.0
# #
# Unless required by applicable law or agreed to in writing, software # Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an 'AS IS' BASIS, # distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.

8
spyglass/utils/utils.py
View File

@ -1,13 +1,13 @@
# Copyright 2018 AT&T Intellectual Property. All other rights reserved. # Copyright 2018 AT&T Intellectual Property. All other rights reserved.
# #
# Licensed under the Apache License, Version 2.0 (the 'License'); # Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License. # you may not use this file except in compliance with the License.
# You may obtain a copy of the License at # You may obtain a copy of the License at
# #
# http://www.apache.org/licenses/LICENSE-2.0 # http://www.apache.org/licenses/LICENSE-2.0
# #
# Unless required by applicable law or agreed to in writing, software # Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an 'AS IS' BASIS, # distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
@ -15,7 +15,7 @@
# Merge two dictionaries # Merge two dictionaries
def dict_merge(dictA, dictB, path=None): def dict_merge(dictA, dictB, path=None):
""" Recursively Merge dictionary dictB into dictA """Recursively Merge dictionary dictB into dictA
DictA represents the data extracted by a plugin and DictB DictA represents the data extracted by a plugin and DictB

8
test-requirements.txt Normal file
View File

@ -0,0 +1,8 @@
# Formatting
yapf==0.20.0
# Linting
hacking>=1.1.0,<1.2.0 # Apache-2.0
# Security
bandit>=1.5.0

9
tools/gate/whitespace-linter.sh Executable file
View File

@ -0,0 +1,9 @@
#!/usr/bin/env bash
set -x
RES=$(git grep -E -l " +$")
if [[ -n $RES ]]; then
exit 1
fi

32
tox.ini
View File

@ -1,30 +1,40 @@
[tox] [tox]
envlist = py35, py36, pep8, docs envlist = pep8, docs
minversion = 2.3.1
skipsdist = True skipsdist = True
[testenv] [testenv]
usedevelop = True
setenv =
VIRTUAL_ENV={envdir}
LANGUAGE=en_US
LC_ALL=en_US.utf-8
deps = deps =
-r{toxinidir}/requirements.txt -r{toxinidir}/requirements.txt
basepython=python3 -r{toxinidir}/test-requirements.txt
passenv = http_proxy https_proxy HTTP_PROXY HTTPS_PROXY no_proxy NO_PROXY PBR_VERSION
whitelist_externals = whitelist_externals =
find find
commands = commands =
find . -type f -name "*.pyc" -delete find . -type f -name "*.pyc" -delete
pytest \ {toxinidir}/tools/gate/run-unit-tests.sh '{posargs}'
{posargs}
[testenv:fmt] [testenv:fmt]
deps = yapf basepython = python3
deps =
-r{toxinidir}/test-requirements.txt
commands = commands =
yapf --style=pep8 -ir {toxinidir}/spyglass {toxinidir}/tests yapf -ir {toxinidir}/spyglass {toxinidir}/tests
[testenv:pep8] [testenv:pep8]
deps = basepython = python3
yapf deps =
flake8 -r{toxinidir}/test-requirements.txt
commands = commands =
#yapf --style=.style.yapf -rd {toxinidir}/spyglass bash -c "{toxinidir}/tools/gate/whitespace-linter.sh"
flake8 {toxinidir}/spyglass flake8 {toxinidir}/spyglass
whitelist_externals =
bash
[testenv:bandit] [testenv:bandit]
deps = deps =