# Copyright 2018 AT&T Intellectual Property. All other rights reserved. # # Licensed under the Apache License, Version 2.0 (the 'License'); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an 'AS IS' BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import logging import pprint import re import requests import formation_client import urllib3 from spyglass.data_extractor.base import BaseDataSourcePlugin from spyglass.data_extractor.custom_exceptions import ( ApiClientError, ConnectionError, MissingAttributeError, TokenGenerationError, ) urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) LOG = logging.getLogger(__name__) class FormationPlugin(BaseDataSourcePlugin): def __init__(self, region): # Save site name is valid try: assert region is not None super().__init__(region) except AssertionError: LOG.error("Site: None! Spyglass exited!") LOG.info("Check spyglass --help for details") exit() self.source_type = "rest" self.source_name = "formation" # Configuration parameters self.formation_api_url = None self.user = None self.password = None self.token = None # Formation objects self.client_config = None self.formation_api_client = None # Site related data self.region_zone_map = {} self.site_name_id_mapping = {} self.zone_name_id_mapping = {} self.region_name_id_mapping = {} self.rack_name_id_mapping = {} self.device_name_id_mapping = {} LOG.info("Initiated data extractor plugin:{}".format(self.source_name)) def set_config_opts(self, conf): """ Sets the config params passed by CLI""" LOG.info("Plugin params passed:\n{}".format(pprint.pformat(conf))) self._validate_config_options(conf) self.formation_api_url = conf["url"] self.user = conf["user"] self.password = conf["password"] self.token = conf.get("token", None) self._get_formation_client() self._update_site_and_zone(self.region) def get_plugin_conf(self, kwargs): """ Validates the plugin param and return if success""" try: assert ( kwargs["formation_url"] ) is not None, "formation_url is Not Specified" url = kwargs["formation_url"] assert ( kwargs["formation_user"] ) is not None, "formation_user is Not Specified" user = kwargs["formation_user"] assert ( kwargs["formation_password"] ) is not None, "formation_password is Not Specified" password = kwargs["formation_password"] except AssertionError: LOG.error("Insufficient plugin parameter! Spyglass exited!") raise exit() plugin_conf = {"url": url, "user": user, "password": password} return plugin_conf def _validate_config_options(self, conf): """Validate the CLI params passed The method checks for missing parameters and terminates Spyglass execution if found so. """ missing_params = [] for key in conf.keys(): if conf[key] is None: missing_params.append(key) if len(missing_params) != 0: LOG.error("Missing Plugin Params{}:".format(missing_params)) exit() # Implement helper classes def _generate_token(self): """Generate token for Formation Formation API does not provide separate resource to generate token. This is a workaround to call directly Formation API to get token instead of using Formation client. """ # Create formation client config object self.client_config = formation_client.Configuration() self.client_config.host = self.formation_api_url self.client_config.username = self.user self.client_config.password = self.password self.client_config.verify_ssl = False # Assumes token is never expired in the execution of this tool if self.token: return self.token url = self.formation_api_url + "/zones" try: token_response = requests.get( url, auth=(self.user, self.password), verify=self.client_config.verify_ssl, ) except requests.exceptions.ConnectionError: raise ConnectionError("Incorrect URL: {}".format(url)) if token_response.status_code == 200: self.token = token_response.json().get("X-Subject-Token", None) else: raise TokenGenerationError( "Unable to generate token because {}".format( token_response.reason ) ) return self.token def _get_formation_client(self): """Create formation client object Formation uses X-Auth-Token for authentication and should be in format "user|token". Generate the token and add it formation config object. """ token = self._generate_token() self.client_config.api_key = {"X-Auth-Token": self.user + "|" + token} self.formation_api_client = formation_client.ApiClient( self.client_config ) def _update_site_and_zone(self, region): """Get Zone name and Site name from region""" zone = self._get_zone_by_region_name(region) site = self._get_site_by_zone_name(zone) # zone = region[:-1] # site = zone[:-1] self.region_zone_map[region] = {} self.region_zone_map[region]["zone"] = zone self.region_zone_map[region]["site"] = site def _get_zone_by_region_name(self, region_name): zone_api = formation_client.ZonesApi(self.formation_api_client) zones = zone_api.zones_get() # Walk through each zone and get regions # Return when region name matches for zone in zones: self.zone_name_id_mapping[zone.name] = zone.id zone_regions = self.get_regions(zone.name) if region_name in zone_regions: return zone.name return None def _get_site_by_zone_name(self, zone_name): site_api = formation_client.SitesApi(self.formation_api_client) sites = site_api.sites_get() # Walk through each site and get zones # Return when site name matches for site in sites: self.site_name_id_mapping[site.name] = site.id site_zones = self.get_zones(site.name) if zone_name in site_zones: return site.name return None def _get_site_id_by_name(self, site_name): if site_name in self.site_name_id_mapping: return self.site_name_id_mapping.get(site_name) site_api = formation_client.SitesApi(self.formation_api_client) sites = site_api.sites_get() for site in sites: self.site_name_id_mapping[site.name] = site.id if site.name == site_name: return site.id def _get_zone_id_by_name(self, zone_name): if zone_name in self.zone_name_id_mapping: return self.zone_name_id_mapping.get(zone_name) zone_api = formation_client.ZonesApi(self.formation_api_client) zones = zone_api.zones_get() for zone in zones: if zone.name == zone_name: self.zone_name_id_mapping[zone.name] = zone.id return zone.id def _get_region_id_by_name(self, region_name): if region_name in self.region_name_id_mapping: return self.region_name_id_mapping.get(region_name) for zone in self.zone_name_id_mapping: self.get_regions(zone) return self.region_name_id_mapping.get(region_name, None) def _get_rack_id_by_name(self, rack_name): if rack_name in self.rack_name_id_mapping: return self.rack_name_id_mapping.get(rack_name) for zone in self.zone_name_id_mapping: self.get_racks(zone) return self.rack_name_id_mapping.get(rack_name, None) def _get_device_id_by_name(self, device_name): if device_name in self.device_name_id_mapping: return self.device_name_id_mapping.get(device_name) self.get_hosts(self.zone) return self.device_name_id_mapping.get(device_name, None) def _get_racks(self, zone, rack_type="compute"): zone_id = self._get_zone_id_by_name(zone) rack_api = formation_client.RacksApi(self.formation_api_client) racks = rack_api.zones_zone_id_racks_get(zone_id) racks_list = [] for rack in racks: rack_name = rack.name self.rack_name_id_mapping[rack_name] = rack.id if rack.rack_type.name == rack_type: racks_list.append(rack_name) return racks_list # Functions that will be used internally within this plugin def get_zones(self, site=None): zone_api = formation_client.ZonesApi(self.formation_api_client) if site is None: zones = zone_api.zones_get() else: site_id = self._get_site_id_by_name(site) zones = zone_api.sites_site_id_zones_get(site_id) zones_list = [] for zone in zones: zone_name = zone.name self.zone_name_id_mapping[zone_name] = zone.id zones_list.append(zone_name) return zones_list def get_regions(self, zone): zone_id = self._get_zone_id_by_name(zone) region_api = formation_client.RegionApi(self.formation_api_client) regions = region_api.zones_zone_id_regions_get(zone_id) regions_list = [] for region in regions: region_name = region.name self.region_name_id_mapping[region_name] = region.id regions_list.append(region_name) return regions_list # Implement Abstract functions def get_racks(self, region): zone = self.region_zone_map[region]["zone"] return self._get_racks(zone, rack_type="compute") def get_hosts(self, region, rack=None): zone = self.region_zone_map[region]["zone"] zone_id = self._get_zone_id_by_name(zone) device_api = formation_client.DevicesApi(self.formation_api_client) control_hosts = device_api.zones_zone_id_control_nodes_get(zone_id) compute_hosts = device_api.zones_zone_id_devices_get( zone_id, type="KVM" ) hosts_list = [] for host in control_hosts: self.device_name_id_mapping[host.aic_standard_name] = host.id hosts_list.append( { "name": host.aic_standard_name, "type": "controller", "rack_name": host.rack_name, "host_profile": host.host_profile_name, } ) for host in compute_hosts: self.device_name_id_mapping[host.aic_standard_name] = host.id hosts_list.append( { "name": host.aic_standard_name, "type": "compute", "rack_name": host.rack_name, "host_profile": host.host_profile_name, } ) """ for host in itertools.chain(control_hosts, compute_hosts): self.device_name_id_mapping[host.aic_standard_name] = host.id hosts_list.append({ 'name': host.aic_standard_name, 'type': host.categories[0], 'rack_name': host.rack_name, 'host_profile': host.host_profile_name }) """ return hosts_list def get_networks(self, region): zone = self.region_zone_map[region]["zone"] zone_id = self._get_zone_id_by_name(zone) region_id = self._get_region_id_by_name(region) vlan_api = formation_client.VlansApi(self.formation_api_client) vlans = vlan_api.zones_zone_id_regions_region_id_vlans_get( zone_id, region_id ) # Case when vlans list is empty from # zones_zone_id_regions_region_id_vlans_get if len(vlans) == 0: # get device-id from the first host and get the network details hosts = self.get_hosts(self.region) host = hosts[0]["name"] device_id = self._get_device_id_by_name(host) vlans = vlan_api.zones_zone_id_devices_device_id_vlans_get( zone_id, device_id ) LOG.debug("Extracted region network information\n{}".format(vlans)) vlans_list = [] for vlan_ in vlans: if len(vlan_.vlan.ipv4) != 0: tmp_vlan = {} tmp_vlan["name"] = self._get_network_name_from_vlan_name( vlan_.vlan.name ) tmp_vlan["vlan"] = vlan_.vlan.vlan_id tmp_vlan["subnet"] = vlan_.vlan.subnet_range tmp_vlan["gateway"] = vlan_.ipv4_gateway tmp_vlan["subnet_level"] = vlan_.vlan.subnet_level vlans_list.append(tmp_vlan) return vlans_list def get_ips(self, region, host=None): zone = self.region_zone_map[region]["zone"] zone_id = self._get_zone_id_by_name(zone) if host: hosts = [host] else: hosts = [] hosts_dict = self.get_hosts(zone) for host in hosts_dict: hosts.append(host["name"]) vlan_api = formation_client.VlansApi(self.formation_api_client) ip_ = {} for host in hosts: device_id = self._get_device_id_by_name(host) vlans = vlan_api.zones_zone_id_devices_device_id_vlans_get( zone_id, device_id ) LOG.debug("Received VLAN Network Information\n{}".format(vlans)) ip_[host] = {} for vlan_ in vlans: # TODO(pg710r) We need to handle the case when incoming ipv4 # list is empty if len(vlan_.vlan.ipv4) != 0: name = self._get_network_name_from_vlan_name( vlan_.vlan.name ) ipv4 = vlan_.vlan.ipv4[0].ip LOG.debug( "vlan:{},name:{},ip:{},vlan_name:{}".format( vlan_.vlan.vlan_id, name, ipv4, vlan_.vlan.name ) ) # TODD(pg710r) This code needs to extended to support ipv4 # and ipv6 # ip_[host][name] = {'ipv4': ipv4} ip_[host][name] = ipv4 return ip_ def _get_network_name_from_vlan_name(self, vlan_name): """ network names are ksn, oam, oob, overlay, storage, pxe The following mapping rules apply: vlan_name contains "ksn" the network name is "calico" vlan_name contains "storage" the network name is "storage" vlan_name contains "server" the network name is "oam" vlan_name contains "ovs" the network name is "overlay" vlan_name contains "ILO" the network name is "oob" """ network_names = { "ksn": "calico", "storage": "storage", "server": "oam", "ovs": "overlay", "ILO": "oob", "pxe": "pxe", } for name in network_names: # Make a pattern that would ignore case. # if name is 'ksn' pattern name is '(?i)(ksn)' name_pattern = "(?i)({})".format(name) if re.search(name_pattern, vlan_name): return network_names[name] # Return empty string is vlan_name is not matched with network_names return "" def get_dns_servers(self, region): try: zone = self.region_zone_map[region]["zone"] zone_id = self._get_zone_id_by_name(zone) zone_api = formation_client.ZonesApi(self.formation_api_client) zone_ = zone_api.zones_zone_id_get(zone_id) except formation_client.rest.ApiException as e: raise ApiClientError(e.msg) if not zone_.ipv4_dns: LOG.warn("No dns server") return [] dns_list = [] for dns in zone_.ipv4_dns: dns_list.append(dns.ip) return dns_list def get_ntp_servers(self, region): return [] def get_ldap_information(self, region): return {} def get_location_information(self, region): """ get location information for a zone and return """ site = self.region_zone_map[region]["site"] site_id = self._get_site_id_by_name(site) site_api = formation_client.SitesApi(self.formation_api_client) site_info = site_api.sites_site_id_get(site_id) try: return { # 'corridor': site_info.corridor, "name": site_info.city, "state": site_info.state, "country": site_info.country, "physical_location_id": site_info.clli, } except AttributeError as e: raise MissingAttributeError( "Missing {} information in {}".format(e, site_info.city) ) def get_domain_name(self, region): try: zone = self.region_zone_map[region]["zone"] zone_id = self._get_zone_id_by_name(zone) zone_api = formation_client.ZonesApi(self.formation_api_client) zone_ = zone_api.zones_zone_id_get(zone_id) except formation_client.rest.ApiException as e: raise ApiClientError(e.msg) if not zone_.dns: LOG.warn("Got None while running get domain name") return None return zone_.dns