From efe24d8a5fefe12f115fb06b89c1fcef8b120f21 Mon Sep 17 00:00:00 2001 From: "Ian H. Pittwood" Date: Thu, 23 May 2019 14:15:43 -0500 Subject: [PATCH] Use data objects for document generation This is a follow-up change to [0] which further implements data objects to be used in the generation of documents in Jinja2. The following additions and changes are made: - Adds helper functions to data objects to filter hosts and networks for ease of use in Jinja2 templates - Adds SiteDocumentData factory function to convert intermediary yaml dictionaries into a SiteDocumentData object with all associated objects - Updates Jinja2 templates to use data objects - Cleans up overly complex looping in Jinja2 - Adds tests for new code in models.py [0] https://review.opendev.org/#/c/662092/ Change-Id: I66ebfeaf5d6ca76b6dee5a2285a74bad8b06b720 --- spyglass/data_extractor/custom_exceptions.py | 4 + spyglass/data_extractor/models.py | 112 ++++++++ .../baremetal/bootactions/promjoin.yaml.j2 | 26 -- .../templates/baremetal/nodes.yaml.j2 | 54 ++-- .../deployment/deployment-strategy.yaml.j2 | 29 +- .../networks/common_addresses.yaml.j2 | 95 +++---- .../networks/physical/networks.yaml.j2 | 216 +++++---------- .../templates/pki/pki-catalogue.yaml.j2 | 145 +++++----- .../examples/templates/profile/region.yaml.j2 | 4 +- .../templates/site-definition.yaml.j2 | 4 +- .../software/charts/kubernetes/etcd.yaml.j2 | 55 ++-- .../charts/kubernetes/etcd/etcd.yaml.j2 | 48 ++-- .../openstack-tenant-ceph/ceph-client.yaml.j2 | 2 +- .../config/common-software-config.yaml.j2 | 3 +- spyglass/site_processors/base.py | 19 -- spyglass/site_processors/site_processor.py | 19 +- tests/unit/data_extractor/test_models.py | 187 ++++++++++++- .../site_processors/test_site_processor.py | 247 +++++++++++------- 18 files changed, 711 insertions(+), 558 deletions(-) delete mode 100644 spyglass/examples/templates/baremetal/bootactions/promjoin.yaml.j2 diff --git a/spyglass/data_extractor/custom_exceptions.py b/spyglass/data_extractor/custom_exceptions.py index bc8fc07..5ec1304 100644 --- a/spyglass/data_extractor/custom_exceptions.py +++ b/spyglass/data_extractor/custom_exceptions.py @@ -59,3 +59,7 @@ class TokenGenerationError(BaseError): class FormationConnectionError(BaseError): pass + + +class InvalidIntermediary(BaseError): + pass diff --git a/spyglass/data_extractor/models.py b/spyglass/data_extractor/models.py index 6fdb774..ef122e3 100644 --- a/spyglass/data_extractor/models.py +++ b/spyglass/data_extractor/models.py @@ -12,9 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +from copy import deepcopy import ipaddress import logging +from spyglass.data_extractor.custom_exceptions import InvalidIntermediary + DATA_DEFAULT = "#CHANGE_ME" LOG = logging.getLogger(__name__) @@ -253,6 +256,19 @@ class Rack(object): return host return None + def get_host_by_type(self, host_type: str): + """Gets host(s) on rack by role + + :param host_type: Role of the host(s) to be retrieved + :return: list of hosts + :rtype: list + """ + matching_hosts = [] + for host in self.hosts: + if host.type == host_type: + matching_hosts.append(host) + return matching_hosts + class VLANNetworkData(object): """Model for single entry of VLAN Network Data""" @@ -594,3 +610,99 @@ class SiteDocumentData(object): if rack.name == name: return rack return None + + def get_baremetal_host_by_type(self, *args): + """Return baremetal host(s) with matching type + + :param args: type(s) of the baremetal host + :return: Host object(s) matching the specified host_type + :rtype: list + """ + host_list = [] + for rack in self.baremetal: + rack_hosts = [] + for arg in args: + rack_hosts.extend(rack.get_host_by_type(arg)) + host_list.extend(rack_hosts) + return host_list + + +def _validate_key_in_intermediary_dict(key: str, dictionary: dict): + if key not in dictionary: + raise InvalidIntermediary( + '%s is not defined in the given intermediary file.' % key) + + +def site_document_data_factory(intermediary_dict: dict) -> SiteDocumentData: + """Uses intermediary file data to create a SiteDocumentData object + + :param intermediary_dict: A loaded intermediary file dictionary + :return: all intermediary dictionary data returned as an object + """ + # Validate baremetal in intermediary + _validate_key_in_intermediary_dict('baremetal', intermediary_dict) + + # Pull out baremetal data into Rack and Host objects + rack_list = [] + for rack, hosts in intermediary_dict['baremetal'].items(): + host_list = [] + for host_name, host_data in hosts.items(): + host_ip_list = IPList(**host_data['ip']) + host_kwargs = { + 'rack_name': rack, + 'host_profile': host_data['host_profile'], + 'type': host_data['type'], + 'ip': host_ip_list + } + new_host = Host(host_name, **host_kwargs) + host_list.append(new_host) + new_rack = Rack(rack, host_list) + rack_list.append(new_rack) + + # Validate network in intermediary + _validate_key_in_intermediary_dict('network', intermediary_dict) + # Validate vlan_network_data in intermediary + _validate_key_in_intermediary_dict( + 'vlan_network_data', intermediary_dict['network']) + # Validate bgp in intermediary + _validate_key_in_intermediary_dict('bgp', intermediary_dict['network']) + + # Pull out network data into Network object + vlan_data_list = [] + for network_type, network_data in \ + intermediary_dict['network']['vlan_network_data'].items(): + vlan_data_list.append(VLANNetworkData(network_type, **network_data)) + network = Network(vlan_data_list, bgp=intermediary_dict['network']['bgp']) + + # Validate site_info in intermediary + _validate_key_in_intermediary_dict('site_info', intermediary_dict) + # Validate dns in intermediary + _validate_key_in_intermediary_dict('dns', intermediary_dict['site_info']) + # Validate ntp in intermediary + _validate_key_in_intermediary_dict('ntp', intermediary_dict['site_info']) + # Validate region_name in intermediary + _validate_key_in_intermediary_dict('region_name', intermediary_dict) + + # Pull out site_info into a SiteInfo object + dns_server_list = ServerList( + intermediary_dict['site_info']['dns']['servers'].split(',')) + ntp_server_list = ServerList( + intermediary_dict['site_info']['ntp']['servers'].split(',')) + site_info_dict = deepcopy(intermediary_dict['site_info']) + site_info_dict.pop('dns') + site_info_dict.pop('ntp') + site_info_dict['dns'] = dns_server_list + site_info_dict['ntp'] = ntp_server_list + site_info_dict['region_name'] = intermediary_dict['region_name'] + site_info = SiteInfo(**site_info_dict) + + # Validate storage in intermediary + _validate_key_in_intermediary_dict('storage', intermediary_dict) + + # Create and return SiteDocumentData object + site_document_data = SiteDocumentData( + site_info=site_info, + network=network, + baremetal=rack_list, + storage=intermediary_dict['storage']) + return site_document_data diff --git a/spyglass/examples/templates/baremetal/bootactions/promjoin.yaml.j2 b/spyglass/examples/templates/baremetal/bootactions/promjoin.yaml.j2 deleted file mode 100644 index f7c37f2..0000000 --- a/spyglass/examples/templates/baremetal/bootactions/promjoin.yaml.j2 +++ /dev/null @@ -1,26 +0,0 @@ ---- -schema: 'drydock/BootAction/v1' -metadata: - schema: 'metadata/Document/v1' - name: promjoin - storagePolicy: 'cleartext' - layeringDefinition: - abstract: false - layer: site - labels: - application: 'drydock' -data: - signaling: false - assets: - - path: /opt/promjoin.sh - type: file - permissions: '555' -{% raw %} - location: promenade+http://promenade-api.ucp.svc.cluster.local/api/v1.0/join-scripts?design_ref={{ action.design_ref | urlencode }}&hostname={{ node.hostname }}&ip={{ node.network.calico.ip }}{% endif %}{% for k, v in node.labels.items() %}&labels.dynamic={{ k }}={{ v }}{% endfor %} - -{% endraw %} - location_pipeline: - - template - data_pipeline: - - utf8_decode -... diff --git a/spyglass/examples/templates/baremetal/nodes.yaml.j2 b/spyglass/examples/templates/baremetal/nodes.yaml.j2 index f730378..af15dcc 100644 --- a/spyglass/examples/templates/baremetal/nodes.yaml.j2 +++ b/spyglass/examples/templates/baremetal/nodes.yaml.j2 @@ -1,50 +1,46 @@ -{% set control_count = [1] %} -{% for rack in data['baremetal'].keys() %} -{% for host in data['baremetal'][rack].keys()%} -{% if data['baremetal'][rack][host]['type'] != 'genesis' %} +{% for rack in data.baremetal %} +{% for host in rack.hosts %} +{% if host.type != 'genesis' %} --- schema: 'drydock/BaremetalNode/v1' metadata: schema: 'metadata/Document/v1' - name: {{ host }} + name: {{ host.name }} layeringDefinition: abstract: false layer: site storagePolicy: cleartext data: - oob: - account: 'root' -{% if data['baremetal'][rack][host]['host_profile'] == 'cp' %} -{% if control_count.append(control_count.pop()+1) %} {% endif %} -{% if control_count[0] < 4 %} - host_profile: nc-{{data['baremetal'][rack][host]['host_profile']}}-primary -{% else %} - host_profile: nc-{{data['baremetal'][rack][host]['host_profile']}}-secondary -{% endif %} -{% else %} - host_profile: nc-{{data['baremetal'][rack][host]['host_profile']}} -{% endif %} + {% if host.host_profile == 'cp' %} + {% if loop.index - 1 < 4 %} + host_profile: nc-{{ host.host_profile }}-primary + {% else %} + host_profile: nc-{{ host.host_profile }}-secondary + {% endif %} + {% else %} + host_profile: nc-{{ host.host_profile }} + {% endif %} addressing: - network: oob - address: {{ data['baremetal'][rack][host]['ip']['oob'] }} + address: {{ host.ip.oob }} - network: oam - address: {{ data['baremetal'][rack][host]['ip']['oam'] }} + address: {{ host.ip.oam }} - network: pxe - address: {{ data['baremetal'][rack][host]['ip']['pxe'] }} + address: {{ host.ip.pxe }} - network: storage - address: {{ data['baremetal'][rack][host]['ip']['storage'] }} + address: {{ host.ip.storage }} - network: calico - address: {{ data['baremetal'][rack][host]['ip']['calico'] }} + address: {{ host.ip.calico }} - network: overlay - address: {{ data['baremetal'][rack][host]['ip']['overlay'] }} + address: {{ host.ip.overlay }} metadata: - rack: RACK{{rack[-2:] }} + rack: {{ rack.name }} tags: -{% if data['baremetal'][rack][host]['type'] == 'compute' %} - - 'workers' -{% else %} - - 'masters' -{% endif %} + {% if host.type == 'compute' %} + - 'workers' + {% else %} + - 'masters' + {% endif %} ... {% endif %} {%endfor%} diff --git a/spyglass/examples/templates/deployment/deployment-strategy.yaml.j2 b/spyglass/examples/templates/deployment/deployment-strategy.yaml.j2 index 5fb1511..07b0323 100644 --- a/spyglass/examples/templates/deployment/deployment-strategy.yaml.j2 +++ b/spyglass/examples/templates/deployment/deployment-strategy.yaml.j2 @@ -11,7 +11,6 @@ schema: shipyard/DeploymentStrategy/v1 metadata: schema: metadata/Document/v1 - replacement: true name: deployment-strategy layeringDefinition: abstract: false @@ -44,13 +43,13 @@ data: selectors: # NEWSITE-CHANGEME: The following should be a list of the computes in the site's first rack - node_names: -{% for rack in data['baremetal'].keys() %} -{% for host in data['baremetal'][rack].keys()%} -{% if rack == 'rack03' or rack == 'rack04' %} - - {{ host }} -{% endif %} -{% endfor %} -{% endfor %} + {% for rack in data.baremetal %} + {% for host in rack.hosts %} + {% if rack.name == 'rack03' or rack.name == 'rack04' %} + - {{ host.name }} + {% endif %} + {% endfor %} + {% endfor %} node_labels: [] node_tags: [] rack_names: [] @@ -61,13 +60,13 @@ data: selectors: # NEWSITE-CHANGEME: The following should be a list of the computes in the site's second rack - node_names: -{% for rack in data['baremetal'].keys() %} -{% for host in data['baremetal'][rack].keys()%} -{% if rack == 'rack05' or rack == 'rack06' %} - - {{ host }} -{% endif %} -{% endfor %} -{% endfor %} + {% for rack in data.baremetal %} + {% for host in rack.hosts %} + {% if rack.name == 'rack05' or rack.name == 'rack06' %} + - {{ host.name }} + {% endif %} + {% endfor %} + {% endfor %} node_labels: [] node_tags: [] rack_names: [] diff --git a/spyglass/examples/templates/networks/common_addresses.yaml.j2 b/spyglass/examples/templates/networks/common_addresses.yaml.j2 index e52ec20..cd0c64f 100644 --- a/spyglass/examples/templates/networks/common_addresses.yaml.j2 +++ b/spyglass/examples/templates/networks/common_addresses.yaml.j2 @@ -6,102 +6,69 @@ metadata: layeringDefinition: abstract: false layer: site + parentSelector: + name: common-addresses-global + actions: + - method: replace + path: .dns.upstream_servers + - method: merge + path: . storagePolicy: cleartext + replacement: true data: calico: - ip_autodetection_method: interface=bond1.{{ data['network']['vlan_network_data']['calico']['vlan']}} + ip_autodetection_method: interface=bond1.{{ data.network.get_vlan_data_by_name('calico').vlan }} etcd: service_ip: 10.96.232.136 ip_rule: - gateway: {{ data['network']['vlan_network_data']['calico']['gateway']}} + gateway: {{ data.network.get_vlan_data_by_name('calico').gateway }} overlap_cidr: 10.96.0.0/15 bgp: ipv4: - public_service_cidr: {{ data['network']['vlan_network_data']['ingress']['subnet'][0] }} - ingress_vip: {{ data['network']['bgp']['ingress_vip'] }} + public_service_cidr: {{ data.network.get_vlan_data_by_name('ingress').subnet[0] }} + ingress_vip: {{ data.network.bgp['ingress_vip'] }} peers: -{% for peer in data['network']['bgp']['peers'] %} + {% for peer in data.network.bgp['peers'] %} - {{ peer }} -{% endfor %} + {% endfor %} dns: - cluster_domain: cluster.local service_ip: 10.96.0.10 upstream_servers: -{% for server in (data['site_info']['dns']['servers']).split(',') %} + {% for server in data.site_info.dns.servers %} - {{ server }} -{% endfor %} - upstream_servers_joined: {{ data['site_info']['dns']['servers']}} - ingress_domain: {{ data['site_info']['domain']|lower }} + {% endfor %} + upstream_servers_joined: {{ data.site_info.dns.__str__() }} + node_domain: {{ data.site_info.domain | lower }} + ingress_domain: {{ data.site_info.domain | lower }} genesis: - hostname: {{ (data|get_role_wise_nodes)['genesis']['name'] }} -{% for rack in data['baremetal'] %} -{% for host in data['baremetal'][rack] %} -{% if data['baremetal'][rack][host]['type'] == 'genesis' %} - ip: {{ data['baremetal'][rack][host]['ip']['calico'] }} -{% endif %} -{% endfor %} -{% endfor %} + hostname: {{ data.get_baremetal_host_by_type('genesis')[0].name }} + ip: {{ data.get_baremetal_host_by_type('genesis')[0].ip.calico }} + bootstrap: - ip: {{ (data|get_role_wise_nodes)['genesis']['pxe'] }} + ip: {{ data.get_baremetal_host_by_type('genesis')[0].ip.pxe }} kubernetes: api_service_ip: 10.96.0.1 etcd_service_ip: 10.96.0.2 pod_cidr: 10.97.0.0/16 service_cidr: 10.96.0.0/16 - # misc k8s port settings - apiserver_port: 6443 - haproxy_port: 6553 - service_node_port_range: 30000-32767 - - # etcd port settings - etcd: - container_port: 2379 - haproxy_port: 2378 masters: -{% for host in (data|get_role_wise_nodes)['masters'] %} - - hostname: {{ host }} -{% endfor %} - # NEWSITE-CHANGEME: Environment proxy information. - # NOTE: Reference Airship sites do not deploy behind a proxy, so this proxy section - # should be commented out. - # However if you are in a lab that requires proxy, ensure that these proxy - # settings are correct and reachable in your environment; otherwise update - # them with the correct values for your environment. - proxy: - http: "" - https: "" - no_proxy: [] - - node_ports: - drydock_api: 30000 - maas_api: 30001 - maas_proxy: 31800 # hardcoded in MAAS - shipyard_api: 30003 - airflow_web: 30004 - ntp: - servers_joined: {{ data['site_info']['ntp']['servers'] }} - - ldap: - base_url: {{ (data['site_info']['ldap']['url']|string).split('//')[1] }} - url: {{ data['site_info']['ldap']['url'] }} - auth_path: DC=test,DC=test,DC=com?sAMAccountName?sub?memberof=CN={{ data['site_info']['ldap']['common_name'] }},OU=Application,OU=Groups,DC=test,DC=test,DC=com - common_name: {{ data['site_info']['ldap']['common_name'] }} - subdomain: {{ data['site_info']['ldap']['subdomain'] }} - domain: {{ (data['site_info']['ldap']['url']|string).split('.')[1] }} + {% for host in data.get_baremetal_host_by_type('controller') %} + - hostname: {{ host.name }} + {% endfor %} storage: ceph: - public_cidr: {{ data['network']['vlan_network_data']['storage']['subnet'] }} - cluster_cidr: {{ data['network']['vlan_network_data']['storage']['subnet'] }} + public_cidr: {{ data.network.get_vlan_data_by_name('storage').subnet[0] }} + cluster_cidr: {{ data.network.get_vlan_data_by_name('storage').subnet[0] }} neutron: - tunnel_device: 'bond1.{{ data['network']['vlan_network_data']['overlay']['vlan'] }}' - external_iface: 'bond1' + tunnel_device: "bond1.{{ data.network.get_vlan_data_by_name('overlay').vlan }}" + external_iface: "bond1" openvswitch: - external_iface: 'bond1' + external_iface: "bond1" ... diff --git a/spyglass/examples/templates/networks/physical/networks.yaml.j2 b/spyglass/examples/templates/networks/physical/networks.yaml.j2 index cb727f9..50b0a79 100644 --- a/spyglass/examples/templates/networks/physical/networks.yaml.j2 +++ b/spyglass/examples/templates/networks/physical/networks.yaml.j2 @@ -1,28 +1,4 @@ --- -schema: 'drydock/NetworkLink/v1' -metadata: - schema: 'metadata/Document/v1' - name: oob - layeringDefinition: - abstract: false - layer: site - storagePolicy: cleartext -data: - # MaaS doesnt own this network like it does the others, so the noconfig label - # is specified. - labels: - noconfig: enabled - bonding: - mode: disabled - mtu: 1500 - linkspeed: auto - trunking: - mode: disabled - default_network: oob - allowed_networks: - - oob -... ---- schema: 'drydock/Network/v1' metadata: schema: 'metadata/Document/v1' @@ -30,42 +6,33 @@ metadata: layeringDefinition: abstract: false layer: site + parentSelector: + network_role: oob + topology: cruiser + actions: + - method: merge + path: . storagePolicy: cleartext data: - cidr: {{ data['network']['vlan_network_data']['oob']['subnet'] }} + cidr: {{ data.network.get_vlan_data_by_name('oob').subnet[0] }} routes: - subnet: '0.0.0.0/0' - gateway: {{ data['network']['vlan_network_data']['oob']['gateway'] }} + gateway: {{ data.network.get_vlan_data_by_name('oob').gateway }} metric: 100 ranges: + - type: reserved + start: {{ data.network.get_vlan_data_by_name('oob').reserved_start }} + end: {{ data.network.get_vlan_data_by_name('oob').reserved_end }} - type: static - start: {{ data['network']['vlan_network_data']['oob']['static_start'] }} - end: {{ data['network']['vlan_network_data']['oob']['static_end'] }} + start: {{ data.network.get_vlan_data_by_name('oob').static_start }} + end: {{ data.network.get_vlan_data_by_name('oob').static_end }} + - type: dhcp + start: {{ data.network.get_vlan_data_by_name('oob').dhcp_start }} + end: {{ data.network.get_vlan_data_by_name('oob').dhcp_end }} ... --- schema: 'drydock/NetworkLink/v1' -metadata: - schema: 'metadata/Document/v1' - name: pxe - layeringDefinition: - abstract: false - layer: site - storagePolicy: cleartext -data: - bonding: - mode: disabled - mtu: 1500 - linkspeed: auto - trunking: - mode: disabled - default_network: pxe - allowed_networks: - - pxe -... - ---- -schema: 'drydock/Network/v1' metadata: schema: 'metadata/Document/v1' name: pxe @@ -80,56 +47,21 @@ metadata: path: . storagePolicy: cleartext data: - cidr: {{ data['network']['vlan_network_data']['pxe']['subnet'] }} + cidr: {{ data.network.get_vlan_data_by_name('pxe').subnet[0] }} routes: -{% for other_subnet in data['network']['vlan_network_data']['pxe']['routes'] %} - - subnet: {{ other_subnet }} - gateway: {{ data['network']['vlan_network_data']['pxe']['gateway'] }} + - subnet: '0.0.0.0/0' + gateway: {{ data.network.get_vlan_data_by_name('pxe').gateway }} metric: 100 -{% endfor %} ranges: - type: reserved - start: {{ data['network']['vlan_network_data']['pxe']['reserved_start'] }} - end: {{ data['network']['vlan_network_data']['pxe']['reserved_end'] }} + start: {{ data.network.get_vlan_data_by_name('pxe').reserved_start }} + end: {{ data.network.get_vlan_data_by_name('pxe').reserved_end }} - type: static - start: {{ data['network']['vlan_network_data']['pxe']['static_start'] }} - end: {{ data['network']['vlan_network_data']['pxe']['static_end'] }} + start: {{ data.network.get_vlan_data_by_name('pxe').static_start }} + end: {{ data.network.get_vlan_data_by_name('pxe').static_end }} - type: dhcp - start: {{ data['network']['vlan_network_data']['pxe']['dhcp_start'] }} - end: {{ data['network']['vlan_network_data']['pxe']['dhcp_end'] }} -... - ---- -schema: 'drydock/NetworkLink/v1' -metadata: - schema: 'metadata/Document/v1' - name: data - layeringDefinition: - abstract: false - layer: site - storagePolicy: cleartext -data: - bonding: - mode: 802.3ad - hash: layer3+4 - peer_rate: fast - mon_rate: 100 - up_delay: 1000 - down_delay: 3000 - # NEWSITE-CHANGEME: Ensure the network switches in the environment are - # configured for this MTU or greater. Even if switches are configured for or - # can support a slightly higher MTU, there is no need (and negliable benefit) - # to squeeze every last byte into the MTU (e.g., 9216 vs 9100). Leave MTU at - # 9100 for maximum compatibility. - mtu: 9100 - linkspeed: auto - trunking: - mode: 802.1q - allowed_networks: - - oam - - storage - - overlay - - calico + start: {{ data.network.get_vlan_data_by_name('pxe').dhcp_start }} + end: {{ data.network.get_vlan_data_by_name('pxe').dhcp_end }} ... --- @@ -148,27 +80,24 @@ metadata: path: . storagePolicy: cleartext data: - cidr: {{ data['network']['vlan_network_data']['oam']['subnet'] }} -{% set flag = [0] %} -{% for route in data['network']['vlan_network_data']['oam']['routes'] %} -{% if flag[0] == 0 %} + cidr: {{ data.network.get_vlan_data_by_name('oam').subnet[0] }} + {% if data.network.get_vlan_data_by_name('oam').routes %} routes: -{% endif %} -{% if flag.append(flag.pop() + 1) %} {% endif %} + {% for route in data.network.get_vlan_data_by_name('oam').routes %} - subnet: {{ route }} - gateway: {{ data['network']['vlan_network_data']['oam']['gateway'] }} + gateway: {{ data.network.get_vlan_data_by_name('oam').gateway }} metric: 100 -{% endfor %} -{% if flag[0] == 0 %} - routes:[] -{% endif %} + {% endfor %} + {% else %} + routes: [] + {% endif %} ranges: - type: reserved - start: {{ data['network']['vlan_network_data']['oam']['reserved_start'] }} - end: {{ data['network']['vlan_network_data']['oam']['reserved_end'] }} + start: {{ data.network.get_vlan_data_by_name('oam').reserved_start }} + end: {{ data.network.get_vlan_data_by_name('oam').reserved_end }} - type: static - start: {{ data['network']['vlan_network_data']['oam']['static_start'] }} - end: {{ data['network']['vlan_network_data']['oam']['static_end'] }} + start: {{ data.network.get_vlan_data_by_name('oam').static_start }} + end: {{ data.network.get_vlan_data_by_name('oam').static_end }} ... --- @@ -187,40 +116,14 @@ metadata: path: . storagePolicy: cleartext data: - cidr: {{ data['network']['vlan_network_data']['storage']['subnet'] }} + cidr: {{ data.network.get_vlan_data_by_name('storage').subnet[0] }} ranges: - type: reserved - start: {{ data['network']['vlan_network_data']['storage']['reserved_start'] }} - end: {{ data['network']['vlan_network_data']['storage']['reserved_end'] }} + start: {{ data.network.get_vlan_data_by_name('storage').reserved_start }} + end: {{ data.network.get_vlan_data_by_name('storage').reserved_end }} - type: static - start: {{ data['network']['vlan_network_data']['storage']['static_start'] }} - end: {{ data['network']['vlan_network_data']['storage']['static_end'] }} -... - ---- -schema: 'drydock/Network/v1' -metadata: - schema: 'metadata/Document/v1' - name: overlay - layeringDefinition: - abstract: false - layer: site - parentSelector: - network_role: os-overlay - topology: cruiser - actions: - - method: merge - path: . - storagePolicy: cleartext -data: - cidr: {{ data['network']['vlan_network_data']['overlay']['subnet'] }} - ranges: - - type: reserved - start: {{ data['network']['vlan_network_data']['overlay']['reserved_start'] }} - end: {{ data['network']['vlan_network_data']['overlay']['reserved_end'] }} - - type: static - start: {{ data['network']['vlan_network_data']['overlay']['static_start'] }} - end: {{ data['network']['vlan_network_data']['overlay']['static_end'] }} + start: {{ data.network.get_vlan_data_by_name('storage').static_start }} + end: {{ data.network.get_vlan_data_by_name('storage').static_end }} ... --- @@ -239,13 +142,38 @@ metadata: path: . storagePolicy: cleartext data: - cidr: {{ data['network']['vlan_network_data']['calico']['subnet'] }} + cidr: {{ data.network.get_vlan_data_by_name('calico').subnet[0] }} ranges: - type: reserved - start: {{ data['network']['vlan_network_data']['calico']['reserved_start'] }} - end: {{ data['network']['vlan_network_data']['calico']['reserved_end'] }} + start: {{ data.network.get_vlan_data_by_name('calico').reserved_start }} + end: {{ data.network.get_vlan_data_by_name('calico').reserved_end }} - type: static - start: {{ data['network']['vlan_network_data']['calico']['static_start'] }} - end: {{ data['network']['vlan_network_data']['calico']['static_end'] }} + start: {{ data.network.get_vlan_data_by_name('calico').static_start }} + end: {{ data.network.get_vlan_data_by_name('calico').static_end }} ... +--- +schema: 'drydock/Network/v1' +metadata: + schema: 'metadata/Document/v1' + name: overlay + layeringDefinition: + abstract: false + layer: site + parentSelector: + network_role: os-overlay + topology: cruiser + actions: + - method: merge + path: . + storagePolicy: cleartext +data: + cidr: {{ data.network.get_vlan_data_by_name('overlay').subnet[0] }} + ranges: + - type: reserved + start: {{ data.network.get_vlan_data_by_name('overlay').reserved_start }} + end: {{ data.network.get_vlan_data_by_name('overlay').reserved_end }} + - type: static + start: {{ data.network.get_vlan_data_by_name('overlay').static_start }} + end: {{ data.network.get_vlan_data_by_name('overlay').static_end }} +... diff --git a/spyglass/examples/templates/pki/pki-catalogue.yaml.j2 b/spyglass/examples/templates/pki/pki-catalogue.yaml.j2 index 7c7d6da..4d79147 100644 --- a/spyglass/examples/templates/pki/pki-catalogue.yaml.j2 +++ b/spyglass/examples/templates/pki/pki-catalogue.yaml.j2 @@ -21,33 +21,29 @@ data: - 10.96.0.1 kubernetes_service_names: - kubernetes.default.svc.cluster.local -{% for racks in data['baremetal'].keys()%} -{% for host in data['baremetal'][racks].keys()%} -{% if data['baremetal'][racks][host]['type'] == 'genesis' %} - + {% for host in data.get_baremetal_host_by_type('genesis') %} - document_name: kubelet-genesis - common_name: system:node:{{ host }} + common_name: system:node:{{ host.name }} hosts: - - {{ host }} - - {{ data['baremetal'][racks][host]['ip']['oam'] }} - - {{ data['baremetal'][racks][host]['ip']['calico']}} + - {{ host.name }} + - {{ host.ip.oam }} + - {{ host.ip.calico }} groups: - system:nodes -{% endif %} -{%endfor%} -{%endfor%} -{% for racks in data['baremetal'].keys()%} -{% for host in data['baremetal'][racks].keys()%} - - document_name: kubelet-{{ host }} - common_name: system:node:{{ host }} + {% endfor %} + + {% for rack in data.baremetal %} + {% for host in rack.hosts %} + - document_name: kubelet-{{ host.name }} + common_name: system:node:{{ host.name }} hosts: - - {{ host }} - - {{ data['baremetal'][racks][host]['ip']['oam'] }} - - {{ data['baremetal'][racks][host]['ip']['calico']}} + - {{ host.name }} + - {{ host.ip.oam }} + - {{ host.ip.calico }} groups: - system:nodes -{%endfor%} -{%endfor%} + {% endfor %} + {% endfor %} - document_name: scheduler description: Service certificate for Kubernetes scheduler common_name: system:kube-scheduler @@ -62,6 +58,7 @@ data: common_name: armada groups: - system:masters + kubernetes-etcd: description: Certificates for Kubernetes's etcd servers certificates: @@ -72,113 +69,93 @@ data: - document_name: kubernetes-etcd-anchor description: anchor common_name: anchor -{% for racks in data['baremetal'].keys()%} -{% for host in data['baremetal'][racks].keys()%} -{% if data['baremetal'][racks][host]['type'] == 'genesis' %} + {% for host in data.get_baremetal_host_by_type('genesis') %} - document_name: kubernetes-etcd-genesis common_name: kubernetes-etcd-genesis hosts: - - {{ host }} - - {{ data['baremetal'][racks][host]['ip']['oam'] }} - - {{ data['baremetal'][racks][host]['ip']['calico']}} + - {{ host.name }} + - {{ host.ip.oam }} + - {{ host.ip.calico }} - 127.0.0.1 - localhost - kubernetes-etcd.kube-system.svc.cluster.local - 10.96.0.2 -{% endif %} -{%endfor%} -{%endfor%} -{% for racks in data['baremetal'].keys()%} -{% for host in data['baremetal'][racks].keys()%} -{% if data['baremetal'][racks][host]['type'] == 'controller' or data['baremetal'][racks][host]['type'] == 'genesis'%} - - document_name: kubernetes-etcd-{{ host }} - common_name: kubernetes-etcd-{{ host }} + {% endfor %} + + {% for host in data.get_baremetal_host_by_type('controller', 'genesis') %} + - document_name: kubernetes-etcd-{{ host.name }} + common_name: kubernetes-etcd-{{ host.name }} hosts: - - {{ host }} - - {{ data['baremetal'][racks][host]['ip']['oam'] }} - - {{ data['baremetal'][racks][host]['ip']['calico']}} + - {{ host.name }} + - {{ host.ip.oam }} + - {{ host.ip.calico }} - 127.0.0.1 - localhost - kubernetes-etcd.kube-system.svc.cluster.local - 10.96.0.2 -{% endif %} -{%endfor%} -{%endfor%} -{% for racks in data['baremetal'].keys()%} -{% for host in data['baremetal'][racks].keys()%} -{% if data['baremetal'][racks][host]['type'] == 'genesis' %} + {% endfor %} + kubernetes-etcd-peer: certificates: + {% for host in data.get_baremetal_host_by_type('genesis') %} - document_name: kubernetes-etcd-genesis-peer common_name: kubernetes-etcd-genesis-peer hosts: - - {{ host }} - - {{ data['baremetal'][racks][host]['ip']['oam'] }} - - {{ data['baremetal'][racks][host]['ip']['calico']}} + - {{ host.name }} + - {{ host.ip.oam }} + - {{ host.ip.calico }} - 127.0.0.1 - localhost - kubernetes-etcd.kube-system.svc.cluster.local - 10.96.0.2 -{% endif %} -{%endfor%} -{%endfor%} -{% for racks in data['baremetal'].keys()%} -{% for host in data['baremetal'][racks].keys()%} -{% if data['baremetal'][racks][host]['type'] == 'controller' or data['baremetal'][racks][host]['type'] == 'genesis' %} - - document_name: kubernetes-etcd-{{ host }}-peer - common_name: kubernetes-etcd-{{ host }}-peer + {% endfor %} + + {% for host in data.get_baremetal_host_by_type('controller', 'genesis') %} + - document_name: kubernetes-etcd-{{ host.name }}-peer + common_name: kubernetes-etcd-{{ host.name }}-peer hosts: - - {{ host }} - - {{ data['baremetal'][racks][host]['ip']['oam'] }} - - {{ data['baremetal'][racks][host]['ip']['calico']}} + - {{ host.name }} + - {{ host.ip.oam }} + - {{ host.ip.calico }} - 127.0.0.1 - localhost - kubernetes-etcd.kube-system.svc.cluster.local - 10.96.0.2 -{% endif %} -{%endfor%} -{%endfor%} - ksn-etcd: + {% endfor %} + + calico-etcd: description: Certificates for Calico etcd client traffic certificates: - document_name: ksn-etcd-anchor description: anchor common_name: anchor -{% for racks in data['baremetal'].keys()%} -{% for host in data['baremetal'][racks].keys()%} -{% if data['baremetal'][racks][host]['type'] == 'controller' or data['baremetal'][racks][host]['type'] == 'genesis' %} - - document_name: ksn-etcd-{{ host }} - common_name: ksn-etcd-{{ host }} + {% for host in data.get_baremetal_host_by_type('controller', 'genesis') %} + - document_name: ksn-etcd-{{ host.name }} + common_name: ksn-etcd-{{ host.name }} hosts: - - {{ host }} - - {{ data['baremetal'][racks][host]['ip']['oam'] }} - - {{ data['baremetal'][racks][host]['ip']['calico']}} + - {{ host.name }} + - {{ host.ip.oam }} + - {{ host.ip.calico }} - 127.0.0.1 - localhost - 10.96.232.136 -{% endif %} -{%endfor%} -{%endfor%} + {% endfor %} - document_name: ksn-node common_name: calcico-node - ksn-etcd-peer: + calico-etcd-peer: description: Certificates for Calico etcd clients certificates: -{% for racks in data['baremetal'].keys()%} -{% for host in data['baremetal'][racks].keys()%} -{% if data['baremetal'][racks][host]['type'] == 'controller' or data['baremetal'][racks][host]['type'] == 'genesis' %} - - document_name: ksn-etcd-{{ host }}-peer - common_name: ksn-etcd-{{ host }}-peer + {% for host in data.get_baremetal_host_by_type('controller', 'genesis') %} + - document_name: ksn-etcd-{{ host.name }}-peer + common_name: ksn-etcd-{{ host.name }}-peer hosts: - - {{ host }} - - {{ data['baremetal'][racks][host]['ip']['oam'] }} - - {{ data['baremetal'][racks][host]['ip']['calico']}} + - {{ host.name }} + - {{ host.ip.oam }} + - {{ host.ip.calico }} - 127.0.0.1 - localhost - 10.96.232.136 -{% endif %} -{%endfor%} -{%endfor%} + {% endfor %} - document_name: ksn-node-peer common_name: calico-node-peer keypairs: diff --git a/spyglass/examples/templates/profile/region.yaml.j2 b/spyglass/examples/templates/profile/region.yaml.j2 index 9f862ab..a6ad518 100644 --- a/spyglass/examples/templates/profile/region.yaml.j2 +++ b/spyglass/examples/templates/profile/region.yaml.j2 @@ -2,7 +2,7 @@ schema: 'drydock/Region/v1' metadata: schema: 'metadata/Document/v1' - name: {{ data['region_name'] }} + name: {{ data.site_info.region_name }} layeringDefinition: abstract: false layer: site @@ -18,7 +18,7 @@ metadata: path: .authorized_keys[1] src: schema: deckhand/PublicKey/v1 - name: {{ data['region_name'] }}_ssh_public_key + name: {{ data.site_info.region_name }}_ssh_public_key path: . - dest: path: .repositories.main_archive diff --git a/spyglass/examples/templates/site-definition.yaml.j2 b/spyglass/examples/templates/site-definition.yaml.j2 index 02affa9..6d72b72 100644 --- a/spyglass/examples/templates/site-definition.yaml.j2 +++ b/spyglass/examples/templates/site-definition.yaml.j2 @@ -7,11 +7,11 @@ metadata: abstract: false layer: site # NEWSITE-CHANGEME: Replace with the site name - name: {{ data['region_name'] }} + name: {{ data.site_info.region_name }} storagePolicy: cleartext data: # The type layer this site will delpoy with. Type layer is found in the # aic-clcp-manifests repo. - site_type: {{ data['site_info']['sitetype'] }} + site_type: {{ data.site_info.sitetype }} ... diff --git a/spyglass/examples/templates/software/charts/kubernetes/etcd.yaml.j2 b/spyglass/examples/templates/software/charts/kubernetes/etcd.yaml.j2 index 40e7f8c..7d2415f 100644 --- a/spyglass/examples/templates/software/charts/kubernetes/etcd.yaml.j2 +++ b/spyglass/examples/templates/software/charts/kubernetes/etcd.yaml.j2 @@ -13,83 +13,70 @@ metadata: path: . storagePolicy: cleartext substitutions: -{% set count = [0] %} -{% for rack in data.baremetal.keys() %} -{% for host in data["baremetal"][rack] %} -{% if data["baremetal"][rack][host]["type"] == 'controller' %} +{% for host in data.get_baremetal_host_by_type('controller') %} - src: schema: pegleg/CommonAddresses/v1 name: common-addresses - path: .masters[{{ count[0] }}].hostname + path: .masters[{{ loop.index - 1 }}].hostname dest: - path: .values.nodes[{{ count[0] }}].name + path: .values.nodes[{{ loop.index - 1 }}].name - src: schema: deckhand/Certificate/v1 - name: calico-etcd-{{ host }} + name: calico-etcd-{{ host.name }} path: . dest: - path: .values.nodes[{{ count[0] }}].tls.client.cert + path: .values.nodes[{{ loop.index - 1 }}].tls.client.cert - src: schema: deckhand/CertificateKey/v1 - name: calico-etcd-{{ host }} + name: calico-etcd-{{ host.name }} path: . dest: - path: .values.nodes[{{ count[0] }}].tls.client.key + path: .values.nodes[{{ loop.index - 1 }}].tls.client.key - src: schema: deckhand/Certificate/v1 - name: calico-etcd-{{ host }} + name: calico-etcd-{{ host.name }} path: . dest: - path: .values.nodes[{{ count[0] }}].tls.peer.cert + path: .values.nodes[{{ loop.index - 1 }}].tls.peer.cert - src: schema: deckhand/CertificateKey/v1 - name: calico-etcd-{{ host }}-peer + name: calico-etcd-{{ host.name }}-peer path: . dest: - path: .values.nodes[{{ count[0] }}].tls.peer.key - - {% if count.append(count.pop() + 1) %}{% endif %} {# increment count by 1 #} - -{% endif %} -{% endfor %} + path: .values.nodes[{{ loop.index - 1 }}].tls.peer.key {% endfor %} -{% for rack in data.baremetal.keys() %} -{% for host in data["baremetal"][rack] %} -{% if data["baremetal"][rack][host]["type"] == 'genesis' %} +{% for host in data.get_baremetal_host_by_type('genesis') %} - src: schema: pegleg/CommonAddresses/v1 name: common-addresses path: .genesis.hostname dest: - path: .values.nodes[{{ count[0] }}].name + path: .values.nodes[{{ loop.index - 1 }}].name - src: schema: deckhand/Certificate/v1 - name: calico-etcd-{{ host }} + name: calico-etcd-{{ host.name }} path: . dest: - path: .values.nodes[{{ count[0] }}].tls.client.cert + path: .values.nodes[{{ loop.index - 1 }}].tls.client.cert - src: schema: deckhand/CertificateKey/v1 - name: calico-etcd-{{ host }} + name: calico-etcd-{{ host.name }} path: . dest: - path: .values.nodes[{{ count[0] }}].tls.client.key + path: .values.nodes[{{ loop.index - 1 }}].tls.client.key - src: schema: deckhand/Certificate/v1 - name: calico-etcd-{{ host }}-peer + name: calico-etcd-{{ host.name }}-peer path: . dest: - path: .values.nodes[{{ count[0] }}].tls.peer.cert + path: .values.nodes[{{ loop.index - 1 }}].tls.peer.cert - src: schema: deckhand/CertificateKey/v1 - name: calico-etcd-{{ host }}-peer + name: calico-etcd-{{ host.name }}-peer path: . dest: - path: .values.nodes[{{ count[0] }}].tls.peer.key - -{% endif %} -{% endfor %} + path: .values.nodes[{{ loop.index - 1 }}].tls.peer.key {% endfor %} data: {} diff --git a/spyglass/examples/templates/software/charts/kubernetes/etcd/etcd.yaml.j2 b/spyglass/examples/templates/software/charts/kubernetes/etcd/etcd.yaml.j2 index d5b85fa..4193d0b 100644 --- a/spyglass/examples/templates/software/charts/kubernetes/etcd/etcd.yaml.j2 +++ b/spyglass/examples/templates/software/charts/kubernetes/etcd/etcd.yaml.j2 @@ -13,80 +13,70 @@ metadata: path: . storagePolicy: cleartext substitutions: -{% set count = [0] %} -{% for rack in data.baremetal.keys() %} -{% for host in data["baremetal"][rack] %} -{% if data["baremetal"][rack][host]["type"] == 'controller' %} + {% for host in data.get_baremetal_host_by_type('controller') %} - src: schema: pegleg/CommonAddresses/v1 name: common-addresses - path: .masters[{{ count[0] }}].hostname + path: .masters[{{ loop.index - 1 }}].hostname dest: - path: .values.nodes[{{ count[0] }}].name + path: .values.nodes[{{ loop.index - 1 }}].name - src: schema: deckhand/Certificate/v1 - name: kubernetes-etcd-{{ host }} + name: kubernetes-etcd-{{ host.name }} path: . dest: - path: .values.nodes[{{ count[0] }}].tls.client.cert + path: .values.nodes[{{ loop.index - 1 }}].tls.client.cert - src: schema: deckhand/CertificateKey/v1 - name: kubernetes-etcd-{{ host }} + name: kubernetes-etcd-{{ host.name }} path: . dest: - path: .values.nodes[{{ count[0] }}].tls.client.key + path: .values.nodes[{{ loop.index - 1 }}].tls.client.key - src: schema: deckhand/Certificate/v1 - name: kubernetes-etcd-{{ host }}-peer + name: kubernetes-etcd-{{ host.name }}-peer path: . dest: - path: .values.nodes[{{ count[0] }}].tls.peer.cert + path: .values.nodes[{{ loop.index - 1 }}].tls.peer.cert - src: schema: deckhand/CertificateKey/v1 - name: kubernetes-etcd-{{ host }}-peer + name: kubernetes-etcd-{{ host.name }}-peer path: . dest: - path: .values.nodes[{{ count[0] }}].tls.peer.key - {% if count.append(count.pop() + 1) %}{% endif %} {# increment count by 1 #} -{% endif %} -{% endfor %} -{% endfor %} -{% for rack in data.baremetal.keys() %} -{% for host in data["baremetal"][rack] %} -{% if data["baremetal"][rack][host]["type"] == 'genesis' %} + path: .values.nodes[{{ loop.index - 1 }}].tls.peer.key + {% endfor %} + {% for host in data.get_baremetal_host_by_type('genesis') %} - src: schema: pegleg/CommonAddresses/v1 name: common-addresses path: .genesis.hostname dest: - path: .values.nodes[{{ count[0] }}].name + path: .values.nodes[{{ loop.index - 1 }}].name - src: schema: deckhand/Certificate/v1 name: kubernetes-etcd-genesis path: . dest: - path: .values.nodes[{{ count[0] }}].tls.client.cert + path: .values.nodes[{{ loop.index - 1 }}].tls.client.cert - src: schema: deckhand/CertificateKey/v1 name: kubernetes-etcd-genesis path: . dest: - path: .values.nodes[{{ count[0] }}].tls.client.key + path: .values.nodes[{{ loop.index - 1 }}].tls.client.key - src: schema: deckhand/Certificate/v1 name: kubernetes-etcd-genesis-peer path: . dest: - path: .values.nodes[{{ count[0] }}].tls.peer.cert + path: .values.nodes[{{ loop.index - 1 }}].tls.peer.cert - src: schema: deckhand/CertificateKey/v1 name: kubernetes-etcd-genesis-peer path: $ dest: - path: .values.nodes[{{ count[0] }}].tls.peer.key -{% endif %} -{% endfor %} -{% endfor %} + path: .values.nodes[{{ loop.index - 1 }}].tls.peer.key + {% endfor %} data: {} ... diff --git a/spyglass/examples/templates/software/charts/osh/openstack-tenant-ceph/ceph-client.yaml.j2 b/spyglass/examples/templates/software/charts/osh/openstack-tenant-ceph/ceph-client.yaml.j2 index f4e4257..c0409a4 100644 --- a/spyglass/examples/templates/software/charts/osh/openstack-tenant-ceph/ceph-client.yaml.j2 +++ b/spyglass/examples/templates/software/charts/osh/openstack-tenant-ceph/ceph-client.yaml.j2 @@ -18,5 +18,5 @@ data: conf: pool: target: - osd: {{ data['storage']['ceph']['controller']['osd_count'] }} + osd: {{ data.storage['ceph']['controller']['osd_count'] }} ... diff --git a/spyglass/examples/templates/software/config/common-software-config.yaml.j2 b/spyglass/examples/templates/software/config/common-software-config.yaml.j2 index d1bb309..2cb6083 100644 --- a/spyglass/examples/templates/software/config/common-software-config.yaml.j2 +++ b/spyglass/examples/templates/software/config/common-software-config.yaml.j2 @@ -13,4 +13,5 @@ metadata: data: osh: # NEWSITE-CHANGEME: Replace with the site name - region_name: {{ data['region_name'] }} + region_name: {{ data.site_info.region_name }} +... diff --git a/spyglass/site_processors/base.py b/spyglass/site_processors/base.py index 625388f..994cb45 100644 --- a/spyglass/site_processors/base.py +++ b/spyglass/site_processors/base.py @@ -20,22 +20,3 @@ class BaseProcessor(object): def render_template(self, template): pass - - @staticmethod - def get_role_wise_nodes(yaml_data): - hosts = {"genesis": {}, "masters": [], "workers": []} - - for rack in yaml_data["baremetal"]: - for host in yaml_data["baremetal"][rack]: - if yaml_data["baremetal"][rack][host]["type"] == "genesis": - hosts["genesis"] = { - "name": host, - "pxe": yaml_data["baremetal"][rack][host]["ip"]["pxe"], - "oam": yaml_data["baremetal"][rack][host]["ip"]["oam"], - } - elif yaml_data["baremetal"][rack][host]["type"] \ - == "controller": - hosts["masters"].append(host) - else: - hosts["workers"].append(host) - return hosts diff --git a/spyglass/site_processors/site_processor.py b/spyglass/site_processors/site_processor.py index e299f39..f3d21a6 100644 --- a/spyglass/site_processors/site_processor.py +++ b/spyglass/site_processors/site_processor.py @@ -18,6 +18,8 @@ import shutil import jinja2 +from spyglass.data_extractor.models import site_document_data_factory +from spyglass.data_extractor.models import SiteDocumentData from spyglass.site_processors.base import BaseProcessor LOG = logging.getLogger(__name__) @@ -25,9 +27,12 @@ LOG = logging.getLogger(__name__) class SiteProcessor(BaseProcessor): - def __init__(self, intermediary_yaml, manifest_dir, force_write): + def __init__(self, site_data, manifest_dir, force_write): super().__init__() - self.yaml_data = intermediary_yaml + if isinstance(site_data, SiteDocumentData): + self.site_data = site_data + else: + self.site_data = site_document_data_factory(site_data) self.manifest_dir = manifest_dir self.force_write = force_write @@ -38,7 +43,7 @@ class SiteProcessor(BaseProcessor): calico) are generated in a single file. Rack specific configs( pxe and oob) are generated per rack. """ - # Check of manifest_dir exists + # Check if manifest_dir exists if self.manifest_dir is not None: site_manifest_dir = os.path.join( self.manifest_dir, 'pegleg_manifests', 'site') @@ -66,16 +71,16 @@ class SiteProcessor(BaseProcessor): autoescape=True, loader=loader, trim_blocks=True, + lstrip_blocks=True, undefined=logging_undefined) - j2_env.filters["get_role_wise_nodes"] = \ - self.get_role_wise_nodes templatefile = os.path.join(dirpath, filename) LOG.debug("Template file: %s", templatefile) outdirs = dirpath.split(template_folder_name)[1].lstrip(os.sep) LOG.debug("outdirs: %s", outdirs) outfile_path = os.path.join( - site_manifest_dir, self.yaml_data["region_name"], outdirs) + site_manifest_dir, self.site_data.site_info.region_name, + outdirs) LOG.debug("outfile path: %s", outfile_path) outfile_yaml = os.path.split(templatefile)[1] outfile_yaml = os.path.splitext(outfile_yaml)[0] @@ -90,7 +95,7 @@ class SiteProcessor(BaseProcessor): out = open(outfile, "w") created_file_list.append(outfile) LOG.info("Rendering {}".format(outfile_yaml)) - rendered = template_j2.render(data=self.yaml_data) + rendered = template_j2.render(data=self.site_data) out.write(rendered) out.close() except IOError as ioe: diff --git a/tests/unit/data_extractor/test_models.py b/tests/unit/data_extractor/test_models.py index 656d4f3..296698f 100644 --- a/tests/unit/data_extractor/test_models.py +++ b/tests/unit/data_extractor/test_models.py @@ -13,11 +13,18 @@ # limitations under the License. from copy import copy +import os import unittest from unittest import mock +import yaml + +from spyglass.data_extractor.custom_exceptions import InvalidIntermediary from spyglass.data_extractor import models +FIXTURE_DIR = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'shared') + class TestParseIp(unittest.TestCase): """Tests the _parse_ip validator for Spyglass models""" @@ -340,11 +347,7 @@ class TestRack(unittest.TestCase): """Tests for the Rack model""" RACK_NAME = 'test_rack1' - HOST_DATA = { - 'rack_name': RACK_NAME, - 'host_profile': 'host', - 'type': 'compute' - } + HOST_DATA = {'rack_name': RACK_NAME, 'host_profile': 'host'} @mock.patch('spyglass.data_extractor.models.IPList', autospec=True) def setUp(self, MockIPList): @@ -353,9 +356,9 @@ class TestRack(unittest.TestCase): self.HOST_DATA['ip'] = MockIPList() self.HOST_DATA['ip'].dict_from_class.return_value = 'success' self.hosts = [ - models.Host('test_host1', **self.HOST_DATA), - models.Host('test_host2', **self.HOST_DATA), - models.Host('test_host3', **self.HOST_DATA), + models.Host('test_host1', **self.HOST_DATA, type='genesis'), + models.Host('test_host2', **self.HOST_DATA, type='compute'), + models.Host('test_host3', **self.HOST_DATA, type='controller'), ] def test___init__(self): @@ -408,6 +411,15 @@ class TestRack(unittest.TestCase): self.assertEqual( self.hosts[1], result.get_host_by_name(self.hosts[1].name)) + def test_get_host_by_type(self): + """Tests retrieval of a Rack's host(s) by type""" + result = models.Rack(self.RACK_NAME, self.hosts) + self.assertEqual(self.hosts[0], result.get_host_by_type('genesis')[0]) + self.assertEqual(self.hosts[1], result.get_host_by_type('compute')[0]) + self.assertEqual( + self.hosts[2], + result.get_host_by_type('controller')[0]) + class TestVLANNetworkData(unittest.TestCase): """Tests for the VLANNetworkData model""" @@ -820,3 +832,162 @@ class TestSiteDocumentData(unittest.TestCase): type(Rack()).name = mock.PropertyMock(side_effect=['rack1', 'rack3']) result = models.SiteDocumentData(site_info, network, baremetal) self.assertIsNone(result.get_baremetal_rack_by_name('rack2')) + + @mock.patch('spyglass.data_extractor.models.SiteInfo') + @mock.patch('spyglass.data_extractor.models.Network') + @mock.patch('spyglass.data_extractor.models.Rack') + @mock.patch('spyglass.data_extractor.models.Host') + def test_get_baremetal_rack_by_name_multiple( + self, Host, Rack, Network, SiteInfo): + """Tests retrieval of baremetal host(s) by type""" + site_info = SiteInfo() + network = Network() + baremetal = [Rack(), Rack()] + Rack().get_host_by_type.return_value = [Host()] + result = models.SiteDocumentData(site_info, network, baremetal) + self.assertEqual(2, len(result.get_baremetal_host_by_type('genesis'))) + self.assertEqual(2, len(result.get_baremetal_host_by_type('computer'))) + self.assertEqual( + 2, len(result.get_baremetal_host_by_type('controller'))) + + +class TestValidateKeyInIntermediaryDict(unittest.TestCase): + """Tests the _validate_key_in_intermediary_dict function""" + + def test__validate_key_in_intermediary_dict(self): + test_dictionary = {'test_key': 'value'} + key = 'test_key' + self.assertIsNone( + models._validate_key_in_intermediary_dict(key, test_dictionary)) + + def test__validate_key_in_intermediary_dict_key_dne(self): + test_dictionary = {'test_key': 'value'} + key = 'not_test_key' + with self.assertRaises(InvalidIntermediary): + models._validate_key_in_intermediary_dict(key, test_dictionary) + + +class TestSiteDocumentDataFactory(unittest.TestCase): + """Tests the site_document_data_factory function""" + + def setUp(self) -> None: + test_intermediary_path = os.path.join( + FIXTURE_DIR, 'test_intermediary.yaml') + with open(test_intermediary_path, 'r') as f: + self.intermediary_dict = yaml.safe_load(f) + + def test_site_document_data_factory(self): + site_document_data = models.site_document_data_factory( + self.intermediary_dict) + + # Check correct return type + self.assertIsInstance(site_document_data, models.SiteDocumentData) + + def test_site_document_data_factory_saves_storage(self): + site_document_data = models.site_document_data_factory( + self.intermediary_dict) + + # Check that storage was saved without changes in the SiteDocumentData + self.assertDictEqual( + self.intermediary_dict['storage'], site_document_data.storage) + + def test_site_document_data_factory_saves_site_info(self): + site_document_data = models.site_document_data_factory( + self.intermediary_dict) + + # Check that site info saved correctly in SiteInfo object + site_info_dict = self.intermediary_dict['site_info'] + self.assertIsInstance(site_document_data.site_info, models.SiteInfo) + self.assertEqual( + site_info_dict['name'], site_document_data.site_info.name) + self.assertEqual( + self.intermediary_dict['region_name'], + site_document_data.site_info.region_name) + self.assertEqual( + site_info_dict['state'], site_document_data.site_info.state) + self.assertEqual( + site_info_dict['physical_location_id'], + site_document_data.site_info.physical_location_id) + self.assertEqual( + site_info_dict['country'], site_document_data.site_info.country) + self.assertEqual( + site_info_dict['corridor'], site_document_data.site_info.corridor) + self.assertEqual( + site_info_dict['sitetype'], site_document_data.site_info.sitetype) + self.assertEqual( + site_info_dict['domain'], site_document_data.site_info.domain) + self.assertDictEqual( + site_info_dict['ldap'], site_document_data.site_info.ldap) + self.assertEqual( + site_info_dict['dns']['servers'], + str(site_document_data.site_info.dns)) + self.assertEqual( + site_info_dict['ntp']['servers'], + str(site_document_data.site_info.ntp)) + + def test_site_document_data_factory_saves_network_data(self): + site_document_data = models.site_document_data_factory( + self.intermediary_dict) + + # Check that network data saved correctly into a Network object + network_dict = self.intermediary_dict['network'] + self.assertIsInstance(site_document_data.network, models.Network) + self.assertDictEqual( + network_dict['bgp'], site_document_data.network.bgp) + for network_type, network_data \ + in network_dict['vlan_network_data'].items(): + vlan_network_data = \ + site_document_data.network.get_vlan_data_by_name(network_type) + self.assertIsInstance(vlan_network_data, models.VLANNetworkData) + self.assertEqual(network_type, vlan_network_data.name) + self.assertEqual(network_type, vlan_network_data.role) + self.assertEqual(network_data['subnet'], vlan_network_data.subnet) + if 'routes' in network_data: + self.assertEqual( + network_data['routes'], vlan_network_data.routes) + if 'gateway' in network_data: + self.assertEqual( + network_data['gateway'], vlan_network_data.gateway) + if 'vlan' in network_data: + self.assertEqual(network_data['vlan'], vlan_network_data.vlan) + if 'dhcp_start' in network_data and 'dhcp_end' in network_data: + self.assertEqual( + network_data['dhcp_start'], vlan_network_data.dhcp_start) + self.assertEqual( + network_data['dhcp_end'], vlan_network_data.dhcp_end) + if 'static_start' in network_data and 'static_end' in network_data: + self.assertEqual( + network_data['static_start'], + vlan_network_data.static_start) + self.assertEqual( + network_data['static_end'], vlan_network_data.static_end) + if 'reserved_start' in network_data \ + and 'reserved_end' in network_data: + self.assertEqual( + network_data['reserved_start'], + vlan_network_data.reserved_start) + self.assertEqual( + network_data['reserved_end'], + vlan_network_data.reserved_end) + + def test_site_document_data_factory_saves_baremetal_data(self): + site_document_data = models.site_document_data_factory( + self.intermediary_dict) + + # Check that baremetal racks saved correctly into Rack objects + for rack_name, hosts \ + in self.intermediary_dict['baremetal'].items(): + rack = site_document_data.get_baremetal_rack_by_name(rack_name) + for host_name, host_data in hosts.items(): + host = rack.get_host_by_name(host_name) + self.assertEqual(host_name, host.name) + self.assertEqual(rack_name, host.rack_name) + self.assertEqual(host_data['type'], host.type) + self.assertEqual(host_data['host_profile'], host.host_profile) + self.assertEqual(host_data['ip']['oob'], host.ip.oob) + self.assertEqual(host_data['ip']['oam'], host.ip.oam) + self.assertEqual(host_data['ip']['calico'], host.ip.calico) + self.assertEqual(host_data['ip']['overlay'], host.ip.overlay) + self.assertEqual(host_data['ip']['pxe'], host.ip.pxe) + self.assertEqual(host_data['ip']['storage'], host.ip.storage) + self.assertEqual(rack_name, rack.name) diff --git a/tests/unit/site_processors/test_site_processor.py b/tests/unit/site_processors/test_site_processor.py index 474b270..8e6ea55 100644 --- a/tests/unit/site_processors/test_site_processor.py +++ b/tests/unit/site_processors/test_site_processor.py @@ -15,117 +15,178 @@ import logging import os from tempfile import mkdtemp +import textwrap +import unittest +from unittest import mock from jinja2 import UndefinedError import pytest +from spyglass.data_extractor import models from spyglass.site_processors.site_processor import SiteProcessor LOG = logging.getLogger(__name__) LOG.level = logging.DEBUG -J2_TPL = """--- -schema: pegleg/SiteDefinition/v1 -metadata: - schema: metadata/Document/v1 - layeringDefinition: - abstract: false - layer: site - name: {{ data['region_name'] }} - storagePolicy: cleartext -data: - site_type:{{ data['site_info']['sitetype'] }} -...""" +class TestSiteProcessor(unittest.TestCase): -def test_render_template(): - _tpl_parent_dir = mkdtemp() - _tpl_dir = mkdtemp(dir=_tpl_parent_dir) - _tpl_file = os.path.join(_tpl_dir, "test.yaml.j2") - with open(_tpl_file, 'w') as f: - f.write(J2_TPL) - LOG.debug("Writing test template to %s", _tpl_file) - _input_yaml = { - "region_name": "test", - "site_info": { - "sitetype": "test_type" - } - } - _out_dir = mkdtemp() - site_processor = SiteProcessor(_input_yaml, _out_dir, force_write=False) - site_processor.render_template(_tpl_parent_dir) + J2_TPL = textwrap.dedent( + """ + --- + schema: pegleg/SiteDefinition/v1 + metadata: + schema: metadata/Document/v1 + layeringDefinition: + abstract: false + layer: site + name: {{ data.site_info.region_name }} + storagePolicy: cleartext + data: + site_type:{{ data.site_info.sitetype }} + ...""") - expected_output = """--- -schema: pegleg/SiteDefinition/v1 -metadata: - schema: metadata/Document/v1 - layeringDefinition: - abstract: false - layer: site - name: test - storagePolicy: cleartext -data: - site_type:test_type -...""" + J2_TPL_UNDEFINED = textwrap.dedent( + """ + --- + schema: pegleg/SiteDefinition/v1 + metadata: + schema: metadata/Document/v1 + layeringDefinition: + abstract: false + layer: site + name: {{ data.site_info.region_name }} + storagePolicy: cleartext + data: + site_type:{{ undefined_param }} + ...""") - output_file = os.path.join( - _out_dir, "pegleg_manifests", "site", _input_yaml["region_name"], - os.path.split(_tpl_dir)[1], "test.yaml") - LOG.debug(output_file) - assert (os.path.exists(output_file)) - with open(output_file, 'r') as f: - content = f.read() - assert (expected_output == content) + @mock.patch( + 'spyglass.data_extractor.models.SiteDocumentData', + spec=models.SiteDocumentData) + @mock.patch('spyglass.data_extractor.models.SiteInfo') + @mock.patch('spyglass.data_extractor.models.ServerList') + def test_render_template(self, ServerList, SiteInfo, SiteDocumentData): + _tpl_parent_dir = mkdtemp() + _tpl_dir = mkdtemp(dir=_tpl_parent_dir) + _tpl_file = os.path.join(_tpl_dir, "test.yaml.j2") + with open(_tpl_file, 'w') as f: + f.write(self.J2_TPL) + LOG.debug("Writing test template to %s", _tpl_file) + site_data = SiteDocumentData() + type(SiteDocumentData()).site_info = SiteInfo() + region_name = 'test' + type(SiteInfo()).region_name = mock.PropertyMock( + return_value=region_name) + site_type = 'test_type' + type(SiteInfo()).sitetype = mock.PropertyMock(return_value=site_type) -def test_render_template_missing_data(): - _tpl_parent_dir = mkdtemp() - _tpl_dir = mkdtemp(dir=_tpl_parent_dir) - _tpl_file = os.path.join(_tpl_dir, "test.yaml.j2") - with open(_tpl_file, 'w') as f: - f.write(J2_TPL) - LOG.debug("Writing test template to %s", _tpl_file) - _input_yaml = {"region_name": "test", "site_info": {}} - _out_dir = mkdtemp() - site_processor = SiteProcessor(_input_yaml, _out_dir, force_write=False) - with pytest.raises(UndefinedError): + _out_dir = mkdtemp() + site_processor = SiteProcessor(site_data, _out_dir, force_write=False) site_processor.render_template(_tpl_parent_dir) - output_file = os.path.join( - _out_dir, "pegleg_manifests", "site", _input_yaml["region_name"], - os.path.split(_tpl_dir)[1], "test.yaml") - assert (not os.path.exists(output_file)) + expected_output = textwrap.dedent( + """ + --- + schema: pegleg/SiteDefinition/v1 + metadata: + schema: metadata/Document/v1 + layeringDefinition: + abstract: false + layer: site + name: test + storagePolicy: cleartext + data: + site_type:test_type + ...""") + output_file = os.path.join( + _out_dir, "pegleg_manifests", "site", region_name, + os.path.split(_tpl_dir)[1], "test.yaml") + LOG.debug(output_file) + self.assertTrue(os.path.exists(output_file)) + with open(output_file, 'r') as f: + content = f.read() + self.assertEqual(expected_output, content) -def test_render_template_missing_data_force(): - _tpl_parent_dir = mkdtemp() - _tpl_dir = mkdtemp(dir=_tpl_parent_dir) - _tpl_file = os.path.join(_tpl_dir, "test.yaml.j2") - with open(_tpl_file, 'w') as f: - f.write(J2_TPL) - LOG.debug("Writing test template to %s", _tpl_file) - _input_yaml = {"region_name": "test", "site_info": {}} - _out_dir = mkdtemp() - site_processor = SiteProcessor(_input_yaml, _out_dir, force_write=True) - site_processor.render_template(_tpl_parent_dir) + @mock.patch( + 'spyglass.data_extractor.models.SiteDocumentData', + spec=models.SiteDocumentData) + @mock.patch('spyglass.data_extractor.models.SiteInfo') + @mock.patch('spyglass.data_extractor.models.ServerList') + def test_render_template_missing_data( + self, ServerList, SiteInfo, SiteDocumentData): + _tpl_parent_dir = mkdtemp() + _tpl_dir = mkdtemp(dir=_tpl_parent_dir) + _tpl_file = os.path.join(_tpl_dir, "test.yaml.j2") + with open(_tpl_file, 'w') as f: + f.write(self.J2_TPL_UNDEFINED) + LOG.debug("Writing test template to %s", _tpl_file) - expected_output = """--- -schema: pegleg/SiteDefinition/v1 -metadata: - schema: metadata/Document/v1 - layeringDefinition: - abstract: false - layer: site - name: test - storagePolicy: cleartext -data: - site_type: -...""" + site_data = SiteDocumentData() + type(SiteDocumentData()).site_info = SiteInfo() + region_name = 'test' + type(SiteInfo()).region_name = mock.PropertyMock( + return_value=region_name) + site_type = 'test_type' + type(SiteInfo()).sitetype = mock.PropertyMock(return_value=site_type) - output_file = os.path.join( - _out_dir, "pegleg_manifests", "site", _input_yaml["region_name"], - os.path.split(_tpl_dir)[1], "test.yaml") - assert (os.path.exists(output_file)) - with open(output_file, 'r') as f: - content = f.read() - assert (expected_output == content) + _out_dir = mkdtemp() + site_processor = SiteProcessor(site_data, _out_dir, force_write=False) + with pytest.raises(UndefinedError): + site_processor.render_template(_tpl_parent_dir) + + output_file = os.path.join( + _out_dir, "pegleg_manifests", "site", region_name, + os.path.split(_tpl_dir)[1], "test.yaml") + self.assertFalse(os.path.exists(output_file)) + + @mock.patch( + 'spyglass.data_extractor.models.SiteDocumentData', + spec=models.SiteDocumentData) + @mock.patch('spyglass.data_extractor.models.SiteInfo') + @mock.patch('spyglass.data_extractor.models.ServerList') + def test_render_template_missing_data_force( + self, ServerList, SiteInfo, SiteDocumentData): + _tpl_parent_dir = mkdtemp() + _tpl_dir = mkdtemp(dir=_tpl_parent_dir) + _tpl_file = os.path.join(_tpl_dir, "test.yaml.j2") + with open(_tpl_file, 'w') as f: + f.write(self.J2_TPL_UNDEFINED) + LOG.debug("Writing test template to %s", _tpl_file) + + site_data = SiteDocumentData() + type(SiteDocumentData()).site_info = SiteInfo() + region_name = 'test' + type(SiteInfo()).region_name = mock.PropertyMock( + return_value=region_name) + site_type = 'test_type' + type(SiteInfo()).sitetype = mock.PropertyMock(return_value=site_type) + + _out_dir = mkdtemp() + site_processor = SiteProcessor(site_data, _out_dir, force_write=True) + site_processor.render_template(_tpl_parent_dir) + + expected_output = textwrap.dedent( + """ + --- + schema: pegleg/SiteDefinition/v1 + metadata: + schema: metadata/Document/v1 + layeringDefinition: + abstract: false + layer: site + name: test + storagePolicy: cleartext + data: + site_type: + ...""") + + output_file = os.path.join( + _out_dir, "pegleg_manifests", "site", region_name, + os.path.split(_tpl_dir)[1], "test.yaml") + self.assertTrue(os.path.exists(output_file)) + with open(output_file, 'r') as f: + content = f.read() + self.assertEqual(expected_output, content)