airship
/
spyglass
Code Issues Proposed changes

Use data objects for document generation 

This is a follow-up change to [0] which further implements data objects
to be used in the generation of documents in Jinja2. The following
additions and changes are made:

- Adds helper functions to data objects to filter hosts and networks for
ease of use in Jinja2 templates
- Adds SiteDocumentData factory function to convert intermediary yaml
dictionaries into a SiteDocumentData object with all associated objects
- Updates Jinja2 templates to use data objects
- Cleans up overly complex looping in Jinja2
- Adds tests for new code in models.py

[0] https://review.opendev.org/#/c/662092/

Change-Id: I66ebfeaf5d6ca76b6dee5a2285a74bad8b06b720
This commit is contained in:
Ian H. Pittwood 2019-05-23 14:15:43 -05:00 committed by Ian H Pittwood
parent 4747222641
commit efe24d8a5f
18 changed files with 711 additions and 558 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
spyglass/data_extractor/custom_exceptions.py
View File

@ -59,3 +59,7 @@ class TokenGenerationError(BaseError):
class FormationConnectionError(BaseError): class FormationConnectionError(BaseError):
pass pass
class InvalidIntermediary(BaseError):
pass

112
spyglass/data_extractor/models.py
View File

@ -12,9 +12,12 @@
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
from copy import deepcopy
import ipaddress import ipaddress
import logging import logging
from spyglass.data_extractor.custom_exceptions import InvalidIntermediary
DATA_DEFAULT = "#CHANGE_ME" DATA_DEFAULT = "#CHANGE_ME"
LOG = logging.getLogger(__name__) LOG = logging.getLogger(__name__)
@ -253,6 +256,19 @@ class Rack(object):
return host return host
return None return None
def get_host_by_type(self, host_type: str):
"""Gets host(s) on rack by role
:param host_type: Role of the host(s) to be retrieved
:return: list of hosts
:rtype: list
"""
matching_hosts = []
for host in self.hosts:
if host.type == host_type:
matching_hosts.append(host)
return matching_hosts
class VLANNetworkData(object): class VLANNetworkData(object):
"""Model for single entry of VLAN Network Data""" """Model for single entry of VLAN Network Data"""
@ -594,3 +610,99 @@ class SiteDocumentData(object):
if rack.name == name: if rack.name == name:
return rack return rack
return None return None
def get_baremetal_host_by_type(self, *args):
"""Return baremetal host(s) with matching type
:param args: type(s) of the baremetal host
:return: Host object(s) matching the specified host_type
:rtype: list
"""
host_list = []
for rack in self.baremetal:
rack_hosts = []
for arg in args:
rack_hosts.extend(rack.get_host_by_type(arg))
host_list.extend(rack_hosts)
return host_list
def _validate_key_in_intermediary_dict(key: str, dictionary: dict):
if key not in dictionary:
raise InvalidIntermediary(
'%s is not defined in the given intermediary file.' % key)
def site_document_data_factory(intermediary_dict: dict) -> SiteDocumentData:
"""Uses intermediary file data to create a SiteDocumentData object
:param intermediary_dict: A loaded intermediary file dictionary
:return: all intermediary dictionary data returned as an object
"""
# Validate baremetal in intermediary
_validate_key_in_intermediary_dict('baremetal', intermediary_dict)
# Pull out baremetal data into Rack and Host objects
rack_list = []
for rack, hosts in intermediary_dict['baremetal'].items():
host_list = []
for host_name, host_data in hosts.items():
host_ip_list = IPList(**host_data['ip'])
host_kwargs = {
'rack_name': rack,
'host_profile': host_data['host_profile'],
'type': host_data['type'],
'ip': host_ip_list
}
new_host = Host(host_name, **host_kwargs)
host_list.append(new_host)
new_rack = Rack(rack, host_list)
rack_list.append(new_rack)
# Validate network in intermediary
_validate_key_in_intermediary_dict('network', intermediary_dict)
# Validate vlan_network_data in intermediary
_validate_key_in_intermediary_dict(
'vlan_network_data', intermediary_dict['network'])
# Validate bgp in intermediary
_validate_key_in_intermediary_dict('bgp', intermediary_dict['network'])
# Pull out network data into Network object
vlan_data_list = []
for network_type, network_data in \
intermediary_dict['network']['vlan_network_data'].items():
vlan_data_list.append(VLANNetworkData(network_type, **network_data))
network = Network(vlan_data_list, bgp=intermediary_dict['network']['bgp'])
# Validate site_info in intermediary
_validate_key_in_intermediary_dict('site_info', intermediary_dict)
# Validate dns in intermediary
_validate_key_in_intermediary_dict('dns', intermediary_dict['site_info'])
# Validate ntp in intermediary
_validate_key_in_intermediary_dict('ntp', intermediary_dict['site_info'])
# Validate region_name in intermediary
_validate_key_in_intermediary_dict('region_name', intermediary_dict)
# Pull out site_info into a SiteInfo object
dns_server_list = ServerList(
intermediary_dict['site_info']['dns']['servers'].split(','))
ntp_server_list = ServerList(
intermediary_dict['site_info']['ntp']['servers'].split(','))
site_info_dict = deepcopy(intermediary_dict['site_info'])
site_info_dict.pop('dns')
site_info_dict.pop('ntp')
site_info_dict['dns'] = dns_server_list
site_info_dict['ntp'] = ntp_server_list
site_info_dict['region_name'] = intermediary_dict['region_name']
site_info = SiteInfo(**site_info_dict)
# Validate storage in intermediary
_validate_key_in_intermediary_dict('storage', intermediary_dict)
# Create and return SiteDocumentData object
site_document_data = SiteDocumentData(
site_info=site_info,
network=network,
baremetal=rack_list,
storage=intermediary_dict['storage'])
return site_document_data

26
spyglass/examples/templates/baremetal/bootactions/promjoin.yaml.j2
View File

@ -1,26 +0,0 @@
---
schema: 'drydock/BootAction/v1'
metadata:
schema: 'metadata/Document/v1'
name: promjoin
storagePolicy: 'cleartext'
layeringDefinition:
abstract: false
layer: site
labels:
application: 'drydock'
data:
signaling: false
assets:
- path: /opt/promjoin.sh
type: file
permissions: '555'
{% raw %}
location: promenade+http://promenade-api.ucp.svc.cluster.local/api/v1.0/join-scripts?design_ref={{ action.design_ref | urlencode }}&hostname={{ node.hostname }}&ip={{ node.network.calico.ip }}{% endif %}{% for k, v in node.labels.items() %}&labels.dynamic={{ k }}={{ v }}{% endfor %}
{% endraw %}
location_pipeline:
- template
data_pipeline:
- utf8_decode
...

54
spyglass/examples/templates/baremetal/nodes.yaml.j2
View File

@ -1,50 +1,46 @@
{% set control_count = [1] %} {% for rack in data.baremetal %}
{% for rack in data['baremetal'].keys() %} {% for host in rack.hosts %}
{% for host in data['baremetal'][rack].keys()%} {% if host.type != 'genesis' %}
{% if data['baremetal'][rack][host]['type'] != 'genesis' %}
--- ---
schema: 'drydock/BaremetalNode/v1' schema: 'drydock/BaremetalNode/v1'
metadata: metadata:
schema: 'metadata/Document/v1' schema: 'metadata/Document/v1'
name: {{ host }} name: {{ host.name }}
layeringDefinition: layeringDefinition:
abstract: false abstract: false
layer: site layer: site
storagePolicy: cleartext storagePolicy: cleartext
data: data:
oob: {% if host.host_profile == 'cp' %}
account: 'root' {% if loop.index - 1 < 4 %}
{% if data['baremetal'][rack][host]['host_profile'] == 'cp' %} host_profile: nc-{{ host.host_profile }}-primary
{% if control_count.append(control_count.pop()+1) %} {% endif %} {% else %}
{% if control_count[0] < 4 %} host_profile: nc-{{ host.host_profile }}-secondary
host_profile: nc-{{data['baremetal'][rack][host]['host_profile']}}-primary {% endif %}
{% else %} {% else %}
host_profile: nc-{{data['baremetal'][rack][host]['host_profile']}}-secondary host_profile: nc-{{ host.host_profile }}
{% endif %} {% endif %}
{% else %}
host_profile: nc-{{data['baremetal'][rack][host]['host_profile']}}
{% endif %}
addressing: addressing:
- network: oob - network: oob
address: {{ data['baremetal'][rack][host]['ip']['oob'] }} address: {{ host.ip.oob }}
- network: oam - network: oam
address: {{ data['baremetal'][rack][host]['ip']['oam'] }} address: {{ host.ip.oam }}
- network: pxe - network: pxe
address: {{ data['baremetal'][rack][host]['ip']['pxe'] }} address: {{ host.ip.pxe }}
- network: storage - network: storage
address: {{ data['baremetal'][rack][host]['ip']['storage'] }} address: {{ host.ip.storage }}
- network: calico - network: calico
address: {{ data['baremetal'][rack][host]['ip']['calico'] }} address: {{ host.ip.calico }}
- network: overlay - network: overlay
address: {{ data['baremetal'][rack][host]['ip']['overlay'] }} address: {{ host.ip.overlay }}
metadata: metadata:
rack: RACK{{rack[-2:] }} rack: {{ rack.name }}
tags: tags:
{% if data['baremetal'][rack][host]['type'] == 'compute' %} {% if host.type == 'compute' %}
- 'workers' - 'workers'
{% else %} {% else %}
- 'masters' - 'masters'
{% endif %} {% endif %}
... ...
{% endif %} {% endif %}
{%endfor%} {%endfor%}

29
spyglass/examples/templates/deployment/deployment-strategy.yaml.j2
View File

@ -11,7 +11,6 @@
schema: shipyard/DeploymentStrategy/v1 schema: shipyard/DeploymentStrategy/v1
metadata: metadata:
schema: metadata/Document/v1 schema: metadata/Document/v1
replacement: true
name: deployment-strategy name: deployment-strategy
layeringDefinition: layeringDefinition:
abstract: false abstract: false
@ -44,13 +43,13 @@ data:
selectors: selectors:
# NEWSITE-CHANGEME: The following should be a list of the computes in the site's first rack # NEWSITE-CHANGEME: The following should be a list of the computes in the site's first rack
- node_names: - node_names:
{% for rack in data['baremetal'].keys() %} {% for rack in data.baremetal %}
{% for host in data['baremetal'][rack].keys()%} {% for host in rack.hosts %}
{% if rack == 'rack03' or rack == 'rack04' %} {% if rack.name == 'rack03' or rack.name == 'rack04' %}
- {{ host }} - {{ host.name }}
{% endif %} {% endif %}
{% endfor %} {% endfor %}
{% endfor %} {% endfor %}
node_labels: [] node_labels: []
node_tags: [] node_tags: []
rack_names: [] rack_names: []
@ -61,13 +60,13 @@ data:
selectors: selectors:
# NEWSITE-CHANGEME: The following should be a list of the computes in the site's second rack # NEWSITE-CHANGEME: The following should be a list of the computes in the site's second rack
- node_names: - node_names:
{% for rack in data['baremetal'].keys() %} {% for rack in data.baremetal %}
{% for host in data['baremetal'][rack].keys()%} {% for host in rack.hosts %}
{% if rack == 'rack05' or rack == 'rack06' %} {% if rack.name == 'rack05' or rack.name == 'rack06' %}
- {{ host }} - {{ host.name }}
{% endif %} {% endif %}
{% endfor %} {% endfor %}
{% endfor %} {% endfor %}
node_labels: [] node_labels: []
node_tags: [] node_tags: []
rack_names: [] rack_names: []

95
spyglass/examples/templates/networks/common_addresses.yaml.j2
View File

@ -6,102 +6,69 @@ metadata:
layeringDefinition: layeringDefinition:
abstract: false abstract: false
layer: site layer: site
parentSelector:
name: common-addresses-global
actions:
- method: replace
path: .dns.upstream_servers
- method: merge
path: .
storagePolicy: cleartext storagePolicy: cleartext
replacement: true
data: data:
calico: calico:
ip_autodetection_method: interface=bond1.{{ data['network']['vlan_network_data']['calico']['vlan']}} ip_autodetection_method: interface=bond1.{{ data.network.get_vlan_data_by_name('calico').vlan }}
etcd: etcd:
service_ip: 10.96.232.136 service_ip: 10.96.232.136
ip_rule: ip_rule:
gateway: {{ data['network']['vlan_network_data']['calico']['gateway']}} gateway: {{ data.network.get_vlan_data_by_name('calico').gateway }}
overlap_cidr: 10.96.0.0/15 overlap_cidr: 10.96.0.0/15
bgp: bgp:
ipv4: ipv4:
public_service_cidr: {{ data['network']['vlan_network_data']['ingress']['subnet'][0] }} public_service_cidr: {{ data.network.get_vlan_data_by_name('ingress').subnet[0] }}
ingress_vip: {{ data['network']['bgp']['ingress_vip'] }} ingress_vip: {{ data.network.bgp['ingress_vip'] }}
peers: peers:
{% for peer in data['network']['bgp']['peers'] %} {% for peer in data.network.bgp['peers'] %}
- {{ peer }} - {{ peer }}
{% endfor %} {% endfor %}
dns: dns:
cluster_domain: cluster.local
service_ip: 10.96.0.10 service_ip: 10.96.0.10
upstream_servers: upstream_servers:
{% for server in (data['site_info']['dns']['servers']).split(',') %} {% for server in data.site_info.dns.servers %}
- {{ server }} - {{ server }}
{% endfor %} {% endfor %}
upstream_servers_joined: {{ data['site_info']['dns']['servers']}} upstream_servers_joined: {{ data.site_info.dns.__str__() }}
ingress_domain: {{ data['site_info']['domain']|lower }} node_domain: {{ data.site_info.domain | lower }}
ingress_domain: {{ data.site_info.domain | lower }}
genesis: genesis:
hostname: {{ (data|get_role_wise_nodes)['genesis']['name'] }} hostname: {{ data.get_baremetal_host_by_type('genesis')[0].name }}
{% for rack in data['baremetal'] %} ip: {{ data.get_baremetal_host_by_type('genesis')[0].ip.calico }}
{% for host in data['baremetal'][rack] %}
{% if data['baremetal'][rack][host]['type'] == 'genesis' %}
ip: {{ data['baremetal'][rack][host]['ip']['calico'] }}
{% endif %}
{% endfor %}
{% endfor %}
bootstrap: bootstrap:
ip: {{ (data|get_role_wise_nodes)['genesis']['pxe'] }} ip: {{ data.get_baremetal_host_by_type('genesis')[0].ip.pxe }}
kubernetes: kubernetes:
api_service_ip: 10.96.0.1 api_service_ip: 10.96.0.1
etcd_service_ip: 10.96.0.2 etcd_service_ip: 10.96.0.2
pod_cidr: 10.97.0.0/16 pod_cidr: 10.97.0.0/16
service_cidr: 10.96.0.0/16 service_cidr: 10.96.0.0/16
# misc k8s port settings
apiserver_port: 6443
haproxy_port: 6553
service_node_port_range: 30000-32767
# etcd port settings
etcd:
container_port: 2379
haproxy_port: 2378
masters: masters:
{% for host in (data|get_role_wise_nodes)['masters'] %} {% for host in data.get_baremetal_host_by_type('controller') %}
- hostname: {{ host }} - hostname: {{ host.name }}
{% endfor %} {% endfor %}
# NEWSITE-CHANGEME: Environment proxy information.
# NOTE: Reference Airship sites do not deploy behind a proxy, so this proxy section
# should be commented out.
# However if you are in a lab that requires proxy, ensure that these proxy
# settings are correct and reachable in your environment; otherwise update
# them with the correct values for your environment.
proxy:
http: ""
https: ""
no_proxy: []
node_ports:
drydock_api: 30000
maas_api: 30001
maas_proxy: 31800 # hardcoded in MAAS
shipyard_api: 30003
airflow_web: 30004
ntp:
servers_joined: {{ data['site_info']['ntp']['servers'] }}
ldap:
base_url: {{ (data['site_info']['ldap']['url']|string).split('//')[1] }}
url: {{ data['site_info']['ldap']['url'] }}
auth_path: DC=test,DC=test,DC=com?sAMAccountName?sub?memberof=CN={{ data['site_info']['ldap']['common_name'] }},OU=Application,OU=Groups,DC=test,DC=test,DC=com
common_name: {{ data['site_info']['ldap']['common_name'] }}
subdomain: {{ data['site_info']['ldap']['subdomain'] }}
domain: {{ (data['site_info']['ldap']['url']|string).split('.')[1] }}
storage: storage:
ceph: ceph:
public_cidr: {{ data['network']['vlan_network_data']['storage']['subnet'] }} public_cidr: {{ data.network.get_vlan_data_by_name('storage').subnet[0] }}
cluster_cidr: {{ data['network']['vlan_network_data']['storage']['subnet'] }} cluster_cidr: {{ data.network.get_vlan_data_by_name('storage').subnet[0] }}
neutron: neutron:
tunnel_device: 'bond1.{{ data['network']['vlan_network_data']['overlay']['vlan'] }}' tunnel_device: "bond1.{{ data.network.get_vlan_data_by_name('overlay').vlan }}"
external_iface: 'bond1' external_iface: "bond1"
openvswitch: openvswitch:
external_iface: 'bond1' external_iface: "bond1"
... ...

216
spyglass/examples/templates/networks/physical/networks.yaml.j2
View File

@ -1,28 +1,4 @@
--- ---
schema: 'drydock/NetworkLink/v1'
metadata:
schema: 'metadata/Document/v1'
name: oob
layeringDefinition:
abstract: false
layer: site
storagePolicy: cleartext
data:
# MaaS doesnt own this network like it does the others, so the noconfig label
# is specified.
labels:
noconfig: enabled
bonding:
mode: disabled
mtu: 1500
linkspeed: auto
trunking:
mode: disabled
default_network: oob
allowed_networks:
- oob
...
---
schema: 'drydock/Network/v1' schema: 'drydock/Network/v1'
metadata: metadata:
schema: 'metadata/Document/v1' schema: 'metadata/Document/v1'
@ -30,42 +6,33 @@ metadata:
layeringDefinition: layeringDefinition:
abstract: false abstract: false
layer: site layer: site
parentSelector:
network_role: oob
topology: cruiser
actions:
- method: merge
path: .
storagePolicy: cleartext storagePolicy: cleartext
data: data:
cidr: {{ data['network']['vlan_network_data']['oob']['subnet'] }} cidr: {{ data.network.get_vlan_data_by_name('oob').subnet[0] }}
routes: routes:
- subnet: '0.0.0.0/0' - subnet: '0.0.0.0/0'
gateway: {{ data['network']['vlan_network_data']['oob']['gateway'] }} gateway: {{ data.network.get_vlan_data_by_name('oob').gateway }}
metric: 100 metric: 100
ranges: ranges:
- type: reserved
start: {{ data.network.get_vlan_data_by_name('oob').reserved_start }}
end: {{ data.network.get_vlan_data_by_name('oob').reserved_end }}
- type: static - type: static
start: {{ data['network']['vlan_network_data']['oob']['static_start'] }} start: {{ data.network.get_vlan_data_by_name('oob').static_start }}
end: {{ data['network']['vlan_network_data']['oob']['static_end'] }} end: {{ data.network.get_vlan_data_by_name('oob').static_end }}
- type: dhcp
start: {{ data.network.get_vlan_data_by_name('oob').dhcp_start }}
end: {{ data.network.get_vlan_data_by_name('oob').dhcp_end }}
... ...
--- ---
schema: 'drydock/NetworkLink/v1' schema: 'drydock/NetworkLink/v1'
metadata:
schema: 'metadata/Document/v1'
name: pxe
layeringDefinition:
abstract: false
layer: site
storagePolicy: cleartext
data:
bonding:
mode: disabled
mtu: 1500
linkspeed: auto
trunking:
mode: disabled
default_network: pxe
allowed_networks:
- pxe
...
---
schema: 'drydock/Network/v1'
metadata: metadata:
schema: 'metadata/Document/v1' schema: 'metadata/Document/v1'
name: pxe name: pxe
@ -80,56 +47,21 @@ metadata:
path: . path: .
storagePolicy: cleartext storagePolicy: cleartext
data: data:
cidr: {{ data['network']['vlan_network_data']['pxe']['subnet'] }} cidr: {{ data.network.get_vlan_data_by_name('pxe').subnet[0] }}
routes: routes:
{% for other_subnet in data['network']['vlan_network_data']['pxe']['routes'] %} - subnet: '0.0.0.0/0'
- subnet: {{ other_subnet }} gateway: {{ data.network.get_vlan_data_by_name('pxe').gateway }}
gateway: {{ data['network']['vlan_network_data']['pxe']['gateway'] }}
metric: 100 metric: 100
{% endfor %}
ranges: ranges:
- type: reserved - type: reserved
start: {{ data['network']['vlan_network_data']['pxe']['reserved_start'] }} start: {{ data.network.get_vlan_data_by_name('pxe').reserved_start }}
end: {{ data['network']['vlan_network_data']['pxe']['reserved_end'] }} end: {{ data.network.get_vlan_data_by_name('pxe').reserved_end }}
- type: static - type: static
start: {{ data['network']['vlan_network_data']['pxe']['static_start'] }} start: {{ data.network.get_vlan_data_by_name('pxe').static_start }}
end: {{ data['network']['vlan_network_data']['pxe']['static_end'] }} end: {{ data.network.get_vlan_data_by_name('pxe').static_end }}
- type: dhcp - type: dhcp
start: {{ data['network']['vlan_network_data']['pxe']['dhcp_start'] }} start: {{ data.network.get_vlan_data_by_name('pxe').dhcp_start }}
end: {{ data['network']['vlan_network_data']['pxe']['dhcp_end'] }} end: {{ data.network.get_vlan_data_by_name('pxe').dhcp_end }}
...
---
schema: 'drydock/NetworkLink/v1'
metadata:
schema: 'metadata/Document/v1'
name: data
layeringDefinition:
abstract: false
layer: site
storagePolicy: cleartext
data:
bonding:
mode: 802.3ad
hash: layer3+4
peer_rate: fast
mon_rate: 100
up_delay: 1000
down_delay: 3000
# NEWSITE-CHANGEME: Ensure the network switches in the environment are
# configured for this MTU or greater. Even if switches are configured for or
# can support a slightly higher MTU, there is no need (and negliable benefit)
# to squeeze every last byte into the MTU (e.g., 9216 vs 9100). Leave MTU at
# 9100 for maximum compatibility.
mtu: 9100
linkspeed: auto
trunking:
mode: 802.1q
allowed_networks:
- oam
- storage
- overlay
- calico
... ...
--- ---
@ -148,27 +80,24 @@ metadata:
path: . path: .
storagePolicy: cleartext storagePolicy: cleartext
data: data:
cidr: {{ data['network']['vlan_network_data']['oam']['subnet'] }} cidr: {{ data.network.get_vlan_data_by_name('oam').subnet[0] }}
{% set flag = [0] %} {% if data.network.get_vlan_data_by_name('oam').routes %}
{% for route in data['network']['vlan_network_data']['oam']['routes'] %}
{% if flag[0] == 0 %}
routes: routes:
{% endif %} {% for route in data.network.get_vlan_data_by_name('oam').routes %}
{% if flag.append(flag.pop() + 1) %} {% endif %}
- subnet: {{ route }} - subnet: {{ route }}
gateway: {{ data['network']['vlan_network_data']['oam']['gateway'] }} gateway: {{ data.network.get_vlan_data_by_name('oam').gateway }}
metric: 100 metric: 100
{% endfor %} {% endfor %}
{% if flag[0] == 0 %} {% else %}
routes:[] routes: []
{% endif %} {% endif %}
ranges: ranges:
- type: reserved - type: reserved
start: {{ data['network']['vlan_network_data']['oam']['reserved_start'] }} start: {{ data.network.get_vlan_data_by_name('oam').reserved_start }}
end: {{ data['network']['vlan_network_data']['oam']['reserved_end'] }} end: {{ data.network.get_vlan_data_by_name('oam').reserved_end }}
- type: static - type: static
start: {{ data['network']['vlan_network_data']['oam']['static_start'] }} start: {{ data.network.get_vlan_data_by_name('oam').static_start }}
end: {{ data['network']['vlan_network_data']['oam']['static_end'] }} end: {{ data.network.get_vlan_data_by_name('oam').static_end }}
... ...
--- ---
@ -187,40 +116,14 @@ metadata:
path: . path: .
storagePolicy: cleartext storagePolicy: cleartext
data: data:
cidr: {{ data['network']['vlan_network_data']['storage']['subnet'] }} cidr: {{ data.network.get_vlan_data_by_name('storage').subnet[0] }}
ranges: ranges:
- type: reserved - type: reserved
start: {{ data['network']['vlan_network_data']['storage']['reserved_start'] }} start: {{ data.network.get_vlan_data_by_name('storage').reserved_start }}
end: {{ data['network']['vlan_network_data']['storage']['reserved_end'] }} end: {{ data.network.get_vlan_data_by_name('storage').reserved_end }}
- type: static - type: static
start: {{ data['network']['vlan_network_data']['storage']['static_start'] }} start: {{ data.network.get_vlan_data_by_name('storage').static_start }}
end: {{ data['network']['vlan_network_data']['storage']['static_end'] }} end: {{ data.network.get_vlan_data_by_name('storage').static_end }}
...
---
schema: 'drydock/Network/v1'
metadata:
schema: 'metadata/Document/v1'
name: overlay
layeringDefinition:
abstract: false
layer: site
parentSelector:
network_role: os-overlay
topology: cruiser
actions:
- method: merge
path: .
storagePolicy: cleartext
data:
cidr: {{ data['network']['vlan_network_data']['overlay']['subnet'] }}
ranges:
- type: reserved
start: {{ data['network']['vlan_network_data']['overlay']['reserved_start'] }}
end: {{ data['network']['vlan_network_data']['overlay']['reserved_end'] }}
- type: static
start: {{ data['network']['vlan_network_data']['overlay']['static_start'] }}
end: {{ data['network']['vlan_network_data']['overlay']['static_end'] }}
... ...
--- ---
@ -239,13 +142,38 @@ metadata:
path: . path: .
storagePolicy: cleartext storagePolicy: cleartext
data: data:
cidr: {{ data['network']['vlan_network_data']['calico']['subnet'] }} cidr: {{ data.network.get_vlan_data_by_name('calico').subnet[0] }}
ranges: ranges:
- type: reserved - type: reserved
start: {{ data['network']['vlan_network_data']['calico']['reserved_start'] }} start: {{ data.network.get_vlan_data_by_name('calico').reserved_start }}
end: {{ data['network']['vlan_network_data']['calico']['reserved_end'] }} end: {{ data.network.get_vlan_data_by_name('calico').reserved_end }}
- type: static - type: static
start: {{ data['network']['vlan_network_data']['calico']['static_start'] }} start: {{ data.network.get_vlan_data_by_name('calico').static_start }}
end: {{ data['network']['vlan_network_data']['calico']['static_end'] }} end: {{ data.network.get_vlan_data_by_name('calico').static_end }}
... ...
---
schema: 'drydock/Network/v1'
metadata:
schema: 'metadata/Document/v1'
name: overlay
layeringDefinition:
abstract: false
layer: site
parentSelector:
network_role: os-overlay
topology: cruiser
actions:
- method: merge
path: .
storagePolicy: cleartext
data:
cidr: {{ data.network.get_vlan_data_by_name('overlay').subnet[0] }}
ranges:
- type: reserved
start: {{ data.network.get_vlan_data_by_name('overlay').reserved_start }}
end: {{ data.network.get_vlan_data_by_name('overlay').reserved_end }}
- type: static
start: {{ data.network.get_vlan_data_by_name('overlay').static_start }}
end: {{ data.network.get_vlan_data_by_name('overlay').static_end }}
...

145
spyglass/examples/templates/pki/pki-catalogue.yaml.j2
View File

@ -21,33 +21,29 @@ data:
- 10.96.0.1 - 10.96.0.1
kubernetes_service_names: kubernetes_service_names:
- kubernetes.default.svc.cluster.local - kubernetes.default.svc.cluster.local
{% for racks in data['baremetal'].keys()%} {% for host in data.get_baremetal_host_by_type('genesis') %}
{% for host in data['baremetal'][racks].keys()%}
{% if data['baremetal'][racks][host]['type'] == 'genesis' %}
- document_name: kubelet-genesis - document_name: kubelet-genesis
common_name: system:node:{{ host }} common_name: system:node:{{ host.name }}
hosts: hosts:
- {{ host }} - {{ host.name }}
- {{ data['baremetal'][racks][host]['ip']['oam'] }} - {{ host.ip.oam }}
- {{ data['baremetal'][racks][host]['ip']['calico']}} - {{ host.ip.calico }}
groups: groups:
- system:nodes - system:nodes
{% endif %} {% endfor %}
{%endfor%}
{%endfor%} {% for rack in data.baremetal %}
{% for racks in data['baremetal'].keys()%} {% for host in rack.hosts %}
{% for host in data['baremetal'][racks].keys()%} - document_name: kubelet-{{ host.name }}
- document_name: kubelet-{{ host }} common_name: system:node:{{ host.name }}
common_name: system:node:{{ host }}
hosts: hosts:
- {{ host }} - {{ host.name }}
- {{ data['baremetal'][racks][host]['ip']['oam'] }} - {{ host.ip.oam }}
- {{ data['baremetal'][racks][host]['ip']['calico']}} - {{ host.ip.calico }}
groups: groups:
- system:nodes - system:nodes
{%endfor%} {% endfor %}
{%endfor%} {% endfor %}
- document_name: scheduler - document_name: scheduler
description: Service certificate for Kubernetes scheduler description: Service certificate for Kubernetes scheduler
common_name: system:kube-scheduler common_name: system:kube-scheduler
@ -62,6 +58,7 @@ data:
common_name: armada common_name: armada
groups: groups:
- system:masters - system:masters
kubernetes-etcd: kubernetes-etcd:
description: Certificates for Kubernetes's etcd servers description: Certificates for Kubernetes's etcd servers
certificates: certificates:
@ -72,113 +69,93 @@ data:
- document_name: kubernetes-etcd-anchor - document_name: kubernetes-etcd-anchor
description: anchor description: anchor
common_name: anchor common_name: anchor
{% for racks in data['baremetal'].keys()%} {% for host in data.get_baremetal_host_by_type('genesis') %}
{% for host in data['baremetal'][racks].keys()%}
{% if data['baremetal'][racks][host]['type'] == 'genesis' %}
- document_name: kubernetes-etcd-genesis - document_name: kubernetes-etcd-genesis
common_name: kubernetes-etcd-genesis common_name: kubernetes-etcd-genesis
hosts: hosts:
- {{ host }} - {{ host.name }}
- {{ data['baremetal'][racks][host]['ip']['oam'] }} - {{ host.ip.oam }}
- {{ data['baremetal'][racks][host]['ip']['calico']}} - {{ host.ip.calico }}
- 127.0.0.1 - 127.0.0.1
- localhost - localhost
- kubernetes-etcd.kube-system.svc.cluster.local - kubernetes-etcd.kube-system.svc.cluster.local
- 10.96.0.2 - 10.96.0.2
{% endif %} {% endfor %}
{%endfor%}
{%endfor%} {% for host in data.get_baremetal_host_by_type('controller', 'genesis') %}
{% for racks in data['baremetal'].keys()%} - document_name: kubernetes-etcd-{{ host.name }}
{% for host in data['baremetal'][racks].keys()%} common_name: kubernetes-etcd-{{ host.name }}
{% if data['baremetal'][racks][host]['type'] == 'controller' or data['baremetal'][racks][host]['type'] == 'genesis'%}
- document_name: kubernetes-etcd-{{ host }}
common_name: kubernetes-etcd-{{ host }}
hosts: hosts:
- {{ host }} - {{ host.name }}
- {{ data['baremetal'][racks][host]['ip']['oam'] }} - {{ host.ip.oam }}
- {{ data['baremetal'][racks][host]['ip']['calico']}} - {{ host.ip.calico }}
- 127.0.0.1 - 127.0.0.1
- localhost - localhost
- kubernetes-etcd.kube-system.svc.cluster.local - kubernetes-etcd.kube-system.svc.cluster.local
- 10.96.0.2 - 10.96.0.2
{% endif %} {% endfor %}
{%endfor%}
{%endfor%}
{% for racks in data['baremetal'].keys()%}
{% for host in data['baremetal'][racks].keys()%}
{% if data['baremetal'][racks][host]['type'] == 'genesis' %}
kubernetes-etcd-peer: kubernetes-etcd-peer:
certificates: certificates:
{% for host in data.get_baremetal_host_by_type('genesis') %}
- document_name: kubernetes-etcd-genesis-peer - document_name: kubernetes-etcd-genesis-peer
common_name: kubernetes-etcd-genesis-peer common_name: kubernetes-etcd-genesis-peer
hosts: hosts:
- {{ host }} - {{ host.name }}
- {{ data['baremetal'][racks][host]['ip']['oam'] }} - {{ host.ip.oam }}
- {{ data['baremetal'][racks][host]['ip']['calico']}} - {{ host.ip.calico }}
- 127.0.0.1 - 127.0.0.1
- localhost - localhost
- kubernetes-etcd.kube-system.svc.cluster.local - kubernetes-etcd.kube-system.svc.cluster.local
- 10.96.0.2 - 10.96.0.2
{% endif %} {% endfor %}
{%endfor%}
{%endfor%} {% for host in data.get_baremetal_host_by_type('controller', 'genesis') %}
{% for racks in data['baremetal'].keys()%} - document_name: kubernetes-etcd-{{ host.name }}-peer
{% for host in data['baremetal'][racks].keys()%} common_name: kubernetes-etcd-{{ host.name }}-peer
{% if data['baremetal'][racks][host]['type'] == 'controller' or data['baremetal'][racks][host]['type'] == 'genesis' %}
- document_name: kubernetes-etcd-{{ host }}-peer
common_name: kubernetes-etcd-{{ host }}-peer
hosts: hosts:
- {{ host }} - {{ host.name }}
- {{ data['baremetal'][racks][host]['ip']['oam'] }} - {{ host.ip.oam }}
- {{ data['baremetal'][racks][host]['ip']['calico']}} - {{ host.ip.calico }}
- 127.0.0.1 - 127.0.0.1
- localhost - localhost
- kubernetes-etcd.kube-system.svc.cluster.local - kubernetes-etcd.kube-system.svc.cluster.local
- 10.96.0.2 - 10.96.0.2
{% endif %} {% endfor %}
{%endfor%}
{%endfor%} calico-etcd:
ksn-etcd:
description: Certificates for Calico etcd client traffic description: Certificates for Calico etcd client traffic
certificates: certificates:
- document_name: ksn-etcd-anchor - document_name: ksn-etcd-anchor
description: anchor description: anchor
common_name: anchor common_name: anchor
{% for racks in data['baremetal'].keys()%} {% for host in data.get_baremetal_host_by_type('controller', 'genesis') %}
{% for host in data['baremetal'][racks].keys()%} - document_name: ksn-etcd-{{ host.name }}
{% if data['baremetal'][racks][host]['type'] == 'controller' or data['baremetal'][racks][host]['type'] == 'genesis' %} common_name: ksn-etcd-{{ host.name }}
- document_name: ksn-etcd-{{ host }}
common_name: ksn-etcd-{{ host }}
hosts: hosts:
- {{ host }} - {{ host.name }}
- {{ data['baremetal'][racks][host]['ip']['oam'] }} - {{ host.ip.oam }}
- {{ data['baremetal'][racks][host]['ip']['calico']}} - {{ host.ip.calico }}
- 127.0.0.1 - 127.0.0.1
- localhost - localhost
- 10.96.232.136 - 10.96.232.136
{% endif %} {% endfor %}
{%endfor%}
{%endfor%}
- document_name: ksn-node - document_name: ksn-node
common_name: calcico-node common_name: calcico-node
ksn-etcd-peer: calico-etcd-peer:
description: Certificates for Calico etcd clients description: Certificates for Calico etcd clients
certificates: certificates:
{% for racks in data['baremetal'].keys()%} {% for host in data.get_baremetal_host_by_type('controller', 'genesis') %}
{% for host in data['baremetal'][racks].keys()%} - document_name: ksn-etcd-{{ host.name }}-peer
{% if data['baremetal'][racks][host]['type'] == 'controller' or data['baremetal'][racks][host]['type'] == 'genesis' %} common_name: ksn-etcd-{{ host.name }}-peer
- document_name: ksn-etcd-{{ host }}-peer
common_name: ksn-etcd-{{ host }}-peer
hosts: hosts:
- {{ host }} - {{ host.name }}
- {{ data['baremetal'][racks][host]['ip']['oam'] }} - {{ host.ip.oam }}
- {{ data['baremetal'][racks][host]['ip']['calico']}} - {{ host.ip.calico }}
- 127.0.0.1 - 127.0.0.1
- localhost - localhost
- 10.96.232.136 - 10.96.232.136
{% endif %} {% endfor %}
{%endfor%}
{%endfor%}
- document_name: ksn-node-peer - document_name: ksn-node-peer
common_name: calico-node-peer common_name: calico-node-peer
keypairs: keypairs:

4
spyglass/examples/templates/profile/region.yaml.j2
View File

@ -2,7 +2,7 @@
schema: 'drydock/Region/v1' schema: 'drydock/Region/v1'
metadata: metadata:
schema: 'metadata/Document/v1' schema: 'metadata/Document/v1'
name: {{ data['region_name'] }} name: {{ data.site_info.region_name }}
layeringDefinition: layeringDefinition:
abstract: false abstract: false
layer: site layer: site
@ -18,7 +18,7 @@ metadata:
path: .authorized_keys[1] path: .authorized_keys[1]
src: src:
schema: deckhand/PublicKey/v1 schema: deckhand/PublicKey/v1
name: {{ data['region_name'] }}_ssh_public_key name: {{ data.site_info.region_name }}_ssh_public_key
path: . path: .
- dest: - dest:
path: .repositories.main_archive path: .repositories.main_archive

4
spyglass/examples/templates/site-definition.yaml.j2
View File

@ -7,11 +7,11 @@ metadata:
abstract: false abstract: false
layer: site layer: site
# NEWSITE-CHANGEME: Replace with the site name # NEWSITE-CHANGEME: Replace with the site name
name: {{ data['region_name'] }} name: {{ data.site_info.region_name }}
storagePolicy: cleartext storagePolicy: cleartext
data: data:
# The type layer this site will delpoy with. Type layer is found in the # The type layer this site will delpoy with. Type layer is found in the
# aic-clcp-manifests repo. # aic-clcp-manifests repo.
site_type: {{ data['site_info']['sitetype'] }} site_type: {{ data.site_info.sitetype }}
... ...

55
spyglass/examples/templates/software/charts/kubernetes/etcd.yaml.j2
View File

@ -13,83 +13,70 @@ metadata:
path: . path: .
storagePolicy: cleartext storagePolicy: cleartext
substitutions: substitutions:
{% set count = [0] %} {% for host in data.get_baremetal_host_by_type('controller') %}
{% for rack in data.baremetal.keys() %}
{% for host in data["baremetal"][rack] %}
{% if data["baremetal"][rack][host]["type"] == 'controller' %}
- src: - src:
schema: pegleg/CommonAddresses/v1 schema: pegleg/CommonAddresses/v1
name: common-addresses name: common-addresses
path: .masters[{{ count[0] }}].hostname path: .masters[{{ loop.index - 1 }}].hostname
dest: dest:
path: .values.nodes[{{ count[0] }}].name path: .values.nodes[{{ loop.index - 1 }}].name
- src: - src:
schema: deckhand/Certificate/v1 schema: deckhand/Certificate/v1
name: calico-etcd-{{ host }} name: calico-etcd-{{ host.name }}
path: . path: .
dest: dest:
path: .values.nodes[{{ count[0] }}].tls.client.cert path: .values.nodes[{{ loop.index - 1 }}].tls.client.cert
- src: - src:
schema: deckhand/CertificateKey/v1 schema: deckhand/CertificateKey/v1
name: calico-etcd-{{ host }} name: calico-etcd-{{ host.name }}
path: . path: .
dest: dest:
path: .values.nodes[{{ count[0] }}].tls.client.key path: .values.nodes[{{ loop.index - 1 }}].tls.client.key
- src: - src:
schema: deckhand/Certificate/v1 schema: deckhand/Certificate/v1
name: calico-etcd-{{ host }} name: calico-etcd-{{ host.name }}
path: . path: .
dest: dest:
path: .values.nodes[{{ count[0] }}].tls.peer.cert path: .values.nodes[{{ loop.index - 1 }}].tls.peer.cert
- src: - src:
schema: deckhand/CertificateKey/v1 schema: deckhand/CertificateKey/v1
name: calico-etcd-{{ host }}-peer name: calico-etcd-{{ host.name }}-peer
path: . path: .
dest: dest:
path: .values.nodes[{{ count[0] }}].tls.peer.key path: .values.nodes[{{ loop.index - 1 }}].tls.peer.key
{% if count.append(count.pop() + 1) %}{% endif %} {# increment count by 1 #}
{% endif %}
{% endfor %}
{% endfor %} {% endfor %}
{% for rack in data.baremetal.keys() %} {% for host in data.get_baremetal_host_by_type('genesis') %}
{% for host in data["baremetal"][rack] %}
{% if data["baremetal"][rack][host]["type"] == 'genesis' %}
- src: - src:
schema: pegleg/CommonAddresses/v1 schema: pegleg/CommonAddresses/v1
name: common-addresses name: common-addresses
path: .genesis.hostname path: .genesis.hostname
dest: dest:
path: .values.nodes[{{ count[0] }}].name path: .values.nodes[{{ loop.index - 1 }}].name
- src: - src:
schema: deckhand/Certificate/v1 schema: deckhand/Certificate/v1
name: calico-etcd-{{ host }} name: calico-etcd-{{ host.name }}
path: . path: .
dest: dest:
path: .values.nodes[{{ count[0] }}].tls.client.cert path: .values.nodes[{{ loop.index - 1 }}].tls.client.cert
- src: - src:
schema: deckhand/CertificateKey/v1 schema: deckhand/CertificateKey/v1
name: calico-etcd-{{ host }} name: calico-etcd-{{ host.name }}
path: . path: .
dest: dest:
path: .values.nodes[{{ count[0] }}].tls.client.key path: .values.nodes[{{ loop.index - 1 }}].tls.client.key
- src: - src:
schema: deckhand/Certificate/v1 schema: deckhand/Certificate/v1
name: calico-etcd-{{ host }}-peer name: calico-etcd-{{ host.name }}-peer
path: . path: .
dest: dest:
path: .values.nodes[{{ count[0] }}].tls.peer.cert path: .values.nodes[{{ loop.index - 1 }}].tls.peer.cert
- src: - src:
schema: deckhand/CertificateKey/v1 schema: deckhand/CertificateKey/v1
name: calico-etcd-{{ host }}-peer name: calico-etcd-{{ host.name }}-peer
path: . path: .
dest: dest:
path: .values.nodes[{{ count[0] }}].tls.peer.key path: .values.nodes[{{ loop.index - 1 }}].tls.peer.key
{% endif %}
{% endfor %}
{% endfor %} {% endfor %}
data: {} data: {}

48
spyglass/examples/templates/software/charts/kubernetes/etcd/etcd.yaml.j2
View File

@ -13,80 +13,70 @@ metadata:
path: . path: .
storagePolicy: cleartext storagePolicy: cleartext
substitutions: substitutions:
{% set count = [0] %} {% for host in data.get_baremetal_host_by_type('controller') %}
{% for rack in data.baremetal.keys() %}
{% for host in data["baremetal"][rack] %}
{% if data["baremetal"][rack][host]["type"] == 'controller' %}
- src: - src:
schema: pegleg/CommonAddresses/v1 schema: pegleg/CommonAddresses/v1
name: common-addresses name: common-addresses
path: .masters[{{ count[0] }}].hostname path: .masters[{{ loop.index - 1 }}].hostname
dest: dest:
path: .values.nodes[{{ count[0] }}].name path: .values.nodes[{{ loop.index - 1 }}].name
- src: - src:
schema: deckhand/Certificate/v1 schema: deckhand/Certificate/v1
name: kubernetes-etcd-{{ host }} name: kubernetes-etcd-{{ host.name }}
path: . path: .
dest: dest:
path: .values.nodes[{{ count[0] }}].tls.client.cert path: .values.nodes[{{ loop.index - 1 }}].tls.client.cert
- src: - src:
schema: deckhand/CertificateKey/v1 schema: deckhand/CertificateKey/v1
name: kubernetes-etcd-{{ host }} name: kubernetes-etcd-{{ host.name }}
path: . path: .
dest: dest:
path: .values.nodes[{{ count[0] }}].tls.client.key path: .values.nodes[{{ loop.index - 1 }}].tls.client.key
- src: - src:
schema: deckhand/Certificate/v1 schema: deckhand/Certificate/v1
name: kubernetes-etcd-{{ host }}-peer name: kubernetes-etcd-{{ host.name }}-peer
path: . path: .
dest: dest:
path: .values.nodes[{{ count[0] }}].tls.peer.cert path: .values.nodes[{{ loop.index - 1 }}].tls.peer.cert
- src: - src:
schema: deckhand/CertificateKey/v1 schema: deckhand/CertificateKey/v1
name: kubernetes-etcd-{{ host }}-peer name: kubernetes-etcd-{{ host.name }}-peer
path: . path: .
dest: dest:
path: .values.nodes[{{ count[0] }}].tls.peer.key path: .values.nodes[{{ loop.index - 1 }}].tls.peer.key
{% if count.append(count.pop() + 1) %}{% endif %} {# increment count by 1 #} {% endfor %}
{% endif %} {% for host in data.get_baremetal_host_by_type('genesis') %}
{% endfor %}
{% endfor %}
{% for rack in data.baremetal.keys() %}
{% for host in data["baremetal"][rack] %}
{% if data["baremetal"][rack][host]["type"] == 'genesis' %}
- src: - src:
schema: pegleg/CommonAddresses/v1 schema: pegleg/CommonAddresses/v1
name: common-addresses name: common-addresses
path: .genesis.hostname path: .genesis.hostname
dest: dest:
path: .values.nodes[{{ count[0] }}].name path: .values.nodes[{{ loop.index - 1 }}].name
- src: - src:
schema: deckhand/Certificate/v1 schema: deckhand/Certificate/v1
name: kubernetes-etcd-genesis name: kubernetes-etcd-genesis
path: . path: .
dest: dest:
path: .values.nodes[{{ count[0] }}].tls.client.cert path: .values.nodes[{{ loop.index - 1 }}].tls.client.cert
- src: - src:
schema: deckhand/CertificateKey/v1 schema: deckhand/CertificateKey/v1
name: kubernetes-etcd-genesis name: kubernetes-etcd-genesis
path: . path: .
dest: dest:
path: .values.nodes[{{ count[0] }}].tls.client.key path: .values.nodes[{{ loop.index - 1 }}].tls.client.key
- src: - src:
schema: deckhand/Certificate/v1 schema: deckhand/Certificate/v1
name: kubernetes-etcd-genesis-peer name: kubernetes-etcd-genesis-peer
path: . path: .
dest: dest:
path: .values.nodes[{{ count[0] }}].tls.peer.cert path: .values.nodes[{{ loop.index - 1 }}].tls.peer.cert
- src: - src:
schema: deckhand/CertificateKey/v1 schema: deckhand/CertificateKey/v1
name: kubernetes-etcd-genesis-peer name: kubernetes-etcd-genesis-peer
path: $ path: $
dest: dest:
path: .values.nodes[{{ count[0] }}].tls.peer.key path: .values.nodes[{{ loop.index - 1 }}].tls.peer.key
{% endif %} {% endfor %}
{% endfor %}
{% endfor %}
data: {} data: {}
... ...

2
spyglass/examples/templates/software/charts/osh/openstack-tenant-ceph/ceph-client.yaml.j2
View File

@ -18,5 +18,5 @@ data:
conf: conf:
pool: pool:
target: target:
osd: {{ data['storage']['ceph']['controller']['osd_count'] }} osd: {{ data.storage['ceph']['controller']['osd_count'] }}
... ...

3
spyglass/examples/templates/software/config/common-software-config.yaml.j2
View File

@ -13,4 +13,5 @@ metadata:
data: data:
osh: osh:
# NEWSITE-CHANGEME: Replace with the site name # NEWSITE-CHANGEME: Replace with the site name
region_name: {{ data['region_name'] }} region_name: {{ data.site_info.region_name }}
...

19
spyglass/site_processors/base.py
View File

@ -20,22 +20,3 @@ class BaseProcessor(object):
def render_template(self, template): def render_template(self, template):
pass pass
@staticmethod
def get_role_wise_nodes(yaml_data):
hosts = {"genesis": {}, "masters": [], "workers": []}
for rack in yaml_data["baremetal"]:
for host in yaml_data["baremetal"][rack]:
if yaml_data["baremetal"][rack][host]["type"] == "genesis":
hosts["genesis"] = {
"name": host,
"pxe": yaml_data["baremetal"][rack][host]["ip"]["pxe"],
"oam": yaml_data["baremetal"][rack][host]["ip"]["oam"],
}
elif yaml_data["baremetal"][rack][host]["type"] \
== "controller":
hosts["masters"].append(host)
else:
hosts["workers"].append(host)
return hosts

19
spyglass/site_processors/site_processor.py
View File

@ -18,6 +18,8 @@ import shutil
import jinja2 import jinja2
from spyglass.data_extractor.models import site_document_data_factory
from spyglass.data_extractor.models import SiteDocumentData
from spyglass.site_processors.base import BaseProcessor from spyglass.site_processors.base import BaseProcessor
LOG = logging.getLogger(__name__) LOG = logging.getLogger(__name__)
@ -25,9 +27,12 @@ LOG = logging.getLogger(__name__)
class SiteProcessor(BaseProcessor): class SiteProcessor(BaseProcessor):
def __init__(self, intermediary_yaml, manifest_dir, force_write): def __init__(self, site_data, manifest_dir, force_write):
super().__init__() super().__init__()
self.yaml_data = intermediary_yaml if isinstance(site_data, SiteDocumentData):
self.site_data = site_data
else:
self.site_data = site_document_data_factory(site_data)
self.manifest_dir = manifest_dir self.manifest_dir = manifest_dir
self.force_write = force_write self.force_write = force_write
@ -38,7 +43,7 @@ class SiteProcessor(BaseProcessor):
calico) are generated in a single file. Rack specific calico) are generated in a single file. Rack specific
configs( pxe and oob) are generated per rack. configs( pxe and oob) are generated per rack.
""" """
# Check of manifest_dir exists # Check if manifest_dir exists
if self.manifest_dir is not None: if self.manifest_dir is not None:
site_manifest_dir = os.path.join( site_manifest_dir = os.path.join(
self.manifest_dir, 'pegleg_manifests', 'site') self.manifest_dir, 'pegleg_manifests', 'site')
@ -66,16 +71,16 @@ class SiteProcessor(BaseProcessor):
autoescape=True, autoescape=True,
loader=loader, loader=loader,
trim_blocks=True, trim_blocks=True,
lstrip_blocks=True,
undefined=logging_undefined) undefined=logging_undefined)
j2_env.filters["get_role_wise_nodes"] = \
self.get_role_wise_nodes
templatefile = os.path.join(dirpath, filename) templatefile = os.path.join(dirpath, filename)
LOG.debug("Template file: %s", templatefile) LOG.debug("Template file: %s", templatefile)
outdirs = dirpath.split(template_folder_name)[1].lstrip(os.sep) outdirs = dirpath.split(template_folder_name)[1].lstrip(os.sep)
LOG.debug("outdirs: %s", outdirs) LOG.debug("outdirs: %s", outdirs)
outfile_path = os.path.join( outfile_path = os.path.join(
site_manifest_dir, self.yaml_data["region_name"], outdirs) site_manifest_dir, self.site_data.site_info.region_name,
outdirs)
LOG.debug("outfile path: %s", outfile_path) LOG.debug("outfile path: %s", outfile_path)
outfile_yaml = os.path.split(templatefile)[1] outfile_yaml = os.path.split(templatefile)[1]
outfile_yaml = os.path.splitext(outfile_yaml)[0] outfile_yaml = os.path.splitext(outfile_yaml)[0]
@ -90,7 +95,7 @@ class SiteProcessor(BaseProcessor):
out = open(outfile, "w") out = open(outfile, "w")
created_file_list.append(outfile) created_file_list.append(outfile)
LOG.info("Rendering {}".format(outfile_yaml)) LOG.info("Rendering {}".format(outfile_yaml))
rendered = template_j2.render(data=self.yaml_data) rendered = template_j2.render(data=self.site_data)
out.write(rendered) out.write(rendered)
out.close() out.close()
except IOError as ioe: except IOError as ioe:

187
tests/unit/data_extractor/test_models.py
View File

@ -13,11 +13,18 @@
# limitations under the License. # limitations under the License.
from copy import copy from copy import copy
import os
import unittest import unittest
from unittest import mock from unittest import mock
import yaml
from spyglass.data_extractor.custom_exceptions import InvalidIntermediary
from spyglass.data_extractor import models from spyglass.data_extractor import models
FIXTURE_DIR = os.path.join(
os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'shared')
class TestParseIp(unittest.TestCase): class TestParseIp(unittest.TestCase):
"""Tests the _parse_ip validator for Spyglass models""" """Tests the _parse_ip validator for Spyglass models"""
@ -340,11 +347,7 @@ class TestRack(unittest.TestCase):
"""Tests for the Rack model""" """Tests for the Rack model"""
RACK_NAME = 'test_rack1' RACK_NAME = 'test_rack1'
HOST_DATA = { HOST_DATA = {'rack_name': RACK_NAME, 'host_profile': 'host'}
'rack_name': RACK_NAME,
'host_profile': 'host',
'type': 'compute'
}
@mock.patch('spyglass.data_extractor.models.IPList', autospec=True) @mock.patch('spyglass.data_extractor.models.IPList', autospec=True)
def setUp(self, MockIPList): def setUp(self, MockIPList):
@ -353,9 +356,9 @@ class TestRack(unittest.TestCase):
self.HOST_DATA['ip'] = MockIPList() self.HOST_DATA['ip'] = MockIPList()
self.HOST_DATA['ip'].dict_from_class.return_value = 'success' self.HOST_DATA['ip'].dict_from_class.return_value = 'success'
self.hosts = [ self.hosts = [
models.Host('test_host1', **self.HOST_DATA), models.Host('test_host1', **self.HOST_DATA, type='genesis'),
models.Host('test_host2', **self.HOST_DATA), models.Host('test_host2', **self.HOST_DATA, type='compute'),
models.Host('test_host3', **self.HOST_DATA), models.Host('test_host3', **self.HOST_DATA, type='controller'),
] ]
def test___init__(self): def test___init__(self):
@ -408,6 +411,15 @@ class TestRack(unittest.TestCase):
self.assertEqual( self.assertEqual(
self.hosts[1], result.get_host_by_name(self.hosts[1].name)) self.hosts[1], result.get_host_by_name(self.hosts[1].name))
def test_get_host_by_type(self):
"""Tests retrieval of a Rack's host(s) by type"""
result = models.Rack(self.RACK_NAME, self.hosts)
self.assertEqual(self.hosts[0], result.get_host_by_type('genesis')[0])
self.assertEqual(self.hosts[1], result.get_host_by_type('compute')[0])
self.assertEqual(
self.hosts[2],
result.get_host_by_type('controller')[0])
class TestVLANNetworkData(unittest.TestCase): class TestVLANNetworkData(unittest.TestCase):
"""Tests for the VLANNetworkData model""" """Tests for the VLANNetworkData model"""
@ -820,3 +832,162 @@ class TestSiteDocumentData(unittest.TestCase):
type(Rack()).name = mock.PropertyMock(side_effect=['rack1', 'rack3']) type(Rack()).name = mock.PropertyMock(side_effect=['rack1', 'rack3'])
result = models.SiteDocumentData(site_info, network, baremetal) result = models.SiteDocumentData(site_info, network, baremetal)
self.assertIsNone(result.get_baremetal_rack_by_name('rack2')) self.assertIsNone(result.get_baremetal_rack_by_name('rack2'))
@mock.patch('spyglass.data_extractor.models.SiteInfo')
@mock.patch('spyglass.data_extractor.models.Network')
@mock.patch('spyglass.data_extractor.models.Rack')
@mock.patch('spyglass.data_extractor.models.Host')
def test_get_baremetal_rack_by_name_multiple(
self, Host, Rack, Network, SiteInfo):
"""Tests retrieval of baremetal host(s) by type"""
site_info = SiteInfo()
network = Network()
baremetal = [Rack(), Rack()]
Rack().get_host_by_type.return_value = [Host()]
result = models.SiteDocumentData(site_info, network, baremetal)
self.assertEqual(2, len(result.get_baremetal_host_by_type('genesis')))
self.assertEqual(2, len(result.get_baremetal_host_by_type('computer')))
self.assertEqual(
2, len(result.get_baremetal_host_by_type('controller')))
class TestValidateKeyInIntermediaryDict(unittest.TestCase):
"""Tests the _validate_key_in_intermediary_dict function"""
def test__validate_key_in_intermediary_dict(self):
test_dictionary = {'test_key': 'value'}
key = 'test_key'
self.assertIsNone(
models._validate_key_in_intermediary_dict(key, test_dictionary))
def test__validate_key_in_intermediary_dict_key_dne(self):
test_dictionary = {'test_key': 'value'}
key = 'not_test_key'
with self.assertRaises(InvalidIntermediary):
models._validate_key_in_intermediary_dict(key, test_dictionary)
class TestSiteDocumentDataFactory(unittest.TestCase):
"""Tests the site_document_data_factory function"""
def setUp(self) -> None:
test_intermediary_path = os.path.join(
FIXTURE_DIR, 'test_intermediary.yaml')
with open(test_intermediary_path, 'r') as f:
self.intermediary_dict = yaml.safe_load(f)
def test_site_document_data_factory(self):
site_document_data = models.site_document_data_factory(
self.intermediary_dict)
# Check correct return type
self.assertIsInstance(site_document_data, models.SiteDocumentData)
def test_site_document_data_factory_saves_storage(self):
site_document_data = models.site_document_data_factory(
self.intermediary_dict)
# Check that storage was saved without changes in the SiteDocumentData
self.assertDictEqual(
self.intermediary_dict['storage'], site_document_data.storage)
def test_site_document_data_factory_saves_site_info(self):
site_document_data = models.site_document_data_factory(
self.intermediary_dict)
# Check that site info saved correctly in SiteInfo object
site_info_dict = self.intermediary_dict['site_info']
self.assertIsInstance(site_document_data.site_info, models.SiteInfo)
self.assertEqual(
site_info_dict['name'], site_document_data.site_info.name)
self.assertEqual(
self.intermediary_dict['region_name'],
site_document_data.site_info.region_name)
self.assertEqual(
site_info_dict['state'], site_document_data.site_info.state)
self.assertEqual(
site_info_dict['physical_location_id'],
site_document_data.site_info.physical_location_id)
self.assertEqual(
site_info_dict['country'], site_document_data.site_info.country)
self.assertEqual(
site_info_dict['corridor'], site_document_data.site_info.corridor)
self.assertEqual(
site_info_dict['sitetype'], site_document_data.site_info.sitetype)
self.assertEqual(
site_info_dict['domain'], site_document_data.site_info.domain)
self.assertDictEqual(
site_info_dict['ldap'], site_document_data.site_info.ldap)
self.assertEqual(
site_info_dict['dns']['servers'],
str(site_document_data.site_info.dns))
self.assertEqual(
site_info_dict['ntp']['servers'],
str(site_document_data.site_info.ntp))
def test_site_document_data_factory_saves_network_data(self):
site_document_data = models.site_document_data_factory(
self.intermediary_dict)
# Check that network data saved correctly into a Network object
network_dict = self.intermediary_dict['network']
self.assertIsInstance(site_document_data.network, models.Network)
self.assertDictEqual(
network_dict['bgp'], site_document_data.network.bgp)
for network_type, network_data \
in network_dict['vlan_network_data'].items():
vlan_network_data = \
site_document_data.network.get_vlan_data_by_name(network_type)
self.assertIsInstance(vlan_network_data, models.VLANNetworkData)
self.assertEqual(network_type, vlan_network_data.name)
self.assertEqual(network_type, vlan_network_data.role)
self.assertEqual(network_data['subnet'], vlan_network_data.subnet)
if 'routes' in network_data:
self.assertEqual(
network_data['routes'], vlan_network_data.routes)
if 'gateway' in network_data:
self.assertEqual(
network_data['gateway'], vlan_network_data.gateway)
if 'vlan' in network_data:
self.assertEqual(network_data['vlan'], vlan_network_data.vlan)
if 'dhcp_start' in network_data and 'dhcp_end' in network_data:
self.assertEqual(
network_data['dhcp_start'], vlan_network_data.dhcp_start)
self.assertEqual(
network_data['dhcp_end'], vlan_network_data.dhcp_end)
if 'static_start' in network_data and 'static_end' in network_data:
self.assertEqual(
network_data['static_start'],
vlan_network_data.static_start)
self.assertEqual(
network_data['static_end'], vlan_network_data.static_end)
if 'reserved_start' in network_data \
and 'reserved_end' in network_data:
self.assertEqual(
network_data['reserved_start'],
vlan_network_data.reserved_start)
self.assertEqual(
network_data['reserved_end'],
vlan_network_data.reserved_end)
def test_site_document_data_factory_saves_baremetal_data(self):
site_document_data = models.site_document_data_factory(
self.intermediary_dict)
# Check that baremetal racks saved correctly into Rack objects
for rack_name, hosts \
in self.intermediary_dict['baremetal'].items():
rack = site_document_data.get_baremetal_rack_by_name(rack_name)
for host_name, host_data in hosts.items():
host = rack.get_host_by_name(host_name)
self.assertEqual(host_name, host.name)
self.assertEqual(rack_name, host.rack_name)
self.assertEqual(host_data['type'], host.type)
self.assertEqual(host_data['host_profile'], host.host_profile)
self.assertEqual(host_data['ip']['oob'], host.ip.oob)
self.assertEqual(host_data['ip']['oam'], host.ip.oam)
self.assertEqual(host_data['ip']['calico'], host.ip.calico)
self.assertEqual(host_data['ip']['overlay'], host.ip.overlay)
self.assertEqual(host_data['ip']['pxe'], host.ip.pxe)
self.assertEqual(host_data['ip']['storage'], host.ip.storage)
self.assertEqual(rack_name, rack.name)

247
tests/unit/site_processors/test_site_processor.py
View File

@ -15,117 +15,178 @@
import logging import logging
import os import os
from tempfile import mkdtemp from tempfile import mkdtemp
import textwrap
import unittest
from unittest import mock
from jinja2 import UndefinedError from jinja2 import UndefinedError
import pytest import pytest
from spyglass.data_extractor import models
from spyglass.site_processors.site_processor import SiteProcessor from spyglass.site_processors.site_processor import SiteProcessor
LOG = logging.getLogger(__name__) LOG = logging.getLogger(__name__)
LOG.level = logging.DEBUG LOG.level = logging.DEBUG
J2_TPL = """---
schema: pegleg/SiteDefinition/v1
metadata:
schema: metadata/Document/v1
layeringDefinition:
abstract: false
layer: site
name: {{ data['region_name'] }}
storagePolicy: cleartext
data:
site_type:{{ data['site_info']['sitetype'] }}
..."""
class TestSiteProcessor(unittest.TestCase):
def test_render_template(): J2_TPL = textwrap.dedent(
_tpl_parent_dir = mkdtemp() """
_tpl_dir = mkdtemp(dir=_tpl_parent_dir) ---
_tpl_file = os.path.join(_tpl_dir, "test.yaml.j2") schema: pegleg/SiteDefinition/v1
with open(_tpl_file, 'w') as f: metadata:
f.write(J2_TPL) schema: metadata/Document/v1
LOG.debug("Writing test template to %s", _tpl_file) layeringDefinition:
_input_yaml = { abstract: false
"region_name": "test", layer: site
"site_info": { name: {{ data.site_info.region_name }}
"sitetype": "test_type" storagePolicy: cleartext
} data:
} site_type:{{ data.site_info.sitetype }}
_out_dir = mkdtemp() ...""")
site_processor = SiteProcessor(_input_yaml, _out_dir, force_write=False)
site_processor.render_template(_tpl_parent_dir)
expected_output = """--- J2_TPL_UNDEFINED = textwrap.dedent(
schema: pegleg/SiteDefinition/v1 """
metadata: ---
schema: metadata/Document/v1 schema: pegleg/SiteDefinition/v1
layeringDefinition: metadata:
abstract: false schema: metadata/Document/v1
layer: site layeringDefinition:
name: test abstract: false
storagePolicy: cleartext layer: site
data: name: {{ data.site_info.region_name }}
site_type:test_type storagePolicy: cleartext
...""" data:
site_type:{{ undefined_param }}
...""")
output_file = os.path.join( @mock.patch(
_out_dir, "pegleg_manifests", "site", _input_yaml["region_name"], 'spyglass.data_extractor.models.SiteDocumentData',
os.path.split(_tpl_dir)[1], "test.yaml") spec=models.SiteDocumentData)
LOG.debug(output_file) @mock.patch('spyglass.data_extractor.models.SiteInfo')
assert (os.path.exists(output_file)) @mock.patch('spyglass.data_extractor.models.ServerList')
with open(output_file, 'r') as f: def test_render_template(self, ServerList, SiteInfo, SiteDocumentData):
content = f.read() _tpl_parent_dir = mkdtemp()
assert (expected_output == content) _tpl_dir = mkdtemp(dir=_tpl_parent_dir)
_tpl_file = os.path.join(_tpl_dir, "test.yaml.j2")
with open(_tpl_file, 'w') as f:
f.write(self.J2_TPL)
LOG.debug("Writing test template to %s", _tpl_file)
site_data = SiteDocumentData()
type(SiteDocumentData()).site_info = SiteInfo()
region_name = 'test'
type(SiteInfo()).region_name = mock.PropertyMock(
return_value=region_name)
site_type = 'test_type'
type(SiteInfo()).sitetype = mock.PropertyMock(return_value=site_type)
def test_render_template_missing_data(): _out_dir = mkdtemp()
_tpl_parent_dir = mkdtemp() site_processor = SiteProcessor(site_data, _out_dir, force_write=False)
_tpl_dir = mkdtemp(dir=_tpl_parent_dir)
_tpl_file = os.path.join(_tpl_dir, "test.yaml.j2")
with open(_tpl_file, 'w') as f:
f.write(J2_TPL)
LOG.debug("Writing test template to %s", _tpl_file)
_input_yaml = {"region_name": "test", "site_info": {}}
_out_dir = mkdtemp()
site_processor = SiteProcessor(_input_yaml, _out_dir, force_write=False)
with pytest.raises(UndefinedError):
site_processor.render_template(_tpl_parent_dir) site_processor.render_template(_tpl_parent_dir)
output_file = os.path.join( expected_output = textwrap.dedent(
_out_dir, "pegleg_manifests", "site", _input_yaml["region_name"], """
os.path.split(_tpl_dir)[1], "test.yaml") ---
assert (not os.path.exists(output_file)) schema: pegleg/SiteDefinition/v1
metadata:
schema: metadata/Document/v1
layeringDefinition:
abstract: false
layer: site
name: test
storagePolicy: cleartext
data:
site_type:test_type
...""")
output_file = os.path.join(
_out_dir, "pegleg_manifests", "site", region_name,
os.path.split(_tpl_dir)[1], "test.yaml")
LOG.debug(output_file)
self.assertTrue(os.path.exists(output_file))
with open(output_file, 'r') as f:
content = f.read()
self.assertEqual(expected_output, content)
def test_render_template_missing_data_force(): @mock.patch(
_tpl_parent_dir = mkdtemp() 'spyglass.data_extractor.models.SiteDocumentData',
_tpl_dir = mkdtemp(dir=_tpl_parent_dir) spec=models.SiteDocumentData)
_tpl_file = os.path.join(_tpl_dir, "test.yaml.j2") @mock.patch('spyglass.data_extractor.models.SiteInfo')
with open(_tpl_file, 'w') as f: @mock.patch('spyglass.data_extractor.models.ServerList')
f.write(J2_TPL) def test_render_template_missing_data(
LOG.debug("Writing test template to %s", _tpl_file) self, ServerList, SiteInfo, SiteDocumentData):
_input_yaml = {"region_name": "test", "site_info": {}} _tpl_parent_dir = mkdtemp()
_out_dir = mkdtemp() _tpl_dir = mkdtemp(dir=_tpl_parent_dir)
site_processor = SiteProcessor(_input_yaml, _out_dir, force_write=True) _tpl_file = os.path.join(_tpl_dir, "test.yaml.j2")
site_processor.render_template(_tpl_parent_dir) with open(_tpl_file, 'w') as f:
f.write(self.J2_TPL_UNDEFINED)
LOG.debug("Writing test template to %s", _tpl_file)
expected_output = """--- site_data = SiteDocumentData()
schema: pegleg/SiteDefinition/v1 type(SiteDocumentData()).site_info = SiteInfo()
metadata: region_name = 'test'
schema: metadata/Document/v1 type(SiteInfo()).region_name = mock.PropertyMock(
layeringDefinition: return_value=region_name)
abstract: false site_type = 'test_type'
layer: site type(SiteInfo()).sitetype = mock.PropertyMock(return_value=site_type)
name: test
storagePolicy: cleartext
data:
site_type:
..."""
output_file = os.path.join( _out_dir = mkdtemp()
_out_dir, "pegleg_manifests", "site", _input_yaml["region_name"], site_processor = SiteProcessor(site_data, _out_dir, force_write=False)
os.path.split(_tpl_dir)[1], "test.yaml") with pytest.raises(UndefinedError):
assert (os.path.exists(output_file)) site_processor.render_template(_tpl_parent_dir)
with open(output_file, 'r') as f:
content = f.read() output_file = os.path.join(
assert (expected_output == content) _out_dir, "pegleg_manifests", "site", region_name,
os.path.split(_tpl_dir)[1], "test.yaml")
self.assertFalse(os.path.exists(output_file))
@mock.patch(
'spyglass.data_extractor.models.SiteDocumentData',
spec=models.SiteDocumentData)
@mock.patch('spyglass.data_extractor.models.SiteInfo')
@mock.patch('spyglass.data_extractor.models.ServerList')
def test_render_template_missing_data_force(
self, ServerList, SiteInfo, SiteDocumentData):
_tpl_parent_dir = mkdtemp()
_tpl_dir = mkdtemp(dir=_tpl_parent_dir)
_tpl_file = os.path.join(_tpl_dir, "test.yaml.j2")
with open(_tpl_file, 'w') as f:
f.write(self.J2_TPL_UNDEFINED)
LOG.debug("Writing test template to %s", _tpl_file)
site_data = SiteDocumentData()
type(SiteDocumentData()).site_info = SiteInfo()
region_name = 'test'
type(SiteInfo()).region_name = mock.PropertyMock(
return_value=region_name)
site_type = 'test_type'
type(SiteInfo()).sitetype = mock.PropertyMock(return_value=site_type)
_out_dir = mkdtemp()
site_processor = SiteProcessor(site_data, _out_dir, force_write=True)
site_processor.render_template(_tpl_parent_dir)
expected_output = textwrap.dedent(
"""
---
schema: pegleg/SiteDefinition/v1
metadata:
schema: metadata/Document/v1
layeringDefinition:
abstract: false
layer: site
name: test
storagePolicy: cleartext
data:
site_type:
...""")
output_file = os.path.join(
_out_dir, "pegleg_manifests", "site", region_name,
os.path.split(_tpl_dir)[1], "test.yaml")
self.assertTrue(os.path.exists(output_file))
with open(output_file, 'r') as f:
content = f.read()
self.assertEqual(expected_output, content)