(multinode) Network configuration by JSON

- Support configuration of the network and VM
  interface configuration via JSON.

Change-Id: I994e00f493145ed0ada6d4ba4e5735fd5753c127
This commit is contained in:
Scott Hussey 2019-08-06 15:16:25 -05:00 committed by Nishant Kumar
parent 2eb00af047
commit d950efcf23
8 changed files with 550 additions and 66 deletions

View File

@ -48,24 +48,66 @@ config_vm_names() {
jq -cr '.vm | keys | join(" ")' < "${GATE_MANIFEST}"
}
config_vm_ip() {
nodename=${1}
jq -cr ".vm.${nodename}.ip" < "${GATE_MANIFEST}"
config_vm_iface_list() {
nodename="$1"
jq -cr ".vm.${nodename}.networking | del(.addresses) | keys | .[]" < "${GATE_MANIFEST}"
}
config_vm_mac() {
nodename=${1}
jq -cr ".vm.${nodename}.mac" < "${GATE_MANIFEST}"
config_vm_iface_mac() {
nodename="$1"
interface="$2"
jq -cr ".vm.${nodename}.networking.${interface}.mac" < "${GATE_MANIFEST}"
}
config_vm_io() {
nodename=${1}
io_profile=$(jq -cr ".vm.${nodename}.io_profile" < "${GATE_MANIFEST}")
if [[ -z "$io_profile" ]]
# What network this VM interface should be attached to
config_vm_iface_network() {
nodename="$1"
interface="$2"
jq -cr ".vm.${nodename}.networking.${interface}.attachment.network" < "${GATE_MANIFEST}"
}
# What VLANs on a network should be attached to this node
config_vm_iface_vlans() {
nodename="$1"
interface="$2"
jq -cr ".vm.${nodename}.networking.${interface}.attachment.vlans | select(.!=null)" < "${GATE_MANIFEST}"
}
# PCI slot for this VM interface
config_vm_iface_slot() {
nodename="$1"
interface="$2"
jq -cr ".vm.${nodename}.networking.${interface}.pci.slot" < "${GATE_MANIFEST}"
}
# PCI card port for this VM interface
config_vm_iface_port() {
nodename="$1"
interface="$2"
jq -cr ".vm.${nodename}.networking.${interface}.pci.port" < "${GATE_MANIFEST}"
}
# The IP address for the VM for a network. If vlan is also specified, the VLAN
# on the network
config_vm_net_ip() {
nodename="$1"
network="$2"
vlan="$3"
if is_netspec "$network"
then
io_profile="fast"
vlan=$(netspec_vlan "$network")
network=$(netspec_netname "$network")
fi
echo -n "$io_profile"
if [[ -z "$vlan" ]]
then
query=".vm.${nodename}.networking.addresses.${network}.ip"
else
query=".vm.${nodename}.networking.addresses.${network}.vlans.${vlan}.ip"
fi
jq -cr "$query" < "${GATE_MANIFEST}"
}
config_vm_vcpus() {
@ -157,6 +199,224 @@ config_bgp_as() {
jq -cr ".bgp.${as_number}" < "${GATE_MANIFEST}"
}
config_net_list() {
jq -cr '.networking | keys | .[]' < "${GATE_MANIFEST}"
}
config_net_vlan_list() {
network="$1"
jq -cr ".networking.${network}.layer2.vlans // {} | keys | .[]" < "${GATE_MANIFEST}"
}
config_net_cidr() {
network="$1"
vlan="$2"
if is_netspec "$network"
then
vlan=$(netspec_vlan "$network")
network=$(netspec_netname "$network")
fi
if [[ -z "$vlan" ]]
then
query=".networking.${network}.layer3.cidr"
else
query=".networking.${network}.vlans.${vlan}.layer3.cidr"
fi
jq -cr "$query" < "${GATE_MANIFEST}"
}
config_net_is_layer3() {
network="$1"
vlan="$2"
if is_netspec "$network"
then
vlan=$(netspec_vlan "$network")
network=$(netspec_netname "$network")
fi
if [[ -z "$vlan" ]]
then
query=".networking.${network} | has(\"layer3\")"
else
query=".network.${network}.vlans.${vlan} | has(\"layer3\")"
fi
jq -cr "$query" < "${GATE_MANIFEST}"
}
# Find the layer 3 network tagged for a particular
# role - this can be either a native or vlan network
# If multiple networks have a role, the results is
# undefined
config_netspec_for_role() {
role="$1"
set -x
for net in $(config_net_list)
do
if config_net_has_role "$net" "$role"
then
netspec="$net"
fi
for vlan in $(config_net_vlan_list "$net")
do
if config_vlan_has_role "$net" "$vlan" "$role"
then
netspec="${vlan}@${net}"
fi
done
done
echo -n "$netspec"
}
config_net_has_role() {
netname="$1"
role="$2"
value="$(jq -cr ".networking.${netname}.roles | contains([\"${role}\"])" < "$GATE_MANIFEST")"
if [ "$value" == "true" ]
then
return 0
else
return 1
fi
}
config_vlan_has_role() {
netname="$1"
vlan="$2"
role="$3"
value="$(jq -cr " .networking.${netname}.vlans.${vlan}.roles | contains([\"${role}\"])" < "$GATE_MANIFEST")"
if [ "$value" == "true" ]
then
return 0
else
return 1
fi
}
config_net_selfip() {
network="$1"
vlan="$2"
if [[ -z "$vlan" ]]
then
query=".networking.${network}.layer3.address"
else
query=".networking.${network}.vlans.${vlan}.layer3.address"
fi
jq -cr "$query" < "${GATE_MANIFEST}"
}
config_net_selfip_cidr() {
network="$1"
vlan="$2"
if is_netspec "$network"
then
vlan=$(netspec_vlan "$network")
network=$(netspec_netname "$network")
fi
selfip=$(config_net_selfip "$network" "$vlan")
netcidr=$(config_net_cidr "$network" "$vlan")
netbits=$(echo "$netcidr" | awk -F '/' '{print $2}')
printf "%s/%s" "$selfip" "$netbits"
}
config_net_gateway() {
network="$1"
vlan="$2"
if is_netspec "$network"
then
vlan=$(netspec_vlan "$network")
network=$(netspec_netname "$network")
fi
if [[ -z "$vlan" ]]
then
query=".networking.${network}.layer3.gateway"
else
query=".networking.${network}.vlans.${vlan}.layer3.gateway"
fi
jq -cr "$query" < "${GATE_MANIFEST}"
}
config_net_routemode() {
network="$1"
vlan="$2"
if is_netspec "$network"
then
vlan=$(netspec_vlan "$network")
network=$(netspec_netname "$network")
fi
if [[ -z "$vlan" ]]
then
query=".networking.${network}.layer3.routing.mode"
else
query=".networking.${network}.vlans.${vlan}.layer3.routing.mode"
fi
jq -cr "$query" < "${GATE_MANIFEST}"
}
config_net_mtu() {
network="$1"
vlan="$2"
if is_netspec "$network"
then
vlan=$(netspec_vlan "$network")
network=$(netspec_netname "$network")
fi
if [[ -z "$vlan" ]]
then
query=".networking.${network}.layer2.mtu // 1500"
else
query=".networking.${network}.vlans.${vlan}.layer2.mtu // 1500"
fi
jq -cr "$query" < "${GATE_MANIFEST}"
}
config_net_mac() {
network="$1"
vlan="$2"
if is_netspec "$network"
then
vlan=$(netspec_vlan "$network")
network=$(netspec_netname "$network")
fi
if [[ -z "$vlan" ]]
then
query=".networking.${network}.layer2.address"
else
query=".networking.${network}.vlans.${vlan}.layer2.address"
fi
jq -cr "$query" < "${GATE_MANIFEST}"
}
config_ingress_domain() {
jq -cr '.ingress.domain' < "${GATE_MANIFEST}"
}

View File

@ -22,6 +22,144 @@ img_base_declare() {
fi
}
netconfig_gen_mtu() {
MTU="$1"
set +e
IFS= read -r -d '' MTU_TMP <<'EOF'
mtu: ${MTU}
EOF
set -e
MTU="$MTU" envsubst <<< "$MTU_TMP" >> network-config
}
netconfig_gen_physical() {
IFACE_NAME="$1"
MTU="$2"
set +e
IFS= read -r -d '' PHYS_TMP <<'EOF'
- type: physical
name: ${IFACE_NAME}
EOF
set -e
IFACE_NAME="$IFACE_NAME" envsubst <<< "$PHYS_TMP" >> network-config
if [ ! -z "$MTU" ]
then
netconfig_gen_mtu "$MTU"
fi
}
netconfig_gen_subnet() {
IP_ADDR="$1"
IP_MASK="$2"
GW_ADDR="$3"
set +e
IFS= read -r -d '' SUBNET_TMP <<'EOF'
subnets:
- type: static
address: ${IP_ADDR}
netmask: ${IP_MASK}
EOF
IFS= read -r -d '' SUBNET_GW_TMP <<'EOF'
gateway: ${GW_ADDR}
EOF
set -e
IP_ADDR="$IP_ADDR" IP_MASK="$IP_MASK" envsubst <<< "$SUBNET_TMP" >> network-config
if [ ! -z "$GW_ADDR" ]
then
GW_ADDR="$GW_ADDR" envsubst <<< "$SUBNET_GW_TMP" >> network-config
fi
}
netconfig_gen_vlan() {
IFACE_NAME="$1"
VLAN_TAG="$2"
MTU="$3"
set +e
IFS= read -r -d '' VLAN_TMP <<'EOF'
- type: vlan
name: ${IFACE_NAME}.${VLAN_TAG}
vlan_link: ${IFACE_NAME}
vlan_id: ${VLAN_TAG}
EOF
set -e
IFACE_NAME="$IFACE_NAME" VLAN_TAG="$VLAN_TAG" envsubst <<< "$VLAN_TMP" >> network-config
if [ ! -z "$MTU" ]
then
netconfig_gen_mtu "$MTU"
fi
}
netconfig_gen_nameservers() {
NAMESERVERS="$1"
set +e
IFS= read -r -d '' NS_TMP <<'EOF'
- type: nameserver
address: [${DNS_SERVERS}]
EOF
set -e
DNS_SERVERS="$NAMESERVERS" envsubst <<< "$NS_TMP" >> network-config
}
netconfig_gen() {
NAME="$1"
IFS= cat << 'EOF' > network-config
version: 1
config:
EOF
# Generate physical interfaces
for iface in $(config_vm_iface_list "$NAME")
do
iface_network=$(config_vm_iface_network "$NAME" "$iface")
netconfig_gen_physical "$iface" "$(config_net_mtu "$iface_network")"
if [ "$(config_net_is_layer3 "$iface_network")" == "true" ]
then
iface_ip="$(config_vm_net_ip "$NAME" "$iface_network")"
netmask="$(cidr_to_netmask "$(config_net_cidr "$iface_network")")"
net_gw="$(config_net_gateway "$iface_network")"
netconfig_gen_subnet "$iface_ip" "$netmask" "$net_gw"
else
if [ ! -z "$(config_vm_iface_vlans "$NAME" "$iface")" ]
then
for vlan in $(config_vm_iface_vlans "$NAME" "$iface")
do
netconfig_gen_vlan "$iface" "$vlan" "$(config_net_mtu "$iface_network" "$vlan")"
if [ "$(config_net_is_layer3 "$iface_network" "$vlan")" == "true" ]
then
iface_ip="$(config_vm_net_ip "$NAME" "$iface_network" "$vlan")"
netmask="$(cidr_to_netmask "$(config_net_cidr "$iface_network" "$vlan")")"
net_gw="$(config_net_gateway "$iface_network" "$vlan")"
netconfig_gen_subnet "$iface_ip" "$netmask" "$net_gw"
fi
done
fi
fi
done
DNS_SERVERS=$(echo "$UPSTREAM_DNS" | tr ' ' ',')
netconfig_gen_nameservers "$DNS_SERVERS"
sed -i -e '/^$/d' network-config
}
iso_gen() {
NAME=${1}
ADDL_USERDATA="${2}"
@ -40,13 +178,14 @@ iso_gen() {
mkdir -p "${ISO_DIR}"
cd "${ISO_DIR}"
BR_IP_NODE=$(config_vm_ip "${NAME}")
netconfig_gen "$NAME"
SSH_PUBLIC_KEY=$(ssh_load_pubkey)
export BR_IP_NODE
export NAME
export SSH_PUBLIC_KEY
export NTP_POOLS=$(join_array ',' $NTP_POOLS)
export NTP_SERVERS=$(join_array ',' $NTP_SERVERS)
export NTP_POOLS="$(join_array ',' "$NTP_POOLS")"
export NTP_SERVERS="$(join_array ',' "$NTP_SERVERS")"
envsubst < "${TEMPLATE_DIR}/user-data.sub" > user-data
fs_header="false"
@ -99,9 +238,6 @@ iso_gen() {
envsubst < "${TEMPLATE_DIR}/meta-data.sub" > meta-data
export DNS_SERVERS=$(join_array ',' $UPSTREAM_DNS)
envsubst < "${TEMPLATE_DIR}/network-config.sub" > network-config
{
genisoimage \
-V cidata \
@ -131,20 +267,79 @@ iso_path() {
echo "${TEMP_DIR}/iso/${NAME}/cidata.iso"
}
net_clean() {
if virsh net-list --name | grep ^airship_gate$ > /dev/null; then
log Destroying Airship gate network
virsh net-destroy "${XML_DIR}/network.xml" &>> "${LOG_FILE}"
fi
net_list() {
namekey="$1"
if [[ -z "$namekey" ]]
then
grepargs=("-v" '^$')
else
grepargs=("^${namekey}")
fi
virsh net-list --name | grep "${grepargs[@]}"
}
net_declare() {
if ! virsh net-list --name | grep ^airship_gate$ > /dev/null; then
log Creating Airship gate network
virsh net-define "${XML_DIR}/network.xml" &>> "${LOG_FILE}"
virsh net-start airship_gate
virsh net-autostart airship_gate
nets_clean() {
namekey=$(get_namekey)
for netname in $(net_list "$namekey")
do
log Destroying Airship gate "$netname"
for iface in $(ip -oneline l show type vlan | grep "$netname" | awk -F ' ' '{print $2}' | tr -d ':' | awk -F '@' '{print $1}')
do
sudo ip l del dev "$iface" &>> "$LOG_FILE"
done
virsh net-destroy "$netname" &>> "${LOG_FILE}"
virsh net-undefine "$netname" &>> "${LOG_FILE}"
done
}
net_create() {
netname="$1"
namekey=$(get_namekey)
virsh_netname="${namekey}"_"${netname}"
if [[ $(config_net_is_layer3 "$net") == "true" ]]; then
net_template="${TEMPLATE_DIR}/l3network-definition.sub"
NETNAME="${virsh_netname}" NETIP="$(config_net_selfip "$netname")" NETMASK="$(cidr_to_netmask $(config_net_cidr "$netname"))" NETMAC="$(config_net_mac "$netname")" envsubst < "$net_template" > ${TEMP_DIR}/net-${netname}.xml
else
net_template="${TEMPLATE_DIR}/l2network-definition.sub"
NETNAME="${virsh_netname}" envsubst < "$net_template" > ${TEMP_DIR}/net-${netname}.xml
fi
log Creating network "${namekey}"_"${netname}"
virsh net-define "${TEMP_DIR}/net-${netname}.xml" &>> "${LOG_FILE}"
virsh net-start "${virsh_netname}"
virsh net-autostart "${virsh_netname}"
for vlan in $(config_net_vlan_list "$netname")
do
if [[ $(config_net_is_layer3 "$netname") == "true" ]]
then
iface_name="${virsh_netname}-${vlan}"
iface_mtu=$(confg_net_mtu "$netname" "$vlan")
if [[ -z "$iface_mtu" ]]
then
iface_mtu="1500"
fi
iface_mac="$(config_net_mac "$netname" "$vlan")"
sudo ip link add link "${virsh_netname}" name "${iface_name}" type vlan id "${vlan}" mtu $iface_mtu address "$iface_mac"
sudo ip addr add "$(config_net_selfip_cidr "$netname" "$vlan")" dev "${iface_name}"
sudo ip link set dev "${iface_name}" up
fi
done
}
nets_declare() {
nets_clean
for net in $(config_net_list); do
net_create "$net"
done
}
pool_declare() {
@ -179,15 +374,53 @@ vm_clean_all() {
wait
}
vm_create() {
NAME=${1}
MAC_ADDRESS=$(config_vm_mac "${NAME}")
IO_PROF=$(config_vm_io "${NAME}")
if [[ "$IO_PROF" == "fast" ]]
then
DISK_OPTS="bus=virtio,cache=none,format=qcow2,io=native"
elif [[ "$IO_PROF" == "safe" ]]
# TODO(sh8121att) - Sort out how to ensure the proper NIC
# is used for PXE boot
vm_render_interface() {
vm="$1"
iface="$2"
namekey="$(get_namekey)"
mac="$(config_vm_iface_mac "$vm" "$iface")"
network="$(config_vm_iface_network "$vm" "$iface")"
network="${namekey}_${network}"
slot="$(config_vm_iface_slot "$vm" "$iface")"
port="$(config_vm_iface_port "$vm" "$iface")"
config_string="model=virtio,network=${network}"
if [[ ! -z "$mac" ]]
then
config_string="${config_string},mac=${mac}"
fi
if [[ ! -z "$slot" ]]
then
config_string="${config_string},address.type=pci,address.slot=${slot}"
if [[ ! -z "$port" ]]
then
config_string="${config_string},address.function=${port}"
fi
fi
echo -n "$config_string"
}
vm_create_interfaces() {
vm="$1"
network_opts=""
for interface in $(config_vm_iface_list "$vm")
do
nic_opts="$(vm_render_interface "$vm" "$interface")"
network_opts="$network_opts --network ${nic_opts}"
done
echo "$network_opts"
}
vm_create_vols(){
NAME="$1"
@ -259,6 +492,7 @@ vm_create() {
set -x
NAME=${1}
DISK_OPTS="$(vm_create_vols "${NAME}")"
NETWORK_OPTS="$(vm_create_interfaces "${NAME}")"
if [[ "$(config_vm_bootstrap "${NAME}")" == "true" ]]; then
iso_gen "${NAME}" "$(config_vm_userdata "${NAME}")"
@ -273,10 +507,9 @@ vm_create() {
--serial file,path=${TEMP_DIR}/console/${NAME}.log \
--graphics none \
--noautoconsole \
--network "network=airship_gate,model=virtio,address.type=pci,address.slot=0x03" \
--mac="${MAC_ADDRESS}" \
--vcpus "$(config_vm_vcpus ${NAME})" \
--memory "$(config_vm_memory ${NAME})" \
$NETWORK_OPTS \
--vcpus "$(config_vm_vcpus "${NAME}")" \
--memory "$(config_vm_memory "${NAME}")" \
--import \
$DISK_OPTS \
--disk "vol=${VIRSH_POOL}/cloud-init-${NAME}.iso,device=cdrom" &>> "${LOG_FILE}"
@ -295,10 +528,9 @@ vm_create() {
--graphics none \
--serial file,path=${TEMP_DIR}/console/${NAME}.log \
--noautoconsole \
--network "network=airship_gate,model=virtio,address.type=pci,address.slot=0x03" \
--mac="${MAC_ADDRESS}" \
--vcpus "$(config_vm_vcpus ${NAME})" \
--memory "$(config_vm_memory ${NAME})" \
$NETWORK_OPTS \
--vcpus "$(config_vm_vcpus "${NAME}")" \
--memory "$(config_vm_memory "${NAME}")" \
--import \
$DISK_OPTS &>> "${LOG_FILE}"
fi

View File

@ -17,5 +17,6 @@ set -e
source "${GATE_UTILS}"
nets_declare
vm_clean_all
vm_create_all

View File

@ -30,7 +30,6 @@ ssh_setup_declare
# Virsh setup
pool_declare
img_base_declare
net_declare
# Make libvirtd available via SSH
make_virtmgr_account

View File

@ -0,0 +1,4 @@
<network>
<name>${NETNAME}</name>
<bridge name='${NETNAME}' stp='on' delay='0'/>
</network>

View File

@ -0,0 +1,7 @@
<network>
<name>${NETNAME}</name>
<bridge name='${NETNAME}' stp='on' delay='0'/>
<ip address="${NETIP}" netmask="${NETMASK}" />
<mac address='${NETMAC}'/>
<forward mode="nat"/>
</network>

View File

@ -1,11 +0,0 @@
#cloud-config
version: 1
config:
- type: physical
name: ens3
subnets:
- type: static
address: ${BR_IP_NODE}/24
gateway: 172.24.1.1
- type: nameserver
address: [${DNS_SERVERS}]

View File

@ -1,8 +0,0 @@
<network>
<name>airship_gate</name>
<forward mode='nat'/>
<bridge name='airship_gate' stp='on' delay='0'/>
<mac address='52:54:00:e7:94:3f'/>
<ip address='172.24.1.1' netmask='255.255.255.0'>
</ip>
</network>