diff --git a/tools/multi_nodes_gate/airship_gate/lib/config.sh b/tools/multi_nodes_gate/airship_gate/lib/config.sh index 9745d74f..48b9f282 100644 --- a/tools/multi_nodes_gate/airship_gate/lib/config.sh +++ b/tools/multi_nodes_gate/airship_gate/lib/config.sh @@ -48,24 +48,66 @@ config_vm_names() { jq -cr '.vm | keys | join(" ")' < "${GATE_MANIFEST}" } -config_vm_ip() { - nodename=${1} - jq -cr ".vm.${nodename}.ip" < "${GATE_MANIFEST}" +config_vm_iface_list() { + nodename="$1" + jq -cr ".vm.${nodename}.networking | del(.addresses) | keys | .[]" < "${GATE_MANIFEST}" } -config_vm_mac() { - nodename=${1} - jq -cr ".vm.${nodename}.mac" < "${GATE_MANIFEST}" +config_vm_iface_mac() { + nodename="$1" + interface="$2" + jq -cr ".vm.${nodename}.networking.${interface}.mac" < "${GATE_MANIFEST}" } -config_vm_io() { - nodename=${1} - io_profile=$(jq -cr ".vm.${nodename}.io_profile" < "${GATE_MANIFEST}") - if [[ -z "$io_profile" ]] +# What network this VM interface should be attached to +config_vm_iface_network() { + nodename="$1" + interface="$2" + jq -cr ".vm.${nodename}.networking.${interface}.attachment.network" < "${GATE_MANIFEST}" +} + +# What VLANs on a network should be attached to this node +config_vm_iface_vlans() { + nodename="$1" + interface="$2" + jq -cr ".vm.${nodename}.networking.${interface}.attachment.vlans | select(.!=null)" < "${GATE_MANIFEST}" +} + +# PCI slot for this VM interface +config_vm_iface_slot() { + nodename="$1" + interface="$2" + jq -cr ".vm.${nodename}.networking.${interface}.pci.slot" < "${GATE_MANIFEST}" +} + +# PCI card port for this VM interface +config_vm_iface_port() { + nodename="$1" + interface="$2" + jq -cr ".vm.${nodename}.networking.${interface}.pci.port" < "${GATE_MANIFEST}" +} + +# The IP address for the VM for a network. If vlan is also specified, the VLAN +# on the network +config_vm_net_ip() { + nodename="$1" + network="$2" + vlan="$3" + + if is_netspec "$network" then - io_profile="fast" + vlan=$(netspec_vlan "$network") + network=$(netspec_netname "$network") fi - echo -n "$io_profile" + + if [[ -z "$vlan" ]] + then + query=".vm.${nodename}.networking.addresses.${network}.ip" + else + query=".vm.${nodename}.networking.addresses.${network}.vlans.${vlan}.ip" + fi + + jq -cr "$query" < "${GATE_MANIFEST}" } config_vm_vcpus() { @@ -157,6 +199,224 @@ config_bgp_as() { jq -cr ".bgp.${as_number}" < "${GATE_MANIFEST}" } +config_net_list() { + jq -cr '.networking | keys | .[]' < "${GATE_MANIFEST}" +} + +config_net_vlan_list() { + network="$1" + + jq -cr ".networking.${network}.layer2.vlans // {} | keys | .[]" < "${GATE_MANIFEST}" +} + +config_net_cidr() { + network="$1" + vlan="$2" + + if is_netspec "$network" + then + vlan=$(netspec_vlan "$network") + network=$(netspec_netname "$network") + fi + + if [[ -z "$vlan" ]] + then + query=".networking.${network}.layer3.cidr" + else + query=".networking.${network}.vlans.${vlan}.layer3.cidr" + fi + + jq -cr "$query" < "${GATE_MANIFEST}" +} + +config_net_is_layer3() { + network="$1" + vlan="$2" + + if is_netspec "$network" + then + vlan=$(netspec_vlan "$network") + network=$(netspec_netname "$network") + fi + + if [[ -z "$vlan" ]] + then + query=".networking.${network} | has(\"layer3\")" + else + query=".network.${network}.vlans.${vlan} | has(\"layer3\")" + fi + + jq -cr "$query" < "${GATE_MANIFEST}" +} + +# Find the layer 3 network tagged for a particular +# role - this can be either a native or vlan network +# If multiple networks have a role, the results is +# undefined +config_netspec_for_role() { + role="$1" + + set -x + + for net in $(config_net_list) + do + if config_net_has_role "$net" "$role" + then + netspec="$net" + fi + + for vlan in $(config_net_vlan_list "$net") + do + if config_vlan_has_role "$net" "$vlan" "$role" + then + netspec="${vlan}@${net}" + fi + done + done + + echo -n "$netspec" +} + +config_net_has_role() { + netname="$1" + role="$2" + + value="$(jq -cr ".networking.${netname}.roles | contains([\"${role}\"])" < "$GATE_MANIFEST")" + + if [ "$value" == "true" ] + then + return 0 + else + return 1 + fi +} + +config_vlan_has_role() { + netname="$1" + vlan="$2" + role="$3" + + value="$(jq -cr " .networking.${netname}.vlans.${vlan}.roles | contains([\"${role}\"])" < "$GATE_MANIFEST")" + + if [ "$value" == "true" ] + then + return 0 + else + return 1 + fi +} + +config_net_selfip() { + network="$1" + vlan="$2" + + if [[ -z "$vlan" ]] + then + query=".networking.${network}.layer3.address" + else + query=".networking.${network}.vlans.${vlan}.layer3.address" + fi + + jq -cr "$query" < "${GATE_MANIFEST}" +} + +config_net_selfip_cidr() { + network="$1" + vlan="$2" + + if is_netspec "$network" + then + vlan=$(netspec_vlan "$network") + network=$(netspec_netname "$network") + fi + + selfip=$(config_net_selfip "$network" "$vlan") + netcidr=$(config_net_cidr "$network" "$vlan") + netbits=$(echo "$netcidr" | awk -F '/' '{print $2}') + + printf "%s/%s" "$selfip" "$netbits" +} + +config_net_gateway() { + network="$1" + vlan="$2" + + if is_netspec "$network" + then + vlan=$(netspec_vlan "$network") + network=$(netspec_netname "$network") + fi + + if [[ -z "$vlan" ]] + then + query=".networking.${network}.layer3.gateway" + else + query=".networking.${network}.vlans.${vlan}.layer3.gateway" + fi + + jq -cr "$query" < "${GATE_MANIFEST}" +} + +config_net_routemode() { + network="$1" + vlan="$2" + + if is_netspec "$network" + then + vlan=$(netspec_vlan "$network") + network=$(netspec_netname "$network") + fi + + if [[ -z "$vlan" ]] + then + query=".networking.${network}.layer3.routing.mode" + else + query=".networking.${network}.vlans.${vlan}.layer3.routing.mode" + fi + + jq -cr "$query" < "${GATE_MANIFEST}" +} + +config_net_mtu() { + network="$1" + vlan="$2" + + if is_netspec "$network" + then + vlan=$(netspec_vlan "$network") + network=$(netspec_netname "$network") + fi + + if [[ -z "$vlan" ]] + then + query=".networking.${network}.layer2.mtu // 1500" + else + query=".networking.${network}.vlans.${vlan}.layer2.mtu // 1500" + fi + + jq -cr "$query" < "${GATE_MANIFEST}" +} + +config_net_mac() { + network="$1" + vlan="$2" + + if is_netspec "$network" + then + vlan=$(netspec_vlan "$network") + network=$(netspec_netname "$network") + fi + + if [[ -z "$vlan" ]] + then + query=".networking.${network}.layer2.address" + else + query=".networking.${network}.vlans.${vlan}.layer2.address" + fi + + jq -cr "$query" < "${GATE_MANIFEST}" +} + config_ingress_domain() { jq -cr '.ingress.domain' < "${GATE_MANIFEST}" } diff --git a/tools/multi_nodes_gate/airship_gate/lib/virsh.sh b/tools/multi_nodes_gate/airship_gate/lib/virsh.sh index 5dbc71ad..65bc1afe 100644 --- a/tools/multi_nodes_gate/airship_gate/lib/virsh.sh +++ b/tools/multi_nodes_gate/airship_gate/lib/virsh.sh @@ -22,6 +22,144 @@ img_base_declare() { fi } +netconfig_gen_mtu() { + MTU="$1" + + set +e + IFS= read -r -d '' MTU_TMP <<'EOF' + mtu: ${MTU} +EOF + set -e + + MTU="$MTU" envsubst <<< "$MTU_TMP" >> network-config +} + +netconfig_gen_physical() { + IFACE_NAME="$1" + MTU="$2" + + set +e + IFS= read -r -d '' PHYS_TMP <<'EOF' + - type: physical + name: ${IFACE_NAME} +EOF + set -e + + IFACE_NAME="$IFACE_NAME" envsubst <<< "$PHYS_TMP" >> network-config + + if [ ! -z "$MTU" ] + then + netconfig_gen_mtu "$MTU" + fi +} + +netconfig_gen_subnet() { + IP_ADDR="$1" + IP_MASK="$2" + GW_ADDR="$3" + + set +e + IFS= read -r -d '' SUBNET_TMP <<'EOF' + subnets: + - type: static + address: ${IP_ADDR} + netmask: ${IP_MASK} +EOF + + IFS= read -r -d '' SUBNET_GW_TMP <<'EOF' + gateway: ${GW_ADDR} +EOF + set -e + + IP_ADDR="$IP_ADDR" IP_MASK="$IP_MASK" envsubst <<< "$SUBNET_TMP" >> network-config + + if [ ! -z "$GW_ADDR" ] + then + GW_ADDR="$GW_ADDR" envsubst <<< "$SUBNET_GW_TMP" >> network-config + fi + +} + +netconfig_gen_vlan() { + IFACE_NAME="$1" + VLAN_TAG="$2" + MTU="$3" + + set +e + IFS= read -r -d '' VLAN_TMP <<'EOF' + - type: vlan + name: ${IFACE_NAME}.${VLAN_TAG} + vlan_link: ${IFACE_NAME} + vlan_id: ${VLAN_TAG} +EOF + set -e + + IFACE_NAME="$IFACE_NAME" VLAN_TAG="$VLAN_TAG" envsubst <<< "$VLAN_TMP" >> network-config + + if [ ! -z "$MTU" ] + then + netconfig_gen_mtu "$MTU" + fi + +} + +netconfig_gen_nameservers() { + NAMESERVERS="$1" + + set +e + IFS= read -r -d '' NS_TMP <<'EOF' + - type: nameserver + address: [${DNS_SERVERS}] +EOF + set -e + + DNS_SERVERS="$NAMESERVERS" envsubst <<< "$NS_TMP" >> network-config +} + +netconfig_gen() { + NAME="$1" + + IFS= cat << 'EOF' > network-config +version: 1 +config: +EOF + + # Generate physical interfaces + for iface in $(config_vm_iface_list "$NAME") + do + iface_network=$(config_vm_iface_network "$NAME" "$iface") + netconfig_gen_physical "$iface" "$(config_net_mtu "$iface_network")" + + if [ "$(config_net_is_layer3 "$iface_network")" == "true" ] + then + iface_ip="$(config_vm_net_ip "$NAME" "$iface_network")" + netmask="$(cidr_to_netmask "$(config_net_cidr "$iface_network")")" + net_gw="$(config_net_gateway "$iface_network")" + netconfig_gen_subnet "$iface_ip" "$netmask" "$net_gw" + else + if [ ! -z "$(config_vm_iface_vlans "$NAME" "$iface")" ] + then + for vlan in $(config_vm_iface_vlans "$NAME" "$iface") + do + netconfig_gen_vlan "$iface" "$vlan" "$(config_net_mtu "$iface_network" "$vlan")" + if [ "$(config_net_is_layer3 "$iface_network" "$vlan")" == "true" ] + then + iface_ip="$(config_vm_net_ip "$NAME" "$iface_network" "$vlan")" + netmask="$(cidr_to_netmask "$(config_net_cidr "$iface_network" "$vlan")")" + net_gw="$(config_net_gateway "$iface_network" "$vlan")" + netconfig_gen_subnet "$iface_ip" "$netmask" "$net_gw" + fi + done + fi + fi + done + + DNS_SERVERS=$(echo "$UPSTREAM_DNS" | tr ' ' ',') + netconfig_gen_nameservers "$DNS_SERVERS" + + sed -i -e '/^$/d' network-config +} + iso_gen() { NAME=${1} ADDL_USERDATA="${2}" @@ -40,13 +178,14 @@ iso_gen() { mkdir -p "${ISO_DIR}" cd "${ISO_DIR}" - BR_IP_NODE=$(config_vm_ip "${NAME}") + netconfig_gen "$NAME" + SSH_PUBLIC_KEY=$(ssh_load_pubkey) - export BR_IP_NODE + export NAME export SSH_PUBLIC_KEY - export NTP_POOLS=$(join_array ',' $NTP_POOLS) - export NTP_SERVERS=$(join_array ',' $NTP_SERVERS) + export NTP_POOLS="$(join_array ',' "$NTP_POOLS")" + export NTP_SERVERS="$(join_array ',' "$NTP_SERVERS")" envsubst < "${TEMPLATE_DIR}/user-data.sub" > user-data fs_header="false" @@ -99,9 +238,6 @@ iso_gen() { envsubst < "${TEMPLATE_DIR}/meta-data.sub" > meta-data - export DNS_SERVERS=$(join_array ',' $UPSTREAM_DNS) - envsubst < "${TEMPLATE_DIR}/network-config.sub" > network-config - { genisoimage \ -V cidata \ @@ -131,20 +267,79 @@ iso_path() { echo "${TEMP_DIR}/iso/${NAME}/cidata.iso" } -net_clean() { - if virsh net-list --name | grep ^airship_gate$ > /dev/null; then - log Destroying Airship gate network - virsh net-destroy "${XML_DIR}/network.xml" &>> "${LOG_FILE}" - fi +net_list() { + namekey="$1" + if [[ -z "$namekey" ]] + then + grepargs=("-v" '^$') + else + grepargs=("^${namekey}") + fi + + virsh net-list --name | grep "${grepargs[@]}" } -net_declare() { - if ! virsh net-list --name | grep ^airship_gate$ > /dev/null; then - log Creating Airship gate network - virsh net-define "${XML_DIR}/network.xml" &>> "${LOG_FILE}" - virsh net-start airship_gate - virsh net-autostart airship_gate +nets_clean() { + namekey=$(get_namekey) + + for netname in $(net_list "$namekey") + do + log Destroying Airship gate "$netname" + + for iface in $(ip -oneline l show type vlan | grep "$netname" | awk -F ' ' '{print $2}' | tr -d ':' | awk -F '@' '{print $1}') + do + sudo ip l del dev "$iface" &>> "$LOG_FILE" + done + virsh net-destroy "$netname" &>> "${LOG_FILE}" + virsh net-undefine "$netname" &>> "${LOG_FILE}" + done +} + +net_create() { + netname="$1" + namekey=$(get_namekey) + virsh_netname="${namekey}"_"${netname}" + + if [[ $(config_net_is_layer3 "$net") == "true" ]]; then + net_template="${TEMPLATE_DIR}/l3network-definition.sub" + + NETNAME="${virsh_netname}" NETIP="$(config_net_selfip "$netname")" NETMASK="$(cidr_to_netmask $(config_net_cidr "$netname"))" NETMAC="$(config_net_mac "$netname")" envsubst < "$net_template" > ${TEMP_DIR}/net-${netname}.xml + else + net_template="${TEMPLATE_DIR}/l2network-definition.sub" + + NETNAME="${virsh_netname}" envsubst < "$net_template" > ${TEMP_DIR}/net-${netname}.xml + fi + + log Creating network "${namekey}"_"${netname}" + + virsh net-define "${TEMP_DIR}/net-${netname}.xml" &>> "${LOG_FILE}" + virsh net-start "${virsh_netname}" + virsh net-autostart "${virsh_netname}" + + for vlan in $(config_net_vlan_list "$netname") + do + if [[ $(config_net_is_layer3 "$netname") == "true" ]] + then + iface_name="${virsh_netname}-${vlan}" + iface_mtu=$(confg_net_mtu "$netname" "$vlan") + if [[ -z "$iface_mtu" ]] + then + iface_mtu="1500" + fi + iface_mac="$(config_net_mac "$netname" "$vlan")" + sudo ip link add link "${virsh_netname}" name "${iface_name}" type vlan id "${vlan}" mtu $iface_mtu address "$iface_mac" + sudo ip addr add "$(config_net_selfip_cidr "$netname" "$vlan")" dev "${iface_name}" + sudo ip link set dev "${iface_name}" up fi + done +} + +nets_declare() { + nets_clean + + for net in $(config_net_list); do + net_create "$net" + done } pool_declare() { @@ -179,15 +374,53 @@ vm_clean_all() { wait } -vm_create() { - NAME=${1} - MAC_ADDRESS=$(config_vm_mac "${NAME}") - IO_PROF=$(config_vm_io "${NAME}") - if [[ "$IO_PROF" == "fast" ]] - then - DISK_OPTS="bus=virtio,cache=none,format=qcow2,io=native" - elif [[ "$IO_PROF" == "safe" ]] +# TODO(sh8121att) - Sort out how to ensure the proper NIC +# is used for PXE boot +vm_render_interface() { + vm="$1" + iface="$2" + + namekey="$(get_namekey)" + + mac="$(config_vm_iface_mac "$vm" "$iface")" + network="$(config_vm_iface_network "$vm" "$iface")" + network="${namekey}_${network}" + slot="$(config_vm_iface_slot "$vm" "$iface")" + port="$(config_vm_iface_port "$vm" "$iface")" + + config_string="model=virtio,network=${network}" + + if [[ ! -z "$mac" ]] + then + config_string="${config_string},mac=${mac}" + fi + + if [[ ! -z "$slot" ]] + then + config_string="${config_string},address.type=pci,address.slot=${slot}" + if [[ ! -z "$port" ]] then + config_string="${config_string},address.function=${port}" + fi + + fi + + + echo -n "$config_string" +} + +vm_create_interfaces() { + vm="$1" + + network_opts="" + for interface in $(config_vm_iface_list "$vm") + do + nic_opts="$(vm_render_interface "$vm" "$interface")" + network_opts="$network_opts --network ${nic_opts}" + done + + echo "$network_opts" +} vm_create_vols(){ NAME="$1" @@ -259,6 +492,7 @@ vm_create() { set -x NAME=${1} DISK_OPTS="$(vm_create_vols "${NAME}")" + NETWORK_OPTS="$(vm_create_interfaces "${NAME}")" if [[ "$(config_vm_bootstrap "${NAME}")" == "true" ]]; then iso_gen "${NAME}" "$(config_vm_userdata "${NAME}")" @@ -273,10 +507,9 @@ vm_create() { --serial file,path=${TEMP_DIR}/console/${NAME}.log \ --graphics none \ --noautoconsole \ - --network "network=airship_gate,model=virtio,address.type=pci,address.slot=0x03" \ - --mac="${MAC_ADDRESS}" \ - --vcpus "$(config_vm_vcpus ${NAME})" \ - --memory "$(config_vm_memory ${NAME})" \ + $NETWORK_OPTS \ + --vcpus "$(config_vm_vcpus "${NAME}")" \ + --memory "$(config_vm_memory "${NAME}")" \ --import \ $DISK_OPTS \ --disk "vol=${VIRSH_POOL}/cloud-init-${NAME}.iso,device=cdrom" &>> "${LOG_FILE}" @@ -295,10 +528,9 @@ vm_create() { --graphics none \ --serial file,path=${TEMP_DIR}/console/${NAME}.log \ --noautoconsole \ - --network "network=airship_gate,model=virtio,address.type=pci,address.slot=0x03" \ - --mac="${MAC_ADDRESS}" \ - --vcpus "$(config_vm_vcpus ${NAME})" \ - --memory "$(config_vm_memory ${NAME})" \ + $NETWORK_OPTS \ + --vcpus "$(config_vm_vcpus "${NAME}")" \ + --memory "$(config_vm_memory "${NAME}")" \ --import \ $DISK_OPTS &>> "${LOG_FILE}" fi diff --git a/tools/multi_nodes_gate/airship_gate/stages/create-vms.sh b/tools/multi_nodes_gate/airship_gate/stages/create-vms.sh index e7780d04..b35e5378 100755 --- a/tools/multi_nodes_gate/airship_gate/stages/create-vms.sh +++ b/tools/multi_nodes_gate/airship_gate/stages/create-vms.sh @@ -17,5 +17,6 @@ set -e source "${GATE_UTILS}" +nets_declare vm_clean_all vm_create_all diff --git a/tools/multi_nodes_gate/airship_gate/stages/gate-setup.sh b/tools/multi_nodes_gate/airship_gate/stages/gate-setup.sh index f2ee0dae..0a7dcbb2 100755 --- a/tools/multi_nodes_gate/airship_gate/stages/gate-setup.sh +++ b/tools/multi_nodes_gate/airship_gate/stages/gate-setup.sh @@ -30,7 +30,6 @@ ssh_setup_declare # Virsh setup pool_declare img_base_declare -net_declare # Make libvirtd available via SSH make_virtmgr_account diff --git a/tools/multi_nodes_gate/airship_gate/templates/l2network-definition.sub b/tools/multi_nodes_gate/airship_gate/templates/l2network-definition.sub new file mode 100644 index 00000000..080a8ff7 --- /dev/null +++ b/tools/multi_nodes_gate/airship_gate/templates/l2network-definition.sub @@ -0,0 +1,4 @@ + + ${NETNAME} + + diff --git a/tools/multi_nodes_gate/airship_gate/templates/l3network-definition.sub b/tools/multi_nodes_gate/airship_gate/templates/l3network-definition.sub new file mode 100644 index 00000000..97216278 --- /dev/null +++ b/tools/multi_nodes_gate/airship_gate/templates/l3network-definition.sub @@ -0,0 +1,7 @@ + + ${NETNAME} + + + + + diff --git a/tools/multi_nodes_gate/airship_gate/templates/network-config.sub b/tools/multi_nodes_gate/airship_gate/templates/network-config.sub deleted file mode 100644 index b41b60de..00000000 --- a/tools/multi_nodes_gate/airship_gate/templates/network-config.sub +++ /dev/null @@ -1,11 +0,0 @@ -#cloud-config -version: 1 -config: - - type: physical - name: ens3 - subnets: - - type: static - address: ${BR_IP_NODE}/24 - gateway: 172.24.1.1 - - type: nameserver - address: [${DNS_SERVERS}] diff --git a/tools/multi_nodes_gate/airship_gate/xml/network.xml b/tools/multi_nodes_gate/airship_gate/xml/network.xml deleted file mode 100644 index c185bfd6..00000000 --- a/tools/multi_nodes_gate/airship_gate/xml/network.xml +++ /dev/null @@ -1,8 +0,0 @@ - - airship_gate - - - - - -