From b85c7e1621ee78ad19fe30d6ea05491255dc24ed Mon Sep 17 00:00:00 2001 From: Scott Hussey Date: Mon, 5 Aug 2019 15:10:43 -0500 Subject: [PATCH] (multinode) Make disk configuration configurable - This makes the disk configuration for VMs configurable to support various configurations. Change-Id: I94c5ce369f16bd142c9653e88d412299c8327a31 --- .../airship_gate/lib/config.sh | 112 ++++++++++++ .../airship_gate/lib/virsh.sh | 163 +++++++++++++----- .../manifests/multinode_deploy.json | 28 ++- .../airship_gate/templates/disk-data.sub | 4 + .../airship_gate/templates/mount-data.sub | 1 + 5 files changed, 266 insertions(+), 42 deletions(-) create mode 100644 tools/multi_nodes_gate/airship_gate/templates/disk-data.sub create mode 100644 tools/multi_nodes_gate/airship_gate/templates/mount-data.sub diff --git a/tools/multi_nodes_gate/airship_gate/lib/config.sh b/tools/multi_nodes_gate/airship_gate/lib/config.sh index b2b0b699..7db8fa20 100644 --- a/tools/multi_nodes_gate/airship_gate/lib/config.sh +++ b/tools/multi_nodes_gate/airship_gate/lib/config.sh @@ -1,4 +1,6 @@ +#!/bin/bash export TEMP_DIR=${TEMP_DIR:-$(mktemp -d)} +export NAMEKEY_FILE=${NAMEKEY_FILE:-"$HOME/.airship_key"} export DEFINITION_DEPOT="${TEMP_DIR}/site_yaml/" export RENDERED_DEPOT="${TEMP_DIR}/rendered_yaml/" export CERT_DEPOT="${TEMP_DIR}/cert_yaml/" @@ -82,6 +84,64 @@ config_vm_bootstrap() { fi } +config_disk_list() { + layout_name="${1}" + jq -cr ".disk_layouts.${layout_name} | keys | join(\" \")" < "${GATE_MANIFEST}" +} + +config_disk_details() { + layout_name="${1}" + disk_device="${2}" + jq -cr ".disk_layouts.${layout_name}.${disk_device}" < "${GATE_MANIFEST}" +} + +config_disk_size() { + layout_name="$1" + disk_device="$2" + jq -cr ".disk_layouts.${layout_name}.${disk_device}.size" < "${GATE_MANIFEST}" +} + +config_disk_format() { + layout_name="$1" + disk_device="$2" + do_format=$(jq -cr ".disk_layouts.${layout_name}.${disk_device} | has(\"format\")") + + if [[ "$do_format" == "true" && "$disk_device" != "$(config_layout_bootstrap "$layout_name")" ]] + then + jq -cr ".disk_layouts.${layout_name}.${disk_device}.format" < "${GATE_MANIFEST}" + else + echo "" + fi +} + +config_format_type() { + JSON="$1" + echo "$JSON" | jq -cr '.type' +} + +config_format_mount() { + JSON="$1" + echo "$JSON" | jq -cr '.mountpoint' +} + +config_disk_ioprofile() { + layout_name="$1" + disk_device="$2" + jq -cr ".disk_layouts.${layout_name}.${disk_device}.io_profile" +} + +# Find which disk in a layout should +# get the bootstrap image +config_layout_bootstrap() { + layout_name="${1}" + jq -cr ".disk_layouts.${layout_name} | keys[] as \$k | {device: (\$k)} + (.[\$k]) | select(.bootstrap) | .device " < "${GATE_MANIFEST}" +} + +config_vm_disk_layout() { + nodename=${1} + jq -cr ".vm.${nodename}.disk_layout" < "${GATE_MANIFEST}" +} + config_vm_userdata() { nodename=${1} val=$(jq -cr ".vm.${nodename}.userdata" < "${GATE_MANIFEST}") @@ -140,3 +200,55 @@ besteffort() { $@ set -e } + +get_namekey() { + if [[ -r "$NAMEKEY_FILE" ]] + then + key=$(cat "$NAMEKEY_FILE") + else + key=$(openssl rand -hex 4) + echo -n "$key" > $NAMEKEY_FILE + fi + + echo -n "$key" +} + +is_netspec(){ + value="$1" + + if echo -n "$value" | grep -q "[0-9]+@.+" + then + return 0 + else + return 1 + fi +} + +netspec_netname(){ + netspec="$1" + + echo -n "$netspec" | awk -F'@' '{print $2}' +} + +netspec_vlan(){ + netspec="$1" + + echo -n "$netspec" | awk -F'@' '{print $1}' +} + +# We'll just add the conversions as needed +cidr_to_netmask() { + cidr="$1" + netbits="$(echo "$cidr" | awk -F'/' '{print $2}')" + + case "$netbits" in + 32) + netmask="255.255.255.255" + ;; + 24) + netmask="255.255.255.0" + ;; + esac + + echo "$netmask" +} diff --git a/tools/multi_nodes_gate/airship_gate/lib/virsh.sh b/tools/multi_nodes_gate/airship_gate/lib/virsh.sh index 89772a6f..5dbc71ad 100644 --- a/tools/multi_nodes_gate/airship_gate/lib/virsh.sh +++ b/tools/multi_nodes_gate/airship_gate/lib/virsh.sh @@ -1,3 +1,4 @@ +#!/bin/bash img_base_declare() { log Validating base image exists if ! virsh vol-key --pool "${VIRSH_POOL}" --vol airship-gate-base.img > /dev/null; then @@ -24,6 +25,8 @@ img_base_declare() { iso_gen() { NAME=${1} ADDL_USERDATA="${2}" + disk_layout="$(config_vm_disk_layout "$NAME")" + vm_disks="$(config_disk_list "$disk_layout")" if virsh vol-key --pool "${VIRSH_POOL}" --vol "cloud-init-${NAME}.iso" &> /dev/null; then log Removing existing cloud-init ISO for "${NAME}" @@ -46,9 +49,51 @@ iso_gen() { export NTP_SERVERS=$(join_array ',' $NTP_SERVERS) envsubst < "${TEMPLATE_DIR}/user-data.sub" > user-data + fs_header="false" + for disk in $vm_disks + do + disk_format="$(config_disk_format "$disk_layout" "$disk")" + if [[ ! -z "$disk_format" ]] + then + if [[ "$fs_header" = "false" ]] + then + echo "fs_header:" >> user-data + fs_header="true" + fi + export FS_TYPE=$(config_format_type "$disk_format") + export DISK_DEVICE="$disk" + envsubst < "${TEMPLATE_DIR}/disk-data.sub" >> user-data + unset FS_TYPE + unset DISK_DEVICE + fi + done + + echo >> user-data + + mount_header="false" + for disk in $vm_disks + do + disk_format="$(config_disk_format "$disk_layout" "$disk")" + if [[ ! -z "$disk_format" ]] + then + if [[ "$mount_header" = "false" ]] + then + echo "mounts:" >> user-data + mount_header="true" + fi + + export MOUNTPOINT=$(config_format_mount "$disk_format") + export DISK_DEVICE="$disk" + envsubst < "${TEMPLATE_DIR}/mount-data.sub" >> user-data + unset MOUNTPOINT + unset DISK_DEVICE + fi + done + + echo >> user-data + if [[ ! -z "${ADDL_USERDATA}" ]] then - echo >> user-data echo -e "${ADDL_USERDATA}" >> user-data fi @@ -143,15 +188,80 @@ vm_create() { DISK_OPTS="bus=virtio,cache=none,format=qcow2,io=native" elif [[ "$IO_PROF" == "safe" ]] then - DISK_OPTS="bus=virtio,cache=directsync,discard=unmap,format=qcow2,io=native" - else - DISK_OPTS="bus=virtio,format=qcow2" - fi - vol_create_root "${NAME}" - wait - if [[ "$(config_vm_bootstrap ${NAME})" == "true" ]]; then - iso_gen "${NAME}" "$(config_vm_userdata ${NAME})" +vm_create_vols(){ + NAME="$1" + disk_layout="$(config_vm_disk_layout "$NAME")" + vm_disks="$(config_disk_list "$disk_layout")" + bs_disk="$(config_layout_bootstrap "$disk_layout")" + bs_vm="$(config_vm_bootstrap "${NAME}")" + + vols=() + for disk in $vm_disks + do + io_prof=$(config_disk_ioprofile "${disk_layout}" "${disk}") + size=$(config_disk_size "${disk_layout}" "${disk}") + + if [[ "$bs_vm" = "true" && "$bs_disk" = "$disk" ]] + then + vol_create_disk "$NAME" "$disk" "$size" "true" + else + vol_create_disk "$NAME" "$disk" "$size" + fi + + if [[ "$io_prof" == "fast" ]] + then + DISK_OPTS="bus=virtio,cache=none,format=qcow2,io=native" + elif [[ "$io_prof" == "safe" ]] + then + DISK_OPTS="bus=virtio,cache=directsync,discard=unmap,format=qcow2,io=native" + else + DISK_OPTS="bus=virtio,format=qcow2" + fi + + vol_cmd="--disk vol=${VIRSH_POOL}/airship-gate-${NAME}-${disk}.img,target=${disk},size=${size},${DISK_OPTS}" + vols+=($vol_cmd) + done + + echo "${vols[@]}" +} + +vol_create_disk() { + NAME=${1} + DISK=${2} + SIZE=${3} + BS=${4} + + if virsh vol-list --pool "${VIRSH_POOL}" | grep "airship-gate-${NAME}-${DISK}.img" &> /dev/null; then + log Deleting previous volume "airship-gate-${NAME}-${DISK}.img" + virsh vol-delete --pool "${VIRSH_POOL}" "airship-gate-${NAME}-${DISK}.img" &>> "${LOG_FILE}" + fi + + log Creating volume "${DISK}" for "${NAME}" + if [[ "$BS" == "true" ]]; then + virsh vol-create-as \ + --pool "${VIRSH_POOL}" \ + --name "airship-gate-${NAME}-${DISK}.img" \ + --capacity "${SIZE}"G \ + --format qcow2 \ + --backing-vol 'airship-gate-base.img' \ + --backing-vol-format qcow2 &>> "${LOG_FILE}" + else + virsh vol-create-as \ + --pool "${VIRSH_POOL}" \ + --name "airship-gate-${NAME}-${DISK}.img" \ + --capacity "${SIZE}"G \ + --format qcow2 &>> "${LOG_FILE}" + fi +} + +vm_create() { + set -x + NAME=${1} + DISK_OPTS="$(vm_create_vols "${NAME}")" + + if [[ "$(config_vm_bootstrap "${NAME}")" == "true" ]]; then + iso_gen "${NAME}" "$(config_vm_userdata "${NAME}")" wait log Creating VM "${NAME}" and bootstrapping the boot drive @@ -168,7 +278,7 @@ vm_create() { --vcpus "$(config_vm_vcpus ${NAME})" \ --memory "$(config_vm_memory ${NAME})" \ --import \ - --disk "vol=${VIRSH_POOL}/airship-gate-${NAME}.img,${DISK_OPTS}" \ + $DISK_OPTS \ --disk "vol=${VIRSH_POOL}/cloud-init-${NAME}.iso,device=cdrom" &>> "${LOG_FILE}" ssh_wait "${NAME}" @@ -190,7 +300,7 @@ vm_create() { --vcpus "$(config_vm_vcpus ${NAME})" \ --memory "$(config_vm_memory ${NAME})" \ --import \ - --disk "vol=${VIRSH_POOL}/airship-gate-${NAME}.img,${DISK_OPTS}" &>> "${LOG_FILE}" + $DISK_OPTS &>> "${LOG_FILE}" fi virsh autostart "${NAME}" } @@ -198,7 +308,7 @@ vm_create() { vm_create_validate() { NAME=${1} vm_create "${name}" - if [[ "$(config_vm_bootstrap ${name})" == "true" ]] + if [[ "$(config_vm_bootstrap "${name}")" == "true" ]] then vm_validate "${name}" fi @@ -256,35 +366,6 @@ vm_validate() { fi } - -vol_create_root() { - NAME=${1} - - if virsh vol-list --pool "${VIRSH_POOL}" | grep "airship-gate-${NAME}.img" &> /dev/null; then - log Deleting previous volume "airship-gate-${NAME}.img" - virsh vol-delete --pool "${VIRSH_POOL}" "airship-gate-${NAME}.img" &>> "${LOG_FILE}" - fi - - log Creating root volume for "${NAME}" - if [[ "$(config_vm_bootstrap ${NAME})" == "true" ]]; then - virsh vol-create-as \ - --pool "${VIRSH_POOL}" \ - --name "airship-gate-${NAME}.img" \ - --capacity 56G \ - --allocation 56G \ - --format qcow2 \ - --backing-vol 'airship-gate-base.img' \ - --backing-vol-format qcow2 &>> "${LOG_FILE}" - else - virsh vol-create-as \ - --pool "${VIRSH_POOL}" \ - --name "airship-gate-${NAME}.img" \ - --capacity 56G \ - --allocation 56G \ - --format raw &>> "${LOG_FILE}" - fi -} - #Find the correct group name for libvirt access get_libvirt_group() { grep -oE '^libvirtd?:' /etc/group | tr -d ':' diff --git a/tools/multi_nodes_gate/airship_gate/manifests/multinode_deploy.json b/tools/multi_nodes_gate/airship_gate/manifests/multinode_deploy.json index 5bf944df..6d1f6105 100644 --- a/tools/multi_nodes_gate/airship_gate/manifests/multinode_deploy.json +++ b/tools/multi_nodes_gate/airship_gate/manifests/multinode_deploy.json @@ -10,6 +10,27 @@ "172.24.1.5": ["maas"], "172.24.1.6": ["drydock","shipyard","keystone"] }, + "disk_layouts":{ + "simple": { + "vda": { + "size": 30, + "io_profile": "fast", + "bootstrap": true + } + }, + "multi": { + "vda": { + "size": 15, + "io_profile": "fast", + "bootstrap": true + }, + "vdb": { + "size": 15, + "io_profile": "fast", + "format": {"type": "ext4", "mountpoint": "/var"} + } + } + }, "stages": [ { "name": "Gate Setup", @@ -80,15 +101,17 @@ "mac": "52:54:00:00:be:31", "ip": "172.24.1.9", "io_profile": "fast", + "disk_profile": "simple", "bootstrap": true, "userdata": "packages: [docker.io]" }, "n0" : { - "memory": 32768, + "memory": 24576, "vcpus": 16, "mac": "52:54:00:00:a4:31", "ip": "172.24.1.10", "io_profile": "fast", + "disk_profile": "simple", "bootstrap": true }, "n1" : { @@ -97,6 +120,7 @@ "mac": "52:54:00:00:a3:31", "ip": "172.24.1.11", "io_profile": "fast", + "disk_profile": "simple", "bootstrap": false }, "n2" : { @@ -105,6 +129,7 @@ "mac": "52:54:00:1a:95:0d", "ip": "172.24.1.12", "io_profile": "fast", + "disk_profile": "simple", "bootstrap": false }, "n3" : { @@ -113,6 +138,7 @@ "mac": "52:54:00:31:c2:36", "ip": "172.24.1.13", "io_profile": "fast", + "disk_profile": "simple", "bootstrap": false } }, diff --git a/tools/multi_nodes_gate/airship_gate/templates/disk-data.sub b/tools/multi_nodes_gate/airship_gate/templates/disk-data.sub new file mode 100644 index 00000000..8b055384 --- /dev/null +++ b/tools/multi_nodes_gate/airship_gate/templates/disk-data.sub @@ -0,0 +1,4 @@ + - label: 'None' + filesystem: ${FS_TYPE} + device: /dev/${DISK_DEVICE} + overwrite: true diff --git a/tools/multi_nodes_gate/airship_gate/templates/mount-data.sub b/tools/multi_nodes_gate/airship_gate/templates/mount-data.sub new file mode 100644 index 00000000..ce8d0ecd --- /dev/null +++ b/tools/multi_nodes_gate/airship_gate/templates/mount-data.sub @@ -0,0 +1 @@ + - [${DISK_DEVICE}, ${MOUNTPOINT}, auto, "defaults", 0, 2]