From b987b2b52f576f2c413c66ad000c3a1deb78c0ce Mon Sep 17 00:00:00 2001 From: Scott Hussey Date: Wed, 5 Sep 2018 16:55:48 -0500 Subject: [PATCH] Emulate external DNS for ingress Ingress requires external DNS or host header forging. For a better gate test, run a external DNS resolver for Ingress entries. Change-Id: I6558230b7fa9f3145c472eb437177d90b7b52835 --- .../v1.0demo/software/config/versions.yaml | 6 ++-- .../software/charts/ucp/drydock/maas.yaml | 8 +++++ .../software/configs/versions.yaml | 11 +++++- .../multi_nodes_gate/airship_gate/lib/all.sh | 1 + .../airship_gate/lib/config.sh | 23 +++++++++++++ .../airship_gate/lib/ingress.sh | 34 +++++++++++++++++++ .../airship_gate/lib/ssh-config-global.sub | 6 ---- .../airship_gate/lib/virsh.sh | 12 +++++-- .../airship_gate/manifest-schema.json | 17 ++++++++-- .../manifests/multinode_deploy.json | 20 +++++++++++ .../airship_gate/stages/ingress-dns.sh | 23 +++++++++++++ .../templates/ingress_corefile.sub | 9 +++++ .../airship_gate/templates/ingress_entry.sub | 1 + .../airship_gate/templates/ingress_header.sub | 4 +++ 14 files changed, 162 insertions(+), 13 deletions(-) create mode 100644 tools/multi_nodes_gate/airship_gate/lib/ingress.sh delete mode 100644 tools/multi_nodes_gate/airship_gate/lib/ssh-config-global.sub create mode 100755 tools/multi_nodes_gate/airship_gate/stages/ingress-dns.sh create mode 100644 tools/multi_nodes_gate/airship_gate/templates/ingress_corefile.sub create mode 100644 tools/multi_nodes_gate/airship_gate/templates/ingress_entry.sub create mode 100644 tools/multi_nodes_gate/airship_gate/templates/ingress_header.sub diff --git a/deployment_files/global/v1.0demo/software/config/versions.yaml b/deployment_files/global/v1.0demo/software/config/versions.yaml index e8a4aefb..69e82dc8 100644 --- a/deployment_files/global/v1.0demo/software/config/versions.yaml +++ b/deployment_files/global/v1.0demo/software/config/versions.yaml @@ -239,7 +239,7 @@ data: type: git location: https://git.openstack.org/openstack/openstack-helm-infra subpath: helm-toolkit - reference: b6fc24b9960bc6de29aa70c7d5b7725319046160 + reference: 9f6194ff770d3776ae3572bcbe841a9289357244 mariadb: type: git location: https://git.openstack.org/openstack/openstack-helm @@ -507,7 +507,9 @@ data: maas_region: quay.io/airshipit/maas-region-controller:dfb92ce07f87648f3423be11e51425923bc9f8b1 bootstrap: quay.io/airshipit/maas-region-controller:dfb92ce07f87648f3423be11e51425923bc9f8b1 export_api_key: quay.io/airshipit/maas-region-controller:dfb92ce07f87648f3423be11e51425923bc9f8b1 - maas_cache: quay.io/attcomdev/sstream-cache@sha256:sha256:90d0dc73785534ef6a3035265275e197a66727beeb9c979f60d3bbf31a6a8864 + maas_cache: quay.io/airshipit/sstream-cache:dfb92ce07f87648f3423be11e51425923bc9f8b1 + ingress: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.9.0 + error_pages: gcr.io/google_containers/defaultbackend:1.0 dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.3.1 keystone: keystone_bootstrap: docker.io/openstackhelm/heat:ocata diff --git a/deployment_files/site/gate-multinode/software/charts/ucp/drydock/maas.yaml b/deployment_files/site/gate-multinode/software/charts/ucp/drydock/maas.yaml index 402815c0..f6da4ce7 100644 --- a/deployment_files/site/gate-multinode/software/charts/ucp/drydock/maas.yaml +++ b/deployment_files/site/gate-multinode/software/charts/ucp/drydock/maas.yaml @@ -24,4 +24,12 @@ data: values: manifests: secret_ssh_key: true + conf: + cache: + enabled: true + dns: + dns_servers: 172.24.1.9 + network: + maas_ingress: + addr: 172.24.1.5/32 ... diff --git a/deployment_files/site/gate-multinode/software/configs/versions.yaml b/deployment_files/site/gate-multinode/software/configs/versions.yaml index 0e8abbcc..da4549d1 100644 --- a/deployment_files/site/gate-multinode/software/configs/versions.yaml +++ b/deployment_files/site/gate-multinode/software/configs/versions.yaml @@ -13,8 +13,17 @@ metadata: - method: merge path: . storagePolicy: cleartext -data: {} # This document is merged with the globals, and is where you can override # versions to match a patchset, for example. Anything specified in the global # versions can be overridden, such as chart or image versions. +data: + charts: + ucp: + maas: + location: https://git.openstack.org/openstack/airship-maas + reference: refs/changes/53/600253/1 + images: + ucp: + maas: + maas_region: docker.io/sthussey/maas-region-controller:dev ... diff --git a/tools/multi_nodes_gate/airship_gate/lib/all.sh b/tools/multi_nodes_gate/airship_gate/lib/all.sh index bd5ad61c..d13cbe20 100644 --- a/tools/multi_nodes_gate/airship_gate/lib/all.sh +++ b/tools/multi_nodes_gate/airship_gate/lib/all.sh @@ -14,6 +14,7 @@ source "$LIB_DIR"/registry.sh source "$LIB_DIR"/ssh.sh source "$LIB_DIR"/virsh.sh source "$LIB_DIR"/airship.sh +source "$LIB_DIR"/ingress.sh if [[ -v GATE_DEBUG && ${GATE_DEBUG} = "1" ]]; then set -x diff --git a/tools/multi_nodes_gate/airship_gate/lib/config.sh b/tools/multi_nodes_gate/airship_gate/lib/config.sh index 351275ca..445b3ec2 100644 --- a/tools/multi_nodes_gate/airship_gate/lib/config.sh +++ b/tools/multi_nodes_gate/airship_gate/lib/config.sh @@ -9,6 +9,7 @@ export BASE_IMAGE_URL=${BASE_IMAGE_URL:-https://cloud-images.ubuntu.com/releases export IMAGE_PROMENADE_CLI=${IMAGE_PROMENADE_CLI:-quay.io/airshipit/promenade:master} export IMAGE_PEGLEG_CLI=${IMAGE_PEGLEG_CLI:-quay.io/airshipit/pegleg:master} export IMAGE_SHIPYARD_CLI=${IMAGE_SHIPYARD_CLI:-quay.io/airshipit/shipyard:master} +export IMAGE_COREDNS=${IMAGE_COREDNS:-docker.io/coredns/coredns:1.2.2} export PROMENADE_DEBUG=${PROMENADE_DEBUG:-0} export SHIPYARD_PASSWORD=${SHIPYARD_OS_PASSWORD:-password18} export REGISTRY_DATA_DIR=${REGISTRY_DATA_DIR:-/mnt/registry} @@ -50,6 +51,28 @@ config_vm_bootstrap() { fi } +config_vm_userdata() { + nodename=${1} + val=$(jq -cr ".vm.${nodename}.userdata" < "${GATE_MANIFEST}") + + if [[ "${val}" != "null" ]] + then + echo "${val}" + fi +} +config_ingress_domain() { + jq -cr '.ingress.domain' < "${GATE_MANIFEST}" +} + +config_ingress_ips() { + jq -cr '.ingress | keys | map(select(. != "domain")) | join(" ")' < "${GATE_MANIFEST}" +} + +config_ingress_entries() { + IP=$1 + jq -cr ".ingress[\"${IP}\"] | join(\" \")" < "${GATE_MANIFEST}" +} + config_pegleg_primary_repo() { jq -cr ".configuration.primary_repo" < "${GATE_MANIFEST}" } diff --git a/tools/multi_nodes_gate/airship_gate/lib/ingress.sh b/tools/multi_nodes_gate/airship_gate/lib/ingress.sh new file mode 100644 index 00000000..e28bd46f --- /dev/null +++ b/tools/multi_nodes_gate/airship_gate/lib/ingress.sh @@ -0,0 +1,34 @@ +DNS_ZONE_FILE="${TEMP_DIR}/ingress.dns" +COREFILE="${TEMP_DIR}/ingress.corefile" + +ingress_dns_config() { + ingress_domain=$(config_ingress_domain) + + INGRESS_DOMAIN=${ingress_domain} envsubst '${INGRESS_DOMAIN}' < "${TEMPLATE_DIR}/ingress_header.sub" > "${DNS_ZONE_FILE}" + + read -a ingress_ip_list <<< $(config_ingress_ips) + + for ip in "${ingress_ip_list[@]}" + do + read -a ip_entries <<< $(config_ingress_entries $ip) + for entry in "${ip_entries[@]}" + do + HOSTNAME=${entry} HOSTIP=${ip} envsubst < "${TEMPLATE_DIR}/ingress_entry.sub" >> "${DNS_ZONE_FILE}" + done + done + + DNS_DOMAIN=${ingress_domain} ZONE_FILE=$(basename $DNS_ZONE_FILE) envsubst < "${TEMPLATE_DIR}/ingress_corefile.sub" > "${COREFILE}" +} + +ingress_dns_start() { + # nodename where DNS should run + nodename=$1 + remote_work_dir="/var/tmp/coredns" + + remote_zone_file="${remote_work_dir}/$(basename $DNS_ZONE_FILE)" + remote_corefile="${remote_work_dir}/$(basename $COREFILE)" + ssh_cmd "${nodename}" mkdir -p "${remote_work_dir}" + rsync_cmd "$DNS_ZONE_FILE" "${nodename}:${remote_zone_file}" + rsync_cmd "$COREFILE" "${nodename}:${remote_corefile}" + ssh_cmd "${nodename}" docker run -d -v /var/tmp/coredns:/data -w /data --network host -P $IMAGE_COREDNS -conf $(basename $remote_corefile) +} diff --git a/tools/multi_nodes_gate/airship_gate/lib/ssh-config-global.sub b/tools/multi_nodes_gate/airship_gate/lib/ssh-config-global.sub deleted file mode 100644 index 3c4a4c44..00000000 --- a/tools/multi_nodes_gate/airship_gate/lib/ssh-config-global.sub +++ /dev/null @@ -1,6 +0,0 @@ -IdentityFile ${SSH_CONFIG_DIR}/id_rsa -LogLevel QUIET -StrictHostKeyChecking no -User root -UserKnownHostsFile /dev/null - diff --git a/tools/multi_nodes_gate/airship_gate/lib/virsh.sh b/tools/multi_nodes_gate/airship_gate/lib/virsh.sh index 18aa7cf4..31e4dfe0 100644 --- a/tools/multi_nodes_gate/airship_gate/lib/virsh.sh +++ b/tools/multi_nodes_gate/airship_gate/lib/virsh.sh @@ -23,6 +23,7 @@ img_base_declare() { iso_gen() { NAME=${1} + ADDL_USERDATA="${2}" if virsh vol-key --pool "${VIRSH_POOL}" --vol "cloud-init-${NAME}.iso" &> /dev/null; then log Removing existing cloud-init ISO for "${NAME}" @@ -42,6 +43,13 @@ iso_gen() { export NAME export SSH_PUBLIC_KEY envsubst < "${TEMPLATE_DIR}/user-data.sub" > user-data + + if [[ ! -z "${ADDL_USERDATA}" ]] + then + echo >> user-data + echo -e "${ADDL_USERDATA}" >> user-data + fi + envsubst < "${TEMPLATE_DIR}/meta-data.sub" > meta-data envsubst < "${TEMPLATE_DIR}/network-config.sub" > network-config @@ -126,7 +134,7 @@ vm_create() { wait if [[ "$(config_vm_bootstrap ${NAME})" == "true" ]]; then - iso_gen "${NAME}" + iso_gen "${NAME}" "$(config_vm_userdata ${NAME})" wait log Creating VM "${NAME}" and bootstrapping the boot drive @@ -260,7 +268,7 @@ make_virtmgr_account() { sudo useradd -m -s /bin/sh -g "${libvirt_group}" virtmgr else sudo usermod -g "${libvirt_group}" virtmgr - fi + fi done } diff --git a/tools/multi_nodes_gate/airship_gate/manifest-schema.json b/tools/multi_nodes_gate/airship_gate/manifest-schema.json index 564ff8d7..db615b4a 100644 --- a/tools/multi_nodes_gate/airship_gate/manifest-schema.json +++ b/tools/multi_nodes_gate/airship_gate/manifest-schema.json @@ -31,6 +31,20 @@ "publish": { "$ref": "#/definitions/publish" }, + "ingress": { + "type": "object", + "properties": { + "domain": { + "type": "string" + }, + "additionalProperties": { + "type": "array", + "items": { + "type": "string" + } + }, + "required": ["domain"] + }, "stages": { "type": "array", "items": { @@ -104,8 +118,7 @@ "required": [ "memory", "names", - "vcpus", - "non_genesis" + "vcpus" ], "additionalProperties": false } diff --git a/tools/multi_nodes_gate/airship_gate/manifests/multinode_deploy.json b/tools/multi_nodes_gate/airship_gate/manifests/multinode_deploy.json index a6a5baab..fdc9e578 100644 --- a/tools/multi_nodes_gate/airship_gate/manifests/multinode_deploy.json +++ b/tools/multi_nodes_gate/airship_gate/manifests/multinode_deploy.json @@ -4,6 +4,13 @@ "primary_repo": "deployment_files", "aux_repos": [] }, + "ingress": { + "domain": "gate.local", + "172.24.1.5": [ + "maas", + "drydock" + ] + }, "stages": [ { "name": "Gate Setup", @@ -29,6 +36,11 @@ "name": "Create VMs", "script": "create-vms.sh" }, + { + "name": "Register Ingress", + "script": "ingress-dns.sh", + "arguments": ["build"] + }, { "name": "Genesis", "script": "genesis.sh", @@ -44,6 +56,14 @@ } ], "vm": { + "build": { + "memory": 3072, + "vcpus": 2, + "mac": "52:54:00:00:be:31", + "ip": "172.24.1.9", + "bootstrap": true, + "userdata": "packages: [docker.io]" + }, "n0" : { "memory": 32768, "vcpus": 8, diff --git a/tools/multi_nodes_gate/airship_gate/stages/ingress-dns.sh b/tools/multi_nodes_gate/airship_gate/stages/ingress-dns.sh new file mode 100755 index 00000000..affabf2f --- /dev/null +++ b/tools/multi_nodes_gate/airship_gate/stages/ingress-dns.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +source "${GATE_UTILS}" + +DNS_SERVER=$1 + +ingress_dns_config +ingress_dns_start ${DNS_SERVER} diff --git a/tools/multi_nodes_gate/airship_gate/templates/ingress_corefile.sub b/tools/multi_nodes_gate/airship_gate/templates/ingress_corefile.sub new file mode 100644 index 00000000..6f12b239 --- /dev/null +++ b/tools/multi_nodes_gate/airship_gate/templates/ingress_corefile.sub @@ -0,0 +1,9 @@ +${DNS_DOMAIN} { + file ${ZONE_FILE} + log +} + +. { + forward . /etc/resolv.conf + log +} diff --git a/tools/multi_nodes_gate/airship_gate/templates/ingress_entry.sub b/tools/multi_nodes_gate/airship_gate/templates/ingress_entry.sub new file mode 100644 index 00000000..b758ff0c --- /dev/null +++ b/tools/multi_nodes_gate/airship_gate/templates/ingress_entry.sub @@ -0,0 +1 @@ +${HOSTNAME} IN A ${HOSTIP} diff --git a/tools/multi_nodes_gate/airship_gate/templates/ingress_header.sub b/tools/multi_nodes_gate/airship_gate/templates/ingress_header.sub new file mode 100644 index 00000000..274d889e --- /dev/null +++ b/tools/multi_nodes_gate/airship_gate/templates/ingress_header.sub @@ -0,0 +1,4 @@ +$ORIGIN ${INGRESS_DOMAIN}. + +${INGRESS_DOMAIN}. IN SOA localhost. root.localhost. ( 2007120710 1d 2h 4w 1h ) +