diff --git a/examples/basic/armada-resources.yaml b/examples/basic/armada-resources.yaml index 77dead13..ea93be30 100644 --- a/examples/basic/armada-resources.yaml +++ b/examples/basic/armada-resources.yaml @@ -121,28 +121,6 @@ metadata: abstract: false layer: site storagePolicy: cleartext - substitutions: - - - src: - schema: deckhand/CertificateAuthority/v1 - name: kubernetes - path: . - dest: - path: '.values.secrets.tls.ca' - - - src: - schema: deckhand/Certificate/v1 - name: proxy - path: . - dest: - path: '.values.secrets.tls.cert' - - - src: - schema: deckhand/CertificateKey/v1 - name: proxy - path: . - dest: - path: '.values.secrets.tls.key' data: chart_name: proxy release: kubernetes-proxy diff --git a/examples/complete/armada-resources.yaml b/examples/complete/armada-resources.yaml index 9b586a34..deb0d392 100644 --- a/examples/complete/armada-resources.yaml +++ b/examples/complete/armada-resources.yaml @@ -164,28 +164,6 @@ metadata: abstract: false layer: site storagePolicy: cleartext - substitutions: - - - src: - schema: deckhand/CertificateAuthority/v1 - name: kubernetes - path: . - dest: - path: '.values.secrets.tls.ca' - - - src: - schema: deckhand/Certificate/v1 - name: proxy - path: . - dest: - path: '.values.secrets.tls.cert' - - - src: - schema: deckhand/CertificateKey/v1 - name: proxy - path: . - dest: - path: '.values.secrets.tls.key' data: chart_name: proxy release: kubernetes-proxy diff --git a/tools/g2/lib/all.sh b/tools/g2/lib/all.sh index c4c77a69..fd2b87d7 100644 --- a/tools/g2/lib/all.sh +++ b/tools/g2/lib/all.sh @@ -5,6 +5,7 @@ LIB_DIR=$(realpath "$(dirname "${BASH_SOURCE}")") source "$LIB_DIR"/config.sh source "$LIB_DIR"/const.sh +source "$LIB_DIR"/docker.sh source "$LIB_DIR"/etcd.sh source "$LIB_DIR"/kube.sh source "$LIB_DIR"/log.sh diff --git a/tools/g2/lib/docker.sh b/tools/g2/lib/docker.sh new file mode 100644 index 00000000..7591b67f --- /dev/null +++ b/tools/g2/lib/docker.sh @@ -0,0 +1,26 @@ +docker_ps() { + VIA="${1}" + ssh_cmd "${VIA}" docker ps -a +} + +docker_info() { + VIA="${1}" + ssh_cmd "${VIA}" docker info 2>&1 +} + +docker_exited_containers() { + VIA="${1}" + ssh_cmd "${VIA}" docker ps -q --filter "status=exited" +} + +docker_inspect() { + VIA="${1}" + CONTAINER_ID="${2}" + ssh_cmd "${VIA}" docker inspect "${CONTAINER_ID}" +} + +docker_logs() { + VIA="${1}" + CONTAINER_ID="${2}" + ssh_cmd "${VIA}" docker logs "${CONTAINER_ID}" +} diff --git a/tools/g2/lib/log.sh b/tools/g2/lib/log.sh index ff699f83..439088cb 100644 --- a/tools/g2/lib/log.sh +++ b/tools/g2/lib/log.sh @@ -46,8 +46,7 @@ log_note() { log_stage_error() { NAME=${1} - TEMP_DIR=${2} - echo -e " ${C_ERROR}== Error in stage ${C_HILIGHT}${NAME}${C_ERROR} ( ${C_TEMP}${TEMP_DIR}${C_ERROR} ) ==${C_CLEAR}" + echo -e " ${C_ERROR}== Error in stage ${C_HILIGHT}${NAME}${C_ERROR} ( ${C_TEMP}${LOG_FILE}${C_ERROR} ) ==${C_CLEAR}" } log_stage_footer() { @@ -65,7 +64,6 @@ log_stage_success() { } log_temp_dir() { - TEMP_DIR=${1} echo -e "Working in ${C_TEMP}${TEMP_DIR}${C_CLEAR}" } diff --git a/tools/g2/lib/nginx.sh b/tools/g2/lib/nginx.sh index a74efe56..0d4dc74a 100644 --- a/tools/g2/lib/nginx.sh +++ b/tools/g2/lib/nginx.sh @@ -20,11 +20,12 @@ nginx_up() { nginx_cache_and_replace_tar_urls() { log "Finding tar_url options to cache.." TAR_NUM=0 + mkdir -p "${NGINX_DIR}" for file in "$@"; do grep -Po "^ +tar_url: \K.+$" "${file}" | while read tar_url ; do # NOTE(mark-burnet): Does not yet ignore repeated files. - log "Caching ${tar_url} in file: ${file}" DEST_PATH="${NGINX_DIR}/cached-tar-${TAR_NUM}.tgz" + log "Caching ${tar_url} in file: ${DEST_PATH}" REPLACEMENT_URL="${NGINX_URL}/cached-tar-${TAR_NUM}.tgz" curl -Lo "${DEST_PATH}" "${tar_url}" sed -i "s;${tar_url};${REPLACEMENT_URL};" "${file}" diff --git a/tools/g2/manifests/genesis.json b/tools/g2/manifests/genesis.json index 572f6506..108518c0 100644 --- a/tools/g2/manifests/genesis.json +++ b/tools/g2/manifests/genesis.json @@ -26,7 +26,8 @@ }, { "name": "Genesis", - "script": "genesis.sh" + "script": "genesis.sh", + "on_error": "collect_genesis_info.sh" } ], "vm": { diff --git a/tools/g2/manifests/integration.json b/tools/g2/manifests/integration.json index 9b1139fa..0edc1ef4 100644 --- a/tools/g2/manifests/integration.json +++ b/tools/g2/manifests/integration.json @@ -26,7 +26,8 @@ }, { "name": "Genesis", - "script": "genesis.sh" + "script": "genesis.sh", + "on_error": "collect_genesis_info.sh" }, { "name": "Load Site Configuration", diff --git a/tools/g2/manifests/resiliency.json b/tools/g2/manifests/resiliency.json index e38a96e3..8a807378 100644 --- a/tools/g2/manifests/resiliency.json +++ b/tools/g2/manifests/resiliency.json @@ -30,7 +30,8 @@ }, { "name": "Genesis", - "script": "genesis.sh" + "script": "genesis.sh", + "on_error": "collect_genesis_info.sh" }, { "name": "Join Masters", diff --git a/tools/g2/on_error/collect_genesis_info.sh b/tools/g2/on_error/collect_genesis_info.sh new file mode 100755 index 00000000..ed1e713f --- /dev/null +++ b/tools/g2/on_error/collect_genesis_info.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +# NOTE(mark-burnett): Keep trying to collect info even if there's an error +set +e +set -x + +source "${GATE_UTILS}" + +ERROR_DIR="${TEMP_DIR}/errors" +VIA=n0 +mkdir -p "${ERROR_DIR}" + +log "Gathering info from failed genesis server (n0) in ${ERROR_DIR}" + +log "Gathering docker info for exitted containers" +mkdir -p "${ERROR_DIR}/docker" +docker_ps "${VIA}" | tee "${ERROR_DIR}/docker/ps" +docker_info "${VIA}" | tee "${ERROR_DIR}/docker/info" + +for container_id in $(docker_exited_containers "${VIA}"); do + docker_inspect "${VIA}" "${container_id}" | tee "${ERROR_DIR}/docker/${container_id}" + echo "=== Begin logs ===" | tee -a "${ERROR_DIR}/docker/${container_id}" + docker_logs "${VIA}" "${container_id}" | tee -a "${ERROR_DIR}/docker/${container_id}" +done + +log "Gathering kubectl output" +mkdir -p "${ERROR_DIR}/kube" +kubectl_cmd "${VIA}" describe nodes n0 | tee "${ERROR_DIR}/kube/n0" +kubectl_cmd "${VIA}" get --all-namespaces -o wide pod | tee "${ERROR_DIR}/kube/pods" diff --git a/tools/g2/stages/genesis.sh b/tools/g2/stages/genesis.sh index 7fb6f126..61461327 100755 --- a/tools/g2/stages/genesis.sh +++ b/tools/g2/stages/genesis.sh @@ -6,8 +6,10 @@ source "${GATE_UTILS}" rsync_cmd "${TEMP_DIR}/scripts"/*genesis* "${GENESIS_NAME}:/root/promenade/" +set -o pipefail ssh_cmd "${GENESIS_NAME}" /root/promenade/genesis.sh 2>&1 | tee -a "${LOG_FILE}" ssh_cmd "${GENESIS_NAME}" /root/promenade/validate-genesis.sh 2>&1 | tee -a "${LOG_FILE}" +set +o pipefail if ! ssh_cmd n0 docker images | tail -n +2 | grep -v registry:5000 ; then log_warn "Using some non-cached docker images. This will slow testing." diff --git a/tools/gate.sh b/tools/gate.sh index ce6ab32d..b6487a87 100755 --- a/tools/gate.sh +++ b/tools/gate.sh @@ -22,7 +22,7 @@ chmod -R 755 "${TEMP_DIR}" STAGES_DIR=${WORKSPACE}/tools/g2/stages -log_temp_dir "${TEMP_DIR}" +log_temp_dir echo STAGES=$(mktemp) @@ -44,10 +44,11 @@ while read -u 3 stage; do log_stage_error "${NAME}" "${LOG_FILE}" if echo "${stage}" | jq -e .on_error > /dev/null; then log_stage_diagnostic_header - ON_ERROR=${WORKSPACE}/$(echo "${stage}" | jq -r .on_error) + ON_ERROR=${WORKSPACE}/tools/g2/on_error/$(echo "${stage}" | jq -r .on_error) set +e $ON_ERROR fi + log_stage_error "${NAME}" "${TEMP_DIR}" exit 1 fi log_stage_footer "${NAME}" diff --git a/tools/gate/config-templates/bootstrap-armada-config.yaml b/tools/gate/config-templates/bootstrap-armada-config.yaml index 22b02c0f..549d82d7 100644 --- a/tools/gate/config-templates/bootstrap-armada-config.yaml +++ b/tools/gate/config-templates/bootstrap-armada-config.yaml @@ -119,28 +119,6 @@ metadata: abstract: false layer: site storagePolicy: cleartext - substitutions: - - - src: - schema: deckhand/CertificateAuthority/v1 - name: kubernetes - path: . - dest: - path: '.values.secrets.tls.ca' - - - src: - schema: deckhand/Certificate/v1 - name: proxy - path: . - dest: - path: '.values.secrets.tls.cert' - - - src: - schema: deckhand/CertificateKey/v1 - name: proxy - path: . - dest: - path: '.values.secrets.tls.key' data: chart_name: proxy release: kubernetes-proxy @@ -149,11 +127,6 @@ data: upgrade: no_hooks: true values: - secrets: - tls: - ca: placeholder - cert: placeholder - key: placeholder images: tags: proxy: ${IMAGE_HYPERKUBE}