Add lightweight debug report for gate-multinode

Change-Id: Ida4b7917772b5c46d898c308328765ad79dda9f4
This commit is contained in:
Kaspars Skels 2019-01-31 16:06:31 -06:00
parent 64a148d6ef
commit 9cd5654b48
2 changed files with 136 additions and 0 deletions

View File

@ -0,0 +1,105 @@
#!/usr/bin/env bash
# Copyright 2019 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -ux
SUBDIR_NAME=debug-$(hostname)
# NOTE(mark-burnett): This should add calicoctl to the path.
export PATH=${PATH}:/opt/cni/bin
TEMP_DIR=$(mktemp -d)
export TEMP_DIR
export BASE_DIR="${TEMP_DIR}/${SUBDIR_NAME}"
export HELM_DIR="${BASE_DIR}/helm"
export CALICO_DIR="${BASE_DIR}/calico"
mkdir -p "${BASE_DIR}"
PARALLELISM_FACTOR=2
function get_namespaces () {
kubectl get namespaces -o name | awk -F '/' '{ print $NF }'
}
function get_pods () {
NAMESPACE=$1
kubectl get pods -n "${NAMESPACE}" -o name --show-all | awk -F '/' '{ print $NF }' | xargs -L1 -P 1 -I {} echo "${NAMESPACE}" {}
}
export -f get_pods
function get_pod_logs () {
NAMESPACE=${1% *}
POD=${1#* }
INIT_CONTAINERS=$(kubectl get pod "${POD}" -n "${NAMESPACE}" -o json | jq -r '.spec.initContainers[]?.name')
CONTAINERS=$(kubectl get pod "${POD}" -n "${NAMESPACE}" -o json | jq -r '.spec.containers[].name')
POD_DIR="${BASE_DIR}/pod-logs/${NAMESPACE}/${POD}"
mkdir -p "${POD_DIR}"
for CONTAINER in ${INIT_CONTAINERS} ${CONTAINERS}; do
echo "get_pod_logs() [${NAMESPACE}] ${POD} ${CONTAINER}"
kubectl logs "${POD}" -n "${NAMESPACE}" -c "${CONTAINER}" > "${POD_DIR}/${CONTAINER}.txt"
done
}
export -f get_pod_logs
function get_releases () {
helm list --all --short
}
function get_release () {
input=($1)
RELEASE=${input[0]}
helm status "${RELEASE}" > "${HELM_DIR}/${RELEASE}.txt"
}
export -f get_release
if which helm; then
mkdir -p "${HELM_DIR}"
helm list --all > "${HELM_DIR}/list"
get_releases | \
xargs -r -n 1 -P "${PARALLELISM_FACTOR}" -I {} bash -c 'get_release "$@"' _ {}
fi
kubectl get --all-namespaces -o wide pods > "${BASE_DIR}/pods.txt"
get_namespaces | \
xargs -r -n 1 -P "${PARALLELISM_FACTOR}" -I {} bash -c 'get_pods "$@"' _ {} | \
xargs -r -n 2 -P "${PARALLELISM_FACTOR}" -I {} bash -c 'get_pod_logs "$@"' _ {}
iptables-save > "${BASE_DIR}/iptables"
cat /var/log/syslog > "${BASE_DIR}/syslog"
cat /var/log/armada/bootstrap-armada.log > "${BASE_DIR}/bootstrap-armada.log"
ip addr show > "${BASE_DIR}/ifconfig"
ip route show > "${BASE_DIR}/ip-route"
cp -p /etc/resolv.conf "${BASE_DIR}/"
env | sort --ignore-case > "${BASE_DIR}/environment"
docker images > "${BASE_DIR}/docker-images"
if which calicoctl; then
mkdir -p "${CALICO_DIR}"
for kind in bgpPeer hostEndpoint ipPool nodes policy profile workloadEndpoint; do
calicoctl get "${kind}" -o yaml > "${CALICO_DIR}/${kind}.yaml"
done
fi
tar zcf "${SUBDIR_NAME}.tgz" -C "${TEMP_DIR}" "${SUBDIR_NAME}"

View File

@ -0,0 +1,31 @@
#!/usr/bin/env bash
# Copyright 2019 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -ex
source "${GATE_UTILS}"
ssh_cmd "${GENESIS_NAME}" apt-get install -y jq
# Copies script and virtmgr private key to genesis VM
ssh_cmd "${GENESIS_NAME}" mkdir -p /root/airship
rsync_cmd "${REPO_ROOT}/tools/multi_nodes_gate/airship_gate/bin/debug-report-lite.sh" "${GENESIS_NAME}:/root/airship/"
set -o pipefail
ssh_cmd "${GENESIS_NAME}" /root/airship/debug-report-lite.sh 2>&1 | tee -a "${LOG_FILE}"
set +o pipefail
rsync_cmd "${GENESIS_NAME}:/root/debug-${GENESIS_NAME}.tgz" .