Add lightweight debug report for gate-multinode
Change-Id: Ida4b7917772b5c46d898c308328765ad79dda9f4
This commit is contained in:
parent
64a148d6ef
commit
9cd5654b48
|
@ -0,0 +1,105 @@
|
|||
#!/usr/bin/env bash
|
||||
# Copyright 2019 AT&T Intellectual Property. All other rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
set -ux
|
||||
|
||||
SUBDIR_NAME=debug-$(hostname)
|
||||
|
||||
# NOTE(mark-burnett): This should add calicoctl to the path.
|
||||
export PATH=${PATH}:/opt/cni/bin
|
||||
|
||||
TEMP_DIR=$(mktemp -d)
|
||||
export TEMP_DIR
|
||||
export BASE_DIR="${TEMP_DIR}/${SUBDIR_NAME}"
|
||||
export HELM_DIR="${BASE_DIR}/helm"
|
||||
export CALICO_DIR="${BASE_DIR}/calico"
|
||||
|
||||
mkdir -p "${BASE_DIR}"
|
||||
|
||||
PARALLELISM_FACTOR=2
|
||||
|
||||
function get_namespaces () {
|
||||
kubectl get namespaces -o name | awk -F '/' '{ print $NF }'
|
||||
}
|
||||
|
||||
function get_pods () {
|
||||
NAMESPACE=$1
|
||||
kubectl get pods -n "${NAMESPACE}" -o name --show-all | awk -F '/' '{ print $NF }' | xargs -L1 -P 1 -I {} echo "${NAMESPACE}" {}
|
||||
}
|
||||
export -f get_pods
|
||||
|
||||
function get_pod_logs () {
|
||||
NAMESPACE=${1% *}
|
||||
POD=${1#* }
|
||||
INIT_CONTAINERS=$(kubectl get pod "${POD}" -n "${NAMESPACE}" -o json | jq -r '.spec.initContainers[]?.name')
|
||||
CONTAINERS=$(kubectl get pod "${POD}" -n "${NAMESPACE}" -o json | jq -r '.spec.containers[].name')
|
||||
POD_DIR="${BASE_DIR}/pod-logs/${NAMESPACE}/${POD}"
|
||||
mkdir -p "${POD_DIR}"
|
||||
for CONTAINER in ${INIT_CONTAINERS} ${CONTAINERS}; do
|
||||
echo "get_pod_logs() [${NAMESPACE}] ${POD} ${CONTAINER}"
|
||||
kubectl logs "${POD}" -n "${NAMESPACE}" -c "${CONTAINER}" > "${POD_DIR}/${CONTAINER}.txt"
|
||||
done
|
||||
}
|
||||
|
||||
|
||||
export -f get_pod_logs
|
||||
|
||||
function get_releases () {
|
||||
helm list --all --short
|
||||
}
|
||||
|
||||
function get_release () {
|
||||
input=($1)
|
||||
RELEASE=${input[0]}
|
||||
helm status "${RELEASE}" > "${HELM_DIR}/${RELEASE}.txt"
|
||||
|
||||
}
|
||||
export -f get_release
|
||||
|
||||
if which helm; then
|
||||
mkdir -p "${HELM_DIR}"
|
||||
helm list --all > "${HELM_DIR}/list"
|
||||
get_releases | \
|
||||
xargs -r -n 1 -P "${PARALLELISM_FACTOR}" -I {} bash -c 'get_release "$@"' _ {}
|
||||
fi
|
||||
|
||||
kubectl get --all-namespaces -o wide pods > "${BASE_DIR}/pods.txt"
|
||||
|
||||
get_namespaces | \
|
||||
xargs -r -n 1 -P "${PARALLELISM_FACTOR}" -I {} bash -c 'get_pods "$@"' _ {} | \
|
||||
xargs -r -n 2 -P "${PARALLELISM_FACTOR}" -I {} bash -c 'get_pod_logs "$@"' _ {}
|
||||
|
||||
iptables-save > "${BASE_DIR}/iptables"
|
||||
|
||||
cat /var/log/syslog > "${BASE_DIR}/syslog"
|
||||
cat /var/log/armada/bootstrap-armada.log > "${BASE_DIR}/bootstrap-armada.log"
|
||||
|
||||
ip addr show > "${BASE_DIR}/ifconfig"
|
||||
ip route show > "${BASE_DIR}/ip-route"
|
||||
cp -p /etc/resolv.conf "${BASE_DIR}/"
|
||||
|
||||
env | sort --ignore-case > "${BASE_DIR}/environment"
|
||||
docker images > "${BASE_DIR}/docker-images"
|
||||
|
||||
if which calicoctl; then
|
||||
mkdir -p "${CALICO_DIR}"
|
||||
for kind in bgpPeer hostEndpoint ipPool nodes policy profile workloadEndpoint; do
|
||||
calicoctl get "${kind}" -o yaml > "${CALICO_DIR}/${kind}.yaml"
|
||||
done
|
||||
fi
|
||||
|
||||
tar zcf "${SUBDIR_NAME}.tgz" -C "${TEMP_DIR}" "${SUBDIR_NAME}"
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
#!/usr/bin/env bash
|
||||
# Copyright 2019 AT&T Intellectual Property. All other rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
set -ex
|
||||
|
||||
source "${GATE_UTILS}"
|
||||
|
||||
ssh_cmd "${GENESIS_NAME}" apt-get install -y jq
|
||||
|
||||
# Copies script and virtmgr private key to genesis VM
|
||||
ssh_cmd "${GENESIS_NAME}" mkdir -p /root/airship
|
||||
rsync_cmd "${REPO_ROOT}/tools/multi_nodes_gate/airship_gate/bin/debug-report-lite.sh" "${GENESIS_NAME}:/root/airship/"
|
||||
|
||||
set -o pipefail
|
||||
ssh_cmd "${GENESIS_NAME}" /root/airship/debug-report-lite.sh 2>&1 | tee -a "${LOG_FILE}"
|
||||
set +o pipefail
|
||||
|
||||
rsync_cmd "${GENESIS_NAME}:/root/debug-${GENESIS_NAME}.tgz" .
|
||||
|
Loading…
Reference in New Issue