summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorKaspars Skels <kaspars.skels@gmail.com>2019-01-31 16:06:31 -0600
committerKaspars Skels <kaspars.skels@gmail.com>2019-02-01 15:45:35 -0600
commit9cd5654b48ba9d28771bec15a7df4d19bd4377dd (patch)
tree2423e561135f9fd6cd3dcac52e0df5641c9c8fcc
parent64a148d6efca5694cd2f6864a7120049100d3c53 (diff)
Add lightweight debug report for gate-multinode
Notes
Notes (review): Code-Review+2: Craig Anderson <craig.anderson@att.com> Code-Review+2: Bryan Strassner <strassner.bryan@gmail.com> Workflow+1: Bryan Strassner <strassner.bryan@gmail.com> Verified+2: Zuul Submitted-by: Zuul Submitted-at: Fri, 01 Feb 2019 22:50:19 +0000 Reviewed-on: https://review.openstack.org/634325 Project: openstack/airship-in-a-bottle Branch: refs/heads/master
-rwxr-xr-xtools/multi_nodes_gate/airship_gate/bin/debug-report-lite.sh105
-rwxr-xr-xtools/multi_nodes_gate/airship_gate/stages/debug-report-lite.sh31
2 files changed, 136 insertions, 0 deletions
diff --git a/tools/multi_nodes_gate/airship_gate/bin/debug-report-lite.sh b/tools/multi_nodes_gate/airship_gate/bin/debug-report-lite.sh
new file mode 100755
index 0000000..396b3c8
--- /dev/null
+++ b/tools/multi_nodes_gate/airship_gate/bin/debug-report-lite.sh
@@ -0,0 +1,105 @@
1#!/usr/bin/env bash
2# Copyright 2019 AT&T Intellectual Property. All other rights reserved.
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16
17set -ux
18
19SUBDIR_NAME=debug-$(hostname)
20
21# NOTE(mark-burnett): This should add calicoctl to the path.
22export PATH=${PATH}:/opt/cni/bin
23
24TEMP_DIR=$(mktemp -d)
25export TEMP_DIR
26export BASE_DIR="${TEMP_DIR}/${SUBDIR_NAME}"
27export HELM_DIR="${BASE_DIR}/helm"
28export CALICO_DIR="${BASE_DIR}/calico"
29
30mkdir -p "${BASE_DIR}"
31
32PARALLELISM_FACTOR=2
33
34function get_namespaces () {
35 kubectl get namespaces -o name | awk -F '/' '{ print $NF }'
36}
37
38function get_pods () {
39 NAMESPACE=$1
40 kubectl get pods -n "${NAMESPACE}" -o name --show-all | awk -F '/' '{ print $NF }' | xargs -L1 -P 1 -I {} echo "${NAMESPACE}" {}
41}
42export -f get_pods
43
44function get_pod_logs () {
45 NAMESPACE=${1% *}
46 POD=${1#* }
47 INIT_CONTAINERS=$(kubectl get pod "${POD}" -n "${NAMESPACE}" -o json | jq -r '.spec.initContainers[]?.name')
48 CONTAINERS=$(kubectl get pod "${POD}" -n "${NAMESPACE}" -o json | jq -r '.spec.containers[].name')
49 POD_DIR="${BASE_DIR}/pod-logs/${NAMESPACE}/${POD}"
50 mkdir -p "${POD_DIR}"
51 for CONTAINER in ${INIT_CONTAINERS} ${CONTAINERS}; do
52 echo "get_pod_logs() [${NAMESPACE}] ${POD} ${CONTAINER}"
53 kubectl logs "${POD}" -n "${NAMESPACE}" -c "${CONTAINER}" > "${POD_DIR}/${CONTAINER}.txt"
54 done
55}
56
57
58export -f get_pod_logs
59
60function get_releases () {
61 helm list --all --short
62}
63
64function get_release () {
65 input=($1)
66 RELEASE=${input[0]}
67 helm status "${RELEASE}" > "${HELM_DIR}/${RELEASE}.txt"
68
69}
70export -f get_release
71
72if which helm; then
73 mkdir -p "${HELM_DIR}"
74 helm list --all > "${HELM_DIR}/list"
75 get_releases | \
76 xargs -r -n 1 -P "${PARALLELISM_FACTOR}" -I {} bash -c 'get_release "$@"' _ {}
77fi
78
79kubectl get --all-namespaces -o wide pods > "${BASE_DIR}/pods.txt"
80
81get_namespaces | \
82 xargs -r -n 1 -P "${PARALLELISM_FACTOR}" -I {} bash -c 'get_pods "$@"' _ {} | \
83 xargs -r -n 2 -P "${PARALLELISM_FACTOR}" -I {} bash -c 'get_pod_logs "$@"' _ {}
84
85iptables-save > "${BASE_DIR}/iptables"
86
87cat /var/log/syslog > "${BASE_DIR}/syslog"
88cat /var/log/armada/bootstrap-armada.log > "${BASE_DIR}/bootstrap-armada.log"
89
90ip addr show > "${BASE_DIR}/ifconfig"
91ip route show > "${BASE_DIR}/ip-route"
92cp -p /etc/resolv.conf "${BASE_DIR}/"
93
94env | sort --ignore-case > "${BASE_DIR}/environment"
95docker images > "${BASE_DIR}/docker-images"
96
97if which calicoctl; then
98 mkdir -p "${CALICO_DIR}"
99 for kind in bgpPeer hostEndpoint ipPool nodes policy profile workloadEndpoint; do
100 calicoctl get "${kind}" -o yaml > "${CALICO_DIR}/${kind}.yaml"
101 done
102fi
103
104tar zcf "${SUBDIR_NAME}.tgz" -C "${TEMP_DIR}" "${SUBDIR_NAME}"
105
diff --git a/tools/multi_nodes_gate/airship_gate/stages/debug-report-lite.sh b/tools/multi_nodes_gate/airship_gate/stages/debug-report-lite.sh
new file mode 100755
index 0000000..f30f022
--- /dev/null
+++ b/tools/multi_nodes_gate/airship_gate/stages/debug-report-lite.sh
@@ -0,0 +1,31 @@
1#!/usr/bin/env bash
2# Copyright 2019 AT&T Intellectual Property. All other rights reserved.
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16set -ex
17
18source "${GATE_UTILS}"
19
20ssh_cmd "${GENESIS_NAME}" apt-get install -y jq
21
22# Copies script and virtmgr private key to genesis VM
23ssh_cmd "${GENESIS_NAME}" mkdir -p /root/airship
24rsync_cmd "${REPO_ROOT}/tools/multi_nodes_gate/airship_gate/bin/debug-report-lite.sh" "${GENESIS_NAME}:/root/airship/"
25
26set -o pipefail
27ssh_cmd "${GENESIS_NAME}" /root/airship/debug-report-lite.sh 2>&1 | tee -a "${LOG_FILE}"
28set +o pipefail
29
30rsync_cmd "${GENESIS_NAME}:/root/debug-${GENESIS_NAME}.tgz" .
31