ceph-csi/system-status.sh

76 lines
2.3 KiB
Bash
Raw Normal View History

#!/bin/bash
#
# Run this script to gather details about the environment where the CI job is
# running. This can be helpful to identify issues why minikube failed to
# deploy, or tests encounter problems while running.
#
function minikube_ssh() {
ssh \
-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
-l docker -i "$(minikube ssh-key)" \
"$(minikube ip)" "${*}"
}
function log() {
echo "###" >/dev/stderr
echo "### going to execute: ${*}" >/dev/stderr
echo "###" >/dev/stderr
"${@}"
echo "###" >/dev/stderr
echo "### execution finished: ${*}" >/dev/stderr
echo "###" >/dev/stderr
}
# get the logs from the host
log journalctl --boot
# get the status of the VM in libvirt
log virsh list
# status of the minikube Kubernetes cluster
log minikube status
log minikube logs
# get the status of processes in the VM
log minikube_ssh top -b -c -n1 -w
# get the logs from the VM
log minikube_ssh sudo journalctl --boot
# get kubelet status
log minikube_ssh sudo systemctl status kubelet
log minikube_ssh sudo journalctl -xeu kubelet
# filesystem status for host and VM
log df -hT
log minikube_ssh df -hT
# get the rbd-nbd logs from the VM and write them to stdout
log minikube_ssh "sudo find /var/log/ceph/ -type f | xargs sudo tail -n +1"
# fetch all logs from /var/lib/rook in the VM and write them to stdout
log minikube_ssh sudo tar c /var/lib/rook | tar xvO
# gets status from all namespaces
for NAMESPACE in $(kubectl get namespaces -o jsonpath='{.items[*].metadata.name}')
do
log kubectl describe namespace "${NAMESPACE}"
log kubectl -n "${NAMESPACE}" get events
log kubectl -n "${NAMESPACE}" get pods
for POD in $(kubectl -n "${NAMESPACE}" get pods -o jsonpath='{.items[*].metadata.name}')
do
log kubectl -n "${NAMESPACE}" describe pod "${POD}"
log kubectl -n "${NAMESPACE}" logs "${POD}" --all-containers
log kubectl -n "${NAMESPACE}" logs "${POD}" --all-containers --previous=true
done
done
log kubectl -n rook-ceph describe CephCluster
log kubectl -n rook-ceph describe CephBlockPool
log kubectl -n rook-ceph describe CephFilesystem
# run "ceph -s" in the toolbox
log kubectl -n rook-ceph exec \
"$(kubectl -n rook-ceph get pod -l app=rook-ceph-tools -o jsonpath='{.items[0].metadata.name}')" \
-- ceph -s