diff --git a/k8s-e2e-external-storage.groovy b/k8s-e2e-external-storage.groovy index d998cc177..76c02e125 100644 --- a/k8s-e2e-external-storage.groovy +++ b/k8s-e2e-external-storage.groovy @@ -181,6 +181,12 @@ node('cico-workspace') { } } + catch (err) { + stage('log system status') { + ssh './system-status.sh' + } + } + finally { stage('return bare-metal machine') { sh 'cico node done ${CICO_SSID}' diff --git a/mini-e2e-helm.groovy b/mini-e2e-helm.groovy index babb86719..1a2f0599d 100644 --- a/mini-e2e-helm.groovy +++ b/mini-e2e-helm.groovy @@ -188,6 +188,12 @@ node('cico-workspace') { } } + catch (err) { + stage('log system status') { + ssh './system-status.sh' + } + } + finally { stage('return bare-metal machine') { sh 'cico node done ${CICO_SSID}' diff --git a/mini-e2e.groovy b/mini-e2e.groovy index 0c3d85497..06c838312 100644 --- a/mini-e2e.groovy +++ b/mini-e2e.groovy @@ -171,6 +171,12 @@ node('cico-workspace') { } } + catch (err) { + stage('log system status') { + ssh './system-status.sh' + } + } + finally { stage('return bare-metal machine') { sh 'cico node done ${CICO_SSID}' diff --git a/system-status.sh b/system-status.sh new file mode 100755 index 000000000..319ac245d --- /dev/null +++ b/system-status.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# +# Run this script to gather details about the environment where the CI job is +# running. This can be helpful to identify issues why minikube failed to +# deploy, or tests encounter problems while running. +# + +function minikube_ssh() { + ssh \ + -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ + -l docker -i "$(minikube ssh-key)" \ + "$(minikube ip)" "${*}" +} + +function log() { + echo "###" + echo "### going to execute: ${*}" + echo "###" + "${@}" + echo "###" + echo "### execution finished: ${*}" + echo "###" +} + +# get the status of the VM in libvirt +log virsh list + +# status of the minikube Kubernetes cluster +log minikube status +log minikube logs + +# get the status of processes in the VM +log minikube_ssh top -b -c -n1 + +# get the logs from the VM +log minikube_ssh journalctl --boot + +# filesystem status for host and VM +log df -h +log minikube_ssh df -h diff --git a/upgrade-tests.groovy b/upgrade-tests.groovy index 9b4464ec7..419036d7b 100644 --- a/upgrade-tests.groovy +++ b/upgrade-tests.groovy @@ -175,6 +175,12 @@ node('cico-workspace') { } } + catch (err) { + stage('log system status') { + ssh './system-status.sh' + } + } + finally { stage('return bare-metal machine') { sh 'cico node done ${CICO_SSID}'