ci: run system-status.sh in case a job fails

The new `system-status.sh` script logs the status of the host and the
minikube VM. This gets executed when a CI job fails, and should aid in
troubleshooting spurious failures.

Updates: #1969
Signed-off-by: Niels de Vos <ndevos@redhat.com>

k8s-e2e-external-storage.groovy

@@ -181,6 +181,12 @@ node('cico-workspace') {
 		}
 	}
 
+	catch (err) {
+		stage('log system status') {
+			ssh './system-status.sh'
+		}
+	}
+
 	finally {
 		stage('return bare-metal machine') {
 			sh 'cico node done ${CICO_SSID}'

mini-e2e-helm.groovy

@@ -188,6 +188,12 @@ node('cico-workspace') {
 		}
 	}
 
+	catch (err) {
+		stage('log system status') {
+			ssh './system-status.sh'
+		}
+	}
+
 	finally {
 		stage('return bare-metal machine') {
 			sh 'cico node done ${CICO_SSID}'

mini-e2e.groovy

@@ -171,6 +171,12 @@ node('cico-workspace') {
 		}
 	}
 
+	catch (err) {
+		stage('log system status') {
+			ssh './system-status.sh'
+		}
+	}
+
 	finally {
 		stage('return bare-metal machine') {
 			sh 'cico node done ${CICO_SSID}'

system-status.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+#
+# Run this script to gather details about the environment where the CI job is
+# running. This can be helpful to identify issues why minikube failed to
+# deploy, or tests encounter problems while running.
+#
+
+function minikube_ssh() {
+    ssh \
+        -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
+        -l docker -i "$(minikube ssh-key)" \
+        "$(minikube ip)" "${*}"
+}
+
+function log() {
+    echo "###"
+    echo "### going to execute: ${*}"
+    echo "###"
+    "${@}"
+    echo "###"
+    echo "### execution finished: ${*}"
+    echo "###"
+}
+
+# get the status of the VM in libvirt
+log virsh list
+
+# status of the minikube Kubernetes cluster
+log minikube status
+log minikube logs
+
+# get the status of processes in the VM
+log minikube_ssh top -b -c -n1
+
+# get the logs from the VM
+log minikube_ssh journalctl --boot
+
+# filesystem status for host and VM
+log df -h
+log minikube_ssh df -h

upgrade-tests.groovy

@@ -175,6 +175,12 @@ node('cico-workspace') {
 		}
 	}
 
+	catch (err) {
+		stage('log system status') {
+			ssh './system-status.sh'
+		}
+	}
+
 	finally {
 		stage('return bare-metal machine') {
 			sh 'cico node done ${CICO_SSID}'