e2e: fetch volume metrics from Kubelet

Test if metrics are available at all. The actual values are a little
difficult to validate.

BlockMode volumes support metrics since Kubernetes 1.22.

See-also: kubernetes/kubernetes#97972
Signed-off-by: Niels de Vos <ndevos@redhat.com>

e2e/node.go

@@ -2,7 +2,9 @@ package e2e
 
 import (
 	"context"
+	"errors"
 
+	core "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
@@ -42,3 +44,20 @@ func checkNodeHasLabel(c kubernetes.Interface, labelKey, labelValue string) erro
 	}
 	return nil
 }
+
+// List all nodes in the cluster (we have one), and return the IP-address.
+// Possibly need to add a selector, pick the node where a Pod is running?
+func getKubeletIP(c kubernetes.Interface) (string, error) {
+	nodes, err := c.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
+	if err != nil {
+		return "", err
+	}
+
+	for _, address := range nodes.Items[0].Status.Addresses {
+		if address.Type == core.NodeInternalIP {
+			return address.Address, nil
+		}
+	}
+
+	return "", errors.New("could not find internal IP for node")
+}

e2e/pvc.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"strings"
 	"time"
 
 	v1 "k8s.io/api/core/v1"
@@ -244,3 +245,44 @@ func checkPVSelectorValuesForPVC(f *framework.Framework, pvc *v1.PersistentVolum
 	}
 	return nil
 }
+
+func getMetricsForPVC(f *framework.Framework, pvc *v1.PersistentVolumeClaim, t int) error {
+	kubelet, err := getKubeletIP(f.ClientSet)
+	if err != nil {
+		return err
+	}
+
+	// kubelet needs to be started with --read-only-port=10255
+	cmd := fmt.Sprintf("curl --silent 'http://%s:10255/metrics'", kubelet)
+
+	// retry as kubelet does not immediately have the metrics available
+	timeout := time.Duration(t) * time.Minute
+	return wait.PollImmediate(poll, timeout, func() (bool, error) {
+		stdOut, stdErr, err := execCommandInToolBoxPod(f, cmd, rookNamespace)
+		if err != nil {
+			e2elog.Logf("failed to get metrics for pvc %q (%v): %v", pvc.Name, err, stdErr)
+			return false, nil
+		}
+		if stdOut == "" {
+			e2elog.Logf("no metrics received from kublet on IP %s", kubelet)
+			return false, nil
+		}
+
+		namespace := fmt.Sprintf("namespace=%q", pvc.Namespace)
+		name := fmt.Sprintf("persistentvolumeclaim=%q", pvc.Name)
+
+		for _, line := range strings.Split(stdOut, "\n") {
+			if !strings.HasPrefix(line, "kubelet_volume_stats_") {
+				continue
+			}
+			if strings.Contains(line, namespace) && strings.Contains(line, name) {
+				// TODO: validate metrics if possible
+				e2elog.Logf("found metrics for pvc %s/%s: %s", pvc.Namespace, pvc.Name, line)
+				return true, nil
+			}
+		}
+
+		e2elog.Logf("no metrics found for pvc %s/%s", pvc.Namespace, pvc.Name)
+		return false, nil
+	})
+}

e2e/utils.go

@@ -302,6 +302,19 @@ func validateNormalUserPVCAccess(pvcPath string, f *framework.Framework) error {
 	if stdErr != "" {
 		return fmt.Errorf("failed to touch a file as non-root user %v", stdErr)
 	}
+
+	// metrics for BlockMode was added in Kubernetes 1.22
+	isBlockMode := false
+	if pvc.Spec.VolumeMode != nil {
+		isBlockMode = (*pvc.Spec.VolumeMode == v1.PersistentVolumeBlock)
+	}
+	if !isBlockMode || k8sVersionGreaterEquals(f.ClientSet, 1, 22) {
+		err = getMetricsForPVC(f, pvc, deployTimeout)
+		if err != nil {
+			return err
+		}
+	}
+
 	err = deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout)
 	if err != nil {
 		return err

scripts/minikube.sh

@@ -165,7 +165,7 @@ fi
 K8S_FEATURE_GATES=${K8S_FEATURE_GATES:-"ExpandCSIVolumes=true"}
 
 #extra-config for kube https://minikube.sigs.k8s.io/docs/reference/configuration/kubernetes/
-EXTRA_CONFIG=${EXTRA_CONFIG:-"--extra-config=apiserver.enable-admission-plugins=PodSecurityPolicy"}
+EXTRA_CONFIG_PSP="--extra-config=apiserver.enable-admission-plugins=PodSecurityPolicy"
 
 # kubelet.resolv-conf needs to point to a file, not a symlink
 # the default minikube VM has /etc/resolv.conf -> /run/systemd/resolve/resolv.conf
@@ -182,6 +182,9 @@ EXTRA_CONFIG="${EXTRA_CONFIG} --extra-config=kubelet.resolv-conf=${RESOLV_CONF}"
 #extra Rook configuration
 ROOK_BLOCK_POOL_NAME=${ROOK_BLOCK_POOL_NAME:-"newrbdpool"}
 
+# enable read-only anonymous access to kubelet metrics
+EXTRA_CONFIG="${EXTRA_CONFIG} --extra-config=kubelet.read-only-port=10255"
+
 if [[ "${KUBE_VERSION}" == "latest" ]]; then
     # update the version string from latest with the real version
     KUBE_VERSION=$(curl -L https://storage.googleapis.com/kubernetes-release/release/stable.txt 2> /dev/null)
@@ -205,16 +208,16 @@ up)
     if minikube_supports_psp; then
         enable_psp
         # shellcheck disable=SC2086
-        ${minikube} start --force --memory="${MEMORY}" --cpus="${CPUS}" -b kubeadm --kubernetes-version="${KUBE_VERSION}" --driver="${VM_DRIVER}" --feature-gates="${K8S_FEATURE_GATES}" --cni="${CNI}" ${EXTRA_CONFIG}
+        ${minikube} start --force --memory="${MEMORY}" --cpus="${CPUS}" -b kubeadm --kubernetes-version="${KUBE_VERSION}" --driver="${VM_DRIVER}" --feature-gates="${K8S_FEATURE_GATES}" --cni="${CNI}" ${EXTRA_CONFIG} ${EXTRA_CONFIG_PSP}
     else
         # This is a workaround to fix psp issues in minikube >1.6.2 and <1.11.0
         # shellcheck disable=SC2086
-        ${minikube} start --force --memory="${MEMORY}" --cpus="${CPUS}" -b kubeadm --kubernetes-version="${KUBE_VERSION}" --driver="${VM_DRIVER}" --feature-gates="${K8S_FEATURE_GATES}" --cni="${CNI}"
+        ${minikube} start --force --memory="${MEMORY}" --cpus="${CPUS}" -b kubeadm --kubernetes-version="${KUBE_VERSION}" --driver="${VM_DRIVER}" --feature-gates="${K8S_FEATURE_GATES}" --cni="${CNI}" ${EXTRA_CONFIG}
         DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
         ${minikube} kubectl -- apply -f "$DIR"/psp.yaml
         ${minikube} stop
         # shellcheck disable=SC2086
-        ${minikube} start --force --memory="${MEMORY}" --cpus="${CPUS}" -b kubeadm --kubernetes-version="${KUBE_VERSION}" --driver="${VM_DRIVER}" --feature-gates="${K8S_FEATURE_GATES}" --cni="${CNI}" ${EXTRA_CONFIG}
+        ${minikube} start --force --memory="${MEMORY}" --cpus="${CPUS}" -b kubeadm --kubernetes-version="${KUBE_VERSION}" --driver="${VM_DRIVER}" --feature-gates="${K8S_FEATURE_GATES}" --cni="${CNI}" ${EXTRA_CONFIG} ${EXTRA_CONFIG_PSP}
     fi
 
     # create a link so the default dataDirHostPath will work for this