e2e: fetch volume metrics from Kubelet

Test if metrics are available at all. The actual values are a little
difficult to validate.

BlockMode volumes support metrics since Kubernetes 1.22.

See-also: kubernetes/kubernetes#97972
Signed-off-by: Niels de Vos <ndevos@redhat.com>

e2e/node.go

@@ -2,7 +2,9 @@ package e2e
 
 import (
 	"context"
+	"errors"
 
+	core "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
@@ -42,3 +44,20 @@ func checkNodeHasLabel(c kubernetes.Interface, labelKey, labelValue string) erro
 	}
 	return nil
 }
+
+// List all nodes in the cluster (we have one), and return the IP-address.
+// Possibly need to add a selector, pick the node where a Pod is running?
+func getKubeletIP(c kubernetes.Interface) (string, error) {
+	nodes, err := c.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
+	if err != nil {
+		return "", err
+	}
+
+	for _, address := range nodes.Items[0].Status.Addresses {
+		if address.Type == core.NodeInternalIP {
+			return address.Address, nil
+		}
+	}
+
+	return "", errors.New("could not find internal IP for node")
+}

e2e/pvc.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"strings"
 	"time"
 
 	v1 "k8s.io/api/core/v1"
@@ -244,3 +245,44 @@ func checkPVSelectorValuesForPVC(f *framework.Framework, pvc *v1.PersistentVolum
 	}
 	return nil
 }
+
+func getMetricsForPVC(f *framework.Framework, pvc *v1.PersistentVolumeClaim, t int) error {
+	kubelet, err := getKubeletIP(f.ClientSet)
+	if err != nil {
+		return err
+	}
+
+	// kubelet needs to be started with --read-only-port=10255
+	cmd := fmt.Sprintf("curl --silent 'http://%s:10255/metrics'", kubelet)
+
+	// retry as kubelet does not immediately have the metrics available
+	timeout := time.Duration(t) * time.Minute
+	return wait.PollImmediate(poll, timeout, func() (bool, error) {
+		stdOut, stdErr, err := execCommandInToolBoxPod(f, cmd, rookNamespace)
+		if err != nil {
+			e2elog.Logf("failed to get metrics for pvc %q (%v): %v", pvc.Name, err, stdErr)
+			return false, nil
+		}
+		if stdOut == "" {
+			e2elog.Logf("no metrics received from kublet on IP %s", kubelet)
+			return false, nil
+		}
+
+		namespace := fmt.Sprintf("namespace=%q", pvc.Namespace)
+		name := fmt.Sprintf("persistentvolumeclaim=%q", pvc.Name)
+
+		for _, line := range strings.Split(stdOut, "\n") {
+			if !strings.HasPrefix(line, "kubelet_volume_stats_") {
+				continue
+			}
+			if strings.Contains(line, namespace) && strings.Contains(line, name) {
+				// TODO: validate metrics if possible
+				e2elog.Logf("found metrics for pvc %s/%s: %s", pvc.Namespace, pvc.Name, line)
+				return true, nil
+			}
+		}
+
+		e2elog.Logf("no metrics found for pvc %s/%s", pvc.Namespace, pvc.Name)
+		return false, nil
+	})
+}

e2e/utils.go

@@ -302,6 +302,19 @@ func validateNormalUserPVCAccess(pvcPath string, f *framework.Framework) error {
 	if stdErr != "" {
 		return fmt.Errorf("failed to touch a file as non-root user %v", stdErr)
 	}
+
+	// metrics for BlockMode was added in Kubernetes 1.22
+	isBlockMode := false
+	if pvc.Spec.VolumeMode != nil {
+		isBlockMode = (*pvc.Spec.VolumeMode == v1.PersistentVolumeBlock)
+	}
+	if !isBlockMode || k8sVersionGreaterEquals(f.ClientSet, 1, 22) {
+		err = getMetricsForPVC(f, pvc, deployTimeout)
+		if err != nil {
+			return err
+		}
+	}
+
 	err = deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout)
 	if err != nil {
 		return err