e2e: add framework for volumegroupsnapshot

adding a framework to test the volumegroupsnapshot
for both cephfs and rbd and added a test case
for cephfs.

Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>

e2e/cephfs.go

@@ -2478,6 +2478,19 @@ var _ = Describe(cephfsType, func() {
 				}
 			})
 
+			By("test volumeGroupSnapshot", func() {
+				scName := "csi-cephfs-sc"
+				snapshotter, err := newCephFSVolumeGroupSnapshot(f, f.UniqueName, scName, false, deployTimeout, 3)
+				if err != nil {
+					framework.Failf("failed to create volumeGroupSnapshot Base: %v", err)
+				}
+
+				err = snapshotter.TestVolumeGroupSnapshot()
+				if err != nil {
+					framework.Failf("failed to test volumeGroupSnapshot: %v", err)
+				}
+			})
+
 			// FIXME: in case NFS testing is done, prevent deletion
 			// of the CephFS filesystem and related pool. This can
 			// probably be addressed in a nicer way, making sure

e2e/utils.go

@@ -52,8 +52,9 @@ const (
 	rbdType    = "rbd"
 	cephfsType = "cephfs"
 
-	volumesType = "volumes"
+	volumesType    = "volumes"
-	snapsType   = "snaps"
+	snapsType      = "snaps"
+	groupSnapsType = "groupsnaps"
 
 	rookToolBoxPodLabel = "app=rook-ceph-tools"
 	rbdMountOptions     = "mountOptions"
@@ -174,17 +175,20 @@ func validateOmapCount(f *framework.Framework, count int, driver, pool, mode str
 			volumeMode: volumesType,
 			driverType: cephfsType,
 			radosLsCmd: fmt.Sprintf("rados ls --pool=%s --namespace csi", pool),
-			radosLsCmdFilter: fmt.Sprintf("rados ls --pool=%s --namespace csi | grep -v default | grep -c ^csi.volume.",
+			radosLsCmdFilter: fmt.Sprintf(
+				"rados ls --pool=%s --namespace csi | grep -v default | grep -v csi.volume.group. | grep -c ^csi.volume.",
 				pool),
 			radosLsKeysCmd: fmt.Sprintf("rados listomapkeys csi.volumes.default --pool=%s --namespace csi", pool),
-			radosLsKeysCmdFilter: fmt.Sprintf("rados listomapkeys csi.volumes.default --pool=%s --namespace csi|wc -l",
+			radosLsKeysCmdFilter: fmt.Sprintf("rados listomapkeys csi.volumes.default --pool=%s --namespace csi | wc -l",
 				pool),
 		},
 		{
-			volumeMode:           volumesType,
+			volumeMode: volumesType,
-			driverType:           rbdType,
+			driverType: rbdType,
-			radosLsCmd:           "rados ls " + rbdOptions(pool),
+			radosLsCmd: "rados ls " + rbdOptions(pool),
-			radosLsCmdFilter:     fmt.Sprintf("rados ls %s | grep -v default | grep -c ^csi.volume.", rbdOptions(pool)),
+			radosLsCmdFilter: fmt.Sprintf(
+				"rados ls %s | grep -v default | grep -v csi.volume.group. |  grep -c ^csi.volume.",
+				rbdOptions(pool)),
 			radosLsKeysCmd:       "rados listomapkeys csi.volumes.default " + rbdOptions(pool),
 			radosLsKeysCmdFilter: fmt.Sprintf("rados listomapkeys csi.volumes.default %s | wc -l", rbdOptions(pool)),
 		},
@@ -195,7 +199,7 @@ func validateOmapCount(f *framework.Framework, count int, driver, pool, mode str
 			radosLsCmdFilter: fmt.Sprintf("rados ls --pool=%s --namespace csi | grep -v default | grep -c ^csi.snap.",
 				pool),
 			radosLsKeysCmd: fmt.Sprintf("rados listomapkeys csi.snaps.default --pool=%s --namespace csi", pool),
-			radosLsKeysCmdFilter: fmt.Sprintf("rados listomapkeys csi.snaps.default --pool=%s --namespace csi|wc -l",
+			radosLsKeysCmdFilter: fmt.Sprintf("rados listomapkeys csi.snaps.default --pool=%s --namespace csi | wc -l",
 				pool),
 		},
 		{
@@ -206,6 +210,16 @@ func validateOmapCount(f *framework.Framework, count int, driver, pool, mode str
 			radosLsKeysCmd:       "rados listomapkeys csi.snaps.default " + rbdOptions(pool),
 			radosLsKeysCmdFilter: fmt.Sprintf("rados listomapkeys csi.snaps.default %s | wc -l", rbdOptions(pool)),
 		},
+		{
+			volumeMode: groupSnapsType,
+			driverType: cephfsType,
+			radosLsCmd: fmt.Sprintf("rados ls --pool=%s --namespace csi", pool),
+			radosLsCmdFilter: fmt.Sprintf("rados ls --pool=%s --namespace csi | grep -v default | grep -c ^csi.volume.group.",
+				pool),
+			radosLsKeysCmd: fmt.Sprintf("rados listomapkeys csi.groups.default --pool=%s --namespace csi", pool),
+			radosLsKeysCmdFilter: fmt.Sprintf("rados listomapkeys csi.groups.default --pool=%s --namespace csi | wc -l",
+				pool),
+		},
 	}
 
 	for _, cmds := range radosListCommands {
@@ -228,7 +242,7 @@ func validateOmapCount(f *framework.Framework, count int, driver, pool, mode str
 			if err == nil {
 				continue
 			}
-			saveErr := err
+			saveErr := fmt.Errorf("failed to validate omap count for %s: %w", cmd, err)
 			if strings.Contains(err.Error(), "expected omap object count") {
 				stdOut, stdErr, err = execCommandInToolBoxPod(f, filterLessCmds[i], rookNamespace)
 				if err == nil {

e2e/volumegroupsnapshot.go

@@ -0,0 +1,119 @@
+/*
+Copyright 2024 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"context"
+	"fmt"
+
+	groupsnapapi "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1alpha1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/kubernetes/test/e2e/framework"
+)
+
+type cephFSVolumeGroupSnapshot struct {
+	*volumeGroupSnapshotterBase
+}
+
+var _ VolumeGroupSnapshotter = &cephFSVolumeGroupSnapshot{}
+
+func newCephFSVolumeGroupSnapshot(f *framework.Framework, namespace,
+	storageClass string,
+	blockPVC bool,
+	timeout, totalPVCCount int,
+) (VolumeGroupSnapshotter, error) {
+	base, err := newVolumeGroupSnapshotBase(f, namespace, storageClass, blockPVC, timeout, totalPVCCount)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create volumeGroupSnapshotterBase: %w", err)
+	}
+
+	return &cephFSVolumeGroupSnapshot{
+		volumeGroupSnapshotterBase: base,
+	}, nil
+}
+
+func (c *cephFSVolumeGroupSnapshot) TestVolumeGroupSnapshot() error {
+	return c.volumeGroupSnapshotterBase.testVolumeGroupSnapshot(c)
+}
+
+func (c *cephFSVolumeGroupSnapshot) GetVolumeGroupSnapshotClass() (*groupsnapapi.VolumeGroupSnapshotClass, error) {
+	vgscPath := fmt.Sprintf("%s/%s", cephFSExamplePath, "groupsnapshotclass.yaml")
+	vgsc := &groupsnapapi.VolumeGroupSnapshotClass{}
+	err := unmarshal(vgscPath, vgsc)
+	if err != nil {
+		return nil, fmt.Errorf("failed to unmarshal VolumeGroupSnapshotClass: %w", err)
+	}
+
+	vgsc.Parameters["csi.storage.k8s.io/group-snapshotter-secret-namespace"] = cephCSINamespace
+	vgsc.Parameters["csi.storage.k8s.io/group-snapshotter-secret-name"] = cephFSProvisionerSecretName
+	vgsc.Parameters["fsName"] = fileSystemName
+
+	fsID, err := getClusterID(c.framework)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get clusterID: %w", err)
+	}
+	vgsc.Parameters["clusterID"] = fsID
+
+	return vgsc, nil
+}
+
+func (c *cephFSVolumeGroupSnapshot) ValidateResourcesForCreate(vgs *groupsnapapi.VolumeGroupSnapshot) error {
+	ctx := context.TODO()
+	metadataPool, err := getCephFSMetadataPoolName(c.framework, fileSystemName)
+	if err != nil {
+		return fmt.Errorf("failed getting cephFS metadata pool name: %w", err)
+	}
+
+	sourcePVCCount := len(vgs.Status.PVCVolumeSnapshotRefList)
+	// we are creating clones for each source PVC
+	clonePVCCount := len(vgs.Status.PVCVolumeSnapshotRefList)
+	totalPVCCount := sourcePVCCount + clonePVCCount
+	validateSubvolumeCount(c.framework, totalPVCCount, fileSystemName, subvolumegroup)
+
+	// we are creating 1 snapshot for each source PVC, validate the snapshot count
+	for _, pvcSnap := range vgs.Status.PVCVolumeSnapshotRefList {
+		pvc, err := c.framework.ClientSet.CoreV1().PersistentVolumeClaims(vgs.Namespace).Get(ctx,
+			pvcSnap.PersistentVolumeClaimRef.Name,
+			metav1.GetOptions{})
+		if err != nil {
+			return fmt.Errorf("failed to get PVC: %w", err)
+		}
+		pv := pvc.Spec.VolumeName
+		pvObj, err := c.framework.ClientSet.CoreV1().PersistentVolumes().Get(ctx, pv, metav1.GetOptions{})
+		if err != nil {
+			return fmt.Errorf("failed to get PV: %w", err)
+		}
+		validateCephFSSnapshotCount(c.framework, 1, subvolumegroup, pvObj)
+	}
+	validateOmapCount(c.framework, totalPVCCount, cephfsType, metadataPool, volumesType)
+	validateOmapCount(c.framework, sourcePVCCount, cephfsType, metadataPool, snapsType)
+	validateOmapCount(c.framework, 1, cephfsType, metadataPool, groupSnapsType)
+
+	return nil
+}
+
+func (c *cephFSVolumeGroupSnapshot) ValidateResourcesForDelete() error {
+	metadataPool, err := getCephFSMetadataPoolName(c.framework, fileSystemName)
+	if err != nil {
+		return fmt.Errorf("failed getting cephFS metadata pool name: %w", err)
+	}
+	validateOmapCount(c.framework, 0, cephfsType, metadataPool, volumesType)
+	validateOmapCount(c.framework, 0, cephfsType, metadataPool, snapsType)
+	validateOmapCount(c.framework, 0, cephfsType, metadataPool, groupSnapsType)
+
+	return nil
+}

e2e/volumegroupsnapshot_base.go

@@ -0,0 +1,432 @@
+/*
+Copyright 2024 The Ceph-CSI Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	groupsnapapi "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumegroupsnapshot/v1alpha1"
+	snapapi "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1"
+	groupsnapclient "github.com/kubernetes-csi/external-snapshotter/client/v8/clientset/versioned/typed/volumegroupsnapshot/v1alpha1"
+	v1 "k8s.io/api/core/v1"
+	apierrs "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/kubernetes/test/e2e/framework"
+)
+
+// volumeGroupSnapshotter defines the common operations for handling
+// volume group snapshots.
+type volumeGroupSnapshotter interface {
+	// CreateVolumeGroupSnapshotClass creates a volume group snapshot class.
+	CreateVolumeGroupSnapshotClass(vgsc *groupsnapapi.VolumeGroupSnapshotClass) error
+	// CreateVolumeGroupSnapshot creates a groupsnapshot with the specified name
+	// namespace and volume group snapshot class.
+	CreateVolumeGroupSnapshot(name,
+		volumeGroupSnapshotClassName string,
+		labels map[string]string) (*groupsnapapi.VolumeGroupSnapshot, error)
+	// DeleteVolumeGroupSnapshot deletes the specified volume
+	// group snapshot.
+	DeleteVolumeGroupSnapshot(volumeGroupSnapshotName string) error
+	// DeleteVolumeGroupSnapshotClass deletes the specified volume
+	// group snapshot class.
+	DeleteVolumeGroupSnapshotClass(snapshotClassName string) error
+	// CreatePVCs creates PVCs with the specified namespace and labels.
+	CreatePVCs(namespace string,
+		labels map[string]string) ([]*v1.PersistentVolumeClaim, error)
+	// DeletePVCs deletes the specified PVCs.
+	DeletePVCs(pvcs []*v1.PersistentVolumeClaim) error
+	// CreatePVCClones creates pvcs from all the snapshots in VolumeGroupSnapshot.
+	CreatePVCClones(vgs *groupsnapapi.VolumeGroupSnapshot,
+	) ([]*v1.PersistentVolumeClaim, error)
+}
+
+// VolumeGroupSnapshotter defines validation operations specific to each driver.
+type VolumeGroupSnapshotter interface {
+	// TestVolumeGroupSnapshot tests the volume group snapshot operations.
+	TestVolumeGroupSnapshot() error
+	// GetVolumeGroupSnapshotClass returns the volume group snapshot class.
+	GetVolumeGroupSnapshotClass() (*groupsnapapi.VolumeGroupSnapshotClass, error)
+	// ValidateResourcesForCreate validates the resources in the backend after
+	// creating clones.
+	ValidateResourcesForCreate(vgs *groupsnapapi.VolumeGroupSnapshot) error
+	// ValidateSnapshotsDeleted checks if all resources are deleted in the
+	// backend after all the resources are deleted.
+	ValidateResourcesForDelete() error
+}
+
+type volumeGroupSnapshotterBase struct {
+	timeout          int
+	framework        *framework.Framework
+	groupclient      *groupsnapclient.GroupsnapshotV1alpha1Client
+	storageClassName string
+	blockPVC         bool
+	totalPVCCount    int
+	namespace        string
+}
+
+func newVolumeGroupSnapshotBase(f *framework.Framework, namespace,
+	storageClass string,
+	blockPVC bool,
+	timeout, totalPVCCount int,
+) (*volumeGroupSnapshotterBase, error) {
+	config, err := framework.LoadConfig()
+	if err != nil {
+		return nil, fmt.Errorf("error creating group snapshot client: %w", err)
+	}
+	c, err := groupsnapclient.NewForConfig(config)
+	if err != nil {
+		return nil, fmt.Errorf("error creating group snapshot client: %w", err)
+	}
+
+	return &volumeGroupSnapshotterBase{
+		framework:        f,
+		groupclient:      c,
+		namespace:        namespace,
+		storageClassName: storageClass,
+		blockPVC:         blockPVC,
+		timeout:          timeout,
+		totalPVCCount:    totalPVCCount,
+	}, err
+}
+
+var _ volumeGroupSnapshotter = &volumeGroupSnapshotterBase{}
+
+func (v *volumeGroupSnapshotterBase) CreatePVCs(namespace string,
+	labels map[string]string,
+) ([]*v1.PersistentVolumeClaim, error) {
+	pvcs := make([]*v1.PersistentVolumeClaim, v.totalPVCCount)
+	for i := range v.totalPVCCount {
+		pvcs[i] = &v1.PersistentVolumeClaim{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      fmt.Sprintf("pvc-%d", i),
+				Namespace: namespace,
+			},
+			Spec: v1.PersistentVolumeClaimSpec{
+				Resources: v1.VolumeResourceRequirements{
+					Requests: v1.ResourceList{
+						v1.ResourceStorage: resource.MustParse("1Gi"),
+					},
+				},
+				AccessModes:      []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},
+				StorageClassName: &v.storageClassName,
+			},
+		}
+		if v.blockPVC {
+			volumeMode := v1.PersistentVolumeBlock
+			pvcs[i].Spec.VolumeMode = &volumeMode
+		} else {
+			volumeMode := v1.PersistentVolumeFilesystem
+			pvcs[i].Spec.VolumeMode = &volumeMode
+		}
+		pvcs[i].Labels = labels
+		err := createPVCAndvalidatePV(v.framework.ClientSet, pvcs[i], v.timeout)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create PVC: %w", err)
+		}
+	}
+
+	return pvcs, nil
+}
+
+func (v *volumeGroupSnapshotterBase) DeletePVCs(pvcs []*v1.PersistentVolumeClaim) error {
+	for _, pvc := range pvcs {
+		err := deletePVCAndValidatePV(v.framework.ClientSet, pvc, v.timeout)
+		if err != nil {
+			return fmt.Errorf("failed to delete PVC: %w", err)
+		}
+	}
+
+	return nil
+}
+
+func (v *volumeGroupSnapshotterBase) CreatePVCClones(
+	vgs *groupsnapapi.VolumeGroupSnapshot,
+) ([]*v1.PersistentVolumeClaim, error) {
+	pvcSnapRef := vgs.Status.PVCVolumeSnapshotRefList
+	namespace := vgs.Namespace
+	ctx := context.TODO()
+	pvcs := make([]*v1.PersistentVolumeClaim, len(pvcSnapRef))
+	for i, pvcSnap := range pvcSnapRef {
+		pvc, err := v.framework.ClientSet.CoreV1().PersistentVolumeClaims(namespace).Get(ctx,
+			pvcSnap.PersistentVolumeClaimRef.Name,
+			metav1.GetOptions{})
+		if err != nil {
+			return nil, fmt.Errorf("failed to get PVC: %w", err)
+		}
+		pvcs[i] = &v1.PersistentVolumeClaim{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      fmt.Sprintf("%s-clone-%d", pvc.Name, i),
+				Namespace: pvc.Namespace,
+			},
+			Spec: *pvc.Spec.DeepCopy(),
+		}
+
+		snap := pvcSnap.VolumeSnapshotRef
+		apiGroup := snapapi.GroupName
+		pvcs[i].Spec.DataSource = &v1.TypedLocalObjectReference{
+			APIGroup: &apiGroup,
+			Kind:     "VolumeSnapshot",
+			Name:     snap.Name,
+		}
+		pvcs[i].Spec.StorageClassName = &v.storageClassName
+		// cleanup the VolumeName as we are creating a new PVC
+		pvcs[i].Spec.VolumeName = ""
+
+		err = createPVCAndvalidatePV(v.framework.ClientSet, pvcs[i], v.timeout)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create PVC: %w", err)
+		}
+	}
+
+	return pvcs, nil
+}
+
+func (v volumeGroupSnapshotterBase) CreateVolumeGroupSnapshotClass(
+	groupSnapshotClass *groupsnapapi.VolumeGroupSnapshotClass,
+) error {
+	return wait.PollUntilContextTimeout(
+		context.TODO(),
+		poll,
+		time.Duration(v.timeout)*time.Minute,
+		true,
+		func(ctx context.Context) (bool, error) {
+			_, err := v.groupclient.VolumeGroupSnapshotClasses().Create(ctx, groupSnapshotClass, metav1.CreateOptions{})
+			if err != nil {
+				framework.Logf("error creating VolumeGroupSnapshotClass %q: %v", groupSnapshotClass.Name, err)
+				if isRetryableAPIError(err) {
+					return false, nil
+				}
+
+				return false, fmt.Errorf("failed to create VolumeGroupSnapshotClass %q: %w", groupSnapshotClass.Name, err)
+			}
+
+			return true, nil
+		})
+}
+
+func (v volumeGroupSnapshotterBase) CreateVolumeGroupSnapshot(name,
+	volumeGroupSnapshotClassName string, labels map[string]string,
+) (*groupsnapapi.VolumeGroupSnapshot, error) {
+	namespace := v.namespace
+	groupSnapshot := &groupsnapapi.VolumeGroupSnapshot{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: namespace,
+		},
+		Spec: groupsnapapi.VolumeGroupSnapshotSpec{
+			Source: groupsnapapi.VolumeGroupSnapshotSource{
+				Selector: &metav1.LabelSelector{
+					MatchLabels: labels,
+				},
+			},
+			VolumeGroupSnapshotClassName: &volumeGroupSnapshotClassName,
+		},
+	}
+	ctx := context.TODO()
+	_, err := v.groupclient.VolumeGroupSnapshots(namespace).Create(ctx, groupSnapshot, metav1.CreateOptions{})
+	if err != nil {
+		return nil, fmt.Errorf("failed to create VolumeGroupSnapshot %q: %w", name, err)
+	}
+
+	framework.Logf("VolumeGroupSnapshot with name %v created in %v namespace", name, namespace)
+
+	timeout := time.Duration(v.timeout) * time.Minute
+	start := time.Now()
+	framework.Logf("waiting for %+v to be in ready state", groupSnapshot)
+
+	err = wait.PollUntilContextTimeout(ctx, poll, timeout, true, func(ctx context.Context) (bool, error) {
+		framework.Logf("waiting for VolumeGroupSnapshot %s (%d seconds elapsed)", name, int(time.Since(start).Seconds()))
+		groupSnapshot, err = v.groupclient.VolumeGroupSnapshots(namespace).
+			Get(ctx, name, metav1.GetOptions{})
+		if err != nil {
+			framework.Logf("Error getting VolumeGroupSnapshot in namespace: '%s': %v", namespace, err)
+			if isRetryableAPIError(err) {
+				return false, nil
+			}
+			if apierrs.IsNotFound(err) {
+				return false, nil
+			}
+
+			return false, fmt.Errorf("failed to get volumesnapshot: %w", err)
+		}
+		if groupSnapshot.Status == nil || groupSnapshot.Status.ReadyToUse == nil {
+			return false, nil
+		}
+
+		if *groupSnapshot.Status.ReadyToUse {
+			return true, nil
+		}
+
+		readyToUse := groupSnapshot.Status.ReadyToUse
+		errMsg := ""
+		if groupSnapshot.Status.Error != nil {
+			errMsg = *groupSnapshot.Status.Error.Message
+		}
+
+		framework.Logf("current state of VolumeGroupSnapshot %s. ReadyToUse: %v, Error: %s", name, *readyToUse, errMsg)
+
+		return false, nil
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to get VolumeGroupSnapshot %s: %w", name, err)
+	}
+
+	return groupSnapshot, nil
+}
+
+func (v volumeGroupSnapshotterBase) DeleteVolumeGroupSnapshot(volumeGroupSnapshotName string) error {
+	namespace := v.namespace
+	ctx := context.TODO()
+	err := v.groupclient.VolumeGroupSnapshots(namespace).Delete(
+		ctx,
+		volumeGroupSnapshotName,
+		metav1.DeleteOptions{})
+	if err != nil {
+		return fmt.Errorf("failed to delete VolumeGroupSnapshot: %w", err)
+	}
+	start := time.Now()
+	framework.Logf("Waiting for VolumeGroupSnapshot %v to be deleted", volumeGroupSnapshotName)
+	timeout := time.Duration(v.timeout) * time.Minute
+
+	return wait.PollUntilContextTimeout(
+		ctx,
+		poll,
+		timeout,
+		true,
+		func(ctx context.Context) (bool, error) {
+			_, err := v.groupclient.VolumeGroupSnapshots(namespace).Get(ctx, volumeGroupSnapshotName, metav1.GetOptions{})
+			if err != nil {
+				if isRetryableAPIError(err) {
+					return false, nil
+				}
+				if apierrs.IsNotFound(err) {
+					return true, nil
+				}
+				framework.Logf("%s VolumeGroupSnapshot to be deleted (%d seconds elapsed)",
+					volumeGroupSnapshotName,
+					int(time.Since(start).Seconds()))
+
+				return false, fmt.Errorf("failed to get VolumeGroupSnapshot: %w", err)
+			}
+
+			return false, nil
+		})
+}
+
+func (v volumeGroupSnapshotterBase) DeleteVolumeGroupSnapshotClass(groupSnapshotClassName string) error {
+	ctx := context.TODO()
+	err := v.groupclient.VolumeGroupSnapshotClasses().Delete(
+		ctx, groupSnapshotClassName, metav1.DeleteOptions{})
+	if err != nil {
+		return fmt.Errorf("failed to delete VolumeGroupSnapshotClass: %w", err)
+	}
+	start := time.Now()
+	framework.Logf("Waiting for VolumeGroupSnapshotClass %v to be deleted", groupSnapshotClassName)
+	timeout := time.Duration(v.timeout) * time.Minute
+
+	return wait.PollUntilContextTimeout(
+		ctx,
+		poll,
+		timeout,
+		true,
+		func(ctx context.Context) (bool, error) {
+			_, err := v.groupclient.VolumeGroupSnapshotClasses().Get(ctx, groupSnapshotClassName, metav1.GetOptions{})
+			if err != nil {
+				if isRetryableAPIError(err) {
+					return false, nil
+				}
+				if apierrs.IsNotFound(err) {
+					return true, nil
+				}
+				framework.Logf("%s VolumeGroupSnapshotClass to be deleted (%d seconds elapsed)",
+					groupSnapshotClassName,
+					int(time.Since(start).Seconds()))
+
+				return false, fmt.Errorf("failed to get VolumeGroupSnapshotClass: %w", err)
+			}
+
+			return false, nil
+		})
+}
+
+func (v *volumeGroupSnapshotterBase) testVolumeGroupSnapshot(vol VolumeGroupSnapshotter) error {
+	pvcLabels := map[string]string{"pvc": "vgsc"}
+	pvcs, err := v.CreatePVCs(v.namespace, pvcLabels)
+	if err != nil {
+		return fmt.Errorf("failed to create PVCs: %w", err)
+	}
+
+	vgsc, err := vol.GetVolumeGroupSnapshotClass()
+	if err != nil {
+		return fmt.Errorf("failed to get volume group snapshot class: %w", err)
+	}
+	// Create a volume group snapshot class
+	vgscName := v.framework.Namespace.Name + "-vgsc"
+	vgsc.Name = vgscName
+	err = v.CreateVolumeGroupSnapshotClass(vgsc)
+	if err != nil {
+		return fmt.Errorf("failed to create volume group snapshot: %w", err)
+	}
+	vgsName := v.framework.Namespace.Name + "-vgs"
+	// Create a volume group snapshot
+	volumeGroupSnapshot, err := v.CreateVolumeGroupSnapshot(vgsName, vgscName, pvcLabels)
+	if err != nil {
+		return fmt.Errorf("failed to create volume group snapshot: %w", err)
+	}
+
+	clonePVCs, err := v.CreatePVCClones(volumeGroupSnapshot)
+	if err != nil {
+		return fmt.Errorf("failed to create clones: %w", err)
+	}
+	// validate the resources in the backend
+	err = vol.ValidateResourcesForCreate(volumeGroupSnapshot)
+	if err != nil {
+		return fmt.Errorf("failed to validate resources for create: %w", err)
+	}
+
+	// Delete the clones
+	err = v.DeletePVCs(clonePVCs)
+	if err != nil {
+		return fmt.Errorf("failed to delete clones: %w", err)
+	}
+	// Delete the PVCs
+	err = v.DeletePVCs(pvcs)
+	if err != nil {
+		return fmt.Errorf("failed to delete PVCs: %w", err)
+	}
+	// Delete the volume group snapshot
+	err = v.DeleteVolumeGroupSnapshot(volumeGroupSnapshot.Name)
+	if err != nil {
+		return fmt.Errorf("failed to delete volume group snapshot: %w", err)
+	}
+	// validate the resources in the backend after deleting the resources
+	err = vol.ValidateResourcesForDelete()
+	if err != nil {
+		return fmt.Errorf("failed to validate resources for delete: %w", err)
+	}
+	// Delete the volume group snapshot class
+	err = v.DeleteVolumeGroupSnapshotClass(vgscName)
+	if err != nil {
+		return fmt.Errorf("failed to delete volume group snapshot class: %w", err)
+	}
+
+	return nil
+}