diff --git a/e2e/rbd.go b/e2e/rbd.go index 4c42067cf..c41a6ac64 100644 --- a/e2e/rbd.go +++ b/e2e/rbd.go @@ -6,9 +6,7 @@ import ( "io/ioutil" "os" "strings" - "sync" - "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1" . "github.com/onsi/ginkgo" // nolint v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -498,247 +496,7 @@ var _ = Describe("RBD", func() { By("create a PVC clone and bind it to an app", func() { // snapshot beta is only supported from v1.17+ if k8sVersionGreaterEquals(f.ClientSet, 1, 17) { - var wg sync.WaitGroup - totalCount := 10 - wgErrs := make([]error, totalCount) - chErrs := make([]error, totalCount) - wg.Add(totalCount) - err := createRBDSnapshotClass(f) - if err != nil { - e2elog.Failf("failed to create storageclass with error %v", err) - } - pvc, err := loadPVC(pvcPath) - if err != nil { - e2elog.Failf("failed to load PVC with error %v", err) - } - label := make(map[string]string) - pvc.Namespace = f.UniqueName - err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout) - if err != nil { - e2elog.Failf("failed to create PVC with error %v", err) - } - app, err := loadApp(appPath) - if err != nil { - e2elog.Failf("failed to load app with error %v", err) - } - // write data in PVC - label[appKey] = appLabel - app.Namespace = f.UniqueName - app.Labels = label - opt := metav1.ListOptions{ - LabelSelector: fmt.Sprintf("%s=%s", appKey, label[appKey]), - } - app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvc.Name - checkSum, err := writeDataAndCalChecksum(app, &opt, f) - if err != nil { - e2elog.Failf("failed to calculate checksum with error %v", err) - } - validateRBDImageCount(f, 1) - snap := getSnapshot(snapshotPath) - snap.Namespace = f.UniqueName - snap.Spec.Source.PersistentVolumeClaimName = &pvc.Name - // create snapshot - for i := 0; i < totalCount; i++ { - go func(w *sync.WaitGroup, n int, s v1beta1.VolumeSnapshot) { - s.Name = fmt.Sprintf("%s%d", f.UniqueName, n) - wgErrs[n] = createSnapshot(&s, deployTimeout) - w.Done() - }(&wg, i, snap) - } - wg.Wait() - - failed := 0 - for i, err := range wgErrs { - if err != nil { - // not using Failf() as it aborts the test and does not log other errors - e2elog.Logf("failed to create snapshot (%s%d): %v", f.UniqueName, i, err) - failed++ - } - } - if failed != 0 { - e2elog.Failf("creating snapshots failed, %d errors were logged", failed) - } - - // total images in cluster is 1 parent rbd image+ total snaps - validateRBDImageCount(f, totalCount+1) - pvcClone, err := loadPVC(pvcClonePath) - if err != nil { - e2elog.Failf("failed to load PVC with error %v", err) - } - appClone, err := loadApp(appClonePath) - if err != nil { - e2elog.Failf("failed to load application with error %v", err) - } - pvcClone.Namespace = f.UniqueName - appClone.Namespace = f.UniqueName - pvcClone.Spec.DataSource.Name = fmt.Sprintf("%s%d", f.UniqueName, 0) - - // create multiple PVC from same snapshot - wg.Add(totalCount) - for i := 0; i < totalCount; i++ { - go func(w *sync.WaitGroup, n int, p v1.PersistentVolumeClaim, a v1.Pod) { - name := fmt.Sprintf("%s%d", f.UniqueName, n) - label := make(map[string]string) - label[appKey] = name - a.Labels = label - opt := metav1.ListOptions{ - LabelSelector: fmt.Sprintf("%s=%s", appKey, label[appKey]), - } - wgErrs[n] = createPVCAndApp(name, f, &p, &a, deployTimeout) - if wgErrs[n] == nil { - filePath := a.Spec.Containers[0].VolumeMounts[0].MountPath + "/test" - checkSumClone := "" - e2elog.Logf("calculating checksum clone for filepath %s", filePath) - checkSumClone, chErrs[n] = calculateSHA512sum(f, &a, filePath, &opt) - e2elog.Logf("checksum value for the clone is %s with pod name %s", checkSumClone, name) - if chErrs[n] != nil { - e2elog.Logf("failed to calculte checksum for clone with error %s", chErrs[n]) - } - if checkSumClone != checkSum { - e2elog.Logf("checksum value didn't match. checksum=%s and checksumclone=%s", checkSum, checkSumClone) - } - } - w.Done() - }(&wg, i, *pvcClone, *appClone) - } - wg.Wait() - - for i, err := range wgErrs { - if err != nil { - // not using Failf() as it aborts the test and does not log other errors - e2elog.Logf("failed to create PVC and application (%s%d): %v", f.UniqueName, i, err) - failed++ - } - } - if failed != 0 { - e2elog.Failf("creating PVCs and applications failed, %d errors were logged", failed) - } - - for i, err := range chErrs { - if err != nil { - // not using Failf() as it aborts the test and does not log other errors - e2elog.Logf("failed to calculate checksum (%s%d): %v", f.UniqueName, i, err) - failed++ - } - } - if failed != 0 { - e2elog.Failf("calculating checksum failed, %d errors were logged", failed) - } - // total images in cluster is 1 parent rbd image+ total - // snaps+ total clones - totalCloneCount := totalCount + totalCount + 1 - validateRBDImageCount(f, totalCloneCount) - wg.Add(totalCount) - // delete clone and app - for i := 0; i < totalCount; i++ { - go func(w *sync.WaitGroup, n int, p v1.PersistentVolumeClaim, a v1.Pod) { - name := fmt.Sprintf("%s%d", f.UniqueName, n) - p.Spec.DataSource.Name = name - wgErrs[n] = deletePVCAndApp(name, f, &p, &a) - w.Done() - }(&wg, i, *pvcClone, *appClone) - } - wg.Wait() - - for i, err := range wgErrs { - if err != nil { - // not using Failf() as it aborts the test and does not log other errors - e2elog.Logf("failed to delete PVC and application (%s%d): %v", f.UniqueName, i, err) - failed++ - } - } - if failed != 0 { - e2elog.Failf("deleting PVCs and applications failed, %d errors were logged", failed) - } - - // total images in cluster is 1 parent rbd image+ total - // snaps - validateRBDImageCount(f, totalCount+1) - // create clones from different snapshots and bind it to an - // app - wg.Add(totalCount) - for i := 0; i < totalCount; i++ { - go func(w *sync.WaitGroup, n int, p v1.PersistentVolumeClaim, a v1.Pod) { - name := fmt.Sprintf("%s%d", f.UniqueName, n) - p.Spec.DataSource.Name = name - wgErrs[n] = createPVCAndApp(name, f, &p, &a, deployTimeout) - w.Done() - }(&wg, i, *pvcClone, *appClone) - } - wg.Wait() - - for i, err := range wgErrs { - if err != nil { - // not using Failf() as it aborts the test and does not log other errors - e2elog.Logf("failed to create PVC and application (%s%d): %v", f.UniqueName, i, err) - failed++ - } - } - if failed != 0 { - e2elog.Failf("creating PVCs and applications failed, %d errors were logged", failed) - } - - // total images in cluster is 1 parent rbd image+ total - // snaps+ total clones - totalCloneCount = totalCount + totalCount + 1 - validateRBDImageCount(f, totalCloneCount) - // delete parent pvc - err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout) - if err != nil { - e2elog.Failf("failed to delete PVC with error %v", err) - } - - // total images in cluster is total snaps+ total clones - totalSnapCount := totalCount + totalCount - validateRBDImageCount(f, totalSnapCount) - wg.Add(totalCount) - // delete snapshot - for i := 0; i < totalCount; i++ { - go func(w *sync.WaitGroup, n int, s v1beta1.VolumeSnapshot) { - s.Name = fmt.Sprintf("%s%d", f.UniqueName, n) - wgErrs[n] = deleteSnapshot(&s, deployTimeout) - w.Done() - }(&wg, i, snap) - } - wg.Wait() - - for i, err := range wgErrs { - if err != nil { - // not using Failf() as it aborts the test and does not log other errors - e2elog.Logf("failed to delete snapshot (%s%d): %v", f.UniqueName, i, err) - failed++ - } - } - if failed != 0 { - e2elog.Failf("deleting snapshots failed, %d errors were logged", failed) - } - - validateRBDImageCount(f, totalCount) - wg.Add(totalCount) - // delete clone and app - for i := 0; i < totalCount; i++ { - go func(w *sync.WaitGroup, n int, p v1.PersistentVolumeClaim, a v1.Pod) { - name := fmt.Sprintf("%s%d", f.UniqueName, n) - p.Spec.DataSource.Name = name - wgErrs[n] = deletePVCAndApp(name, f, &p, &a) - w.Done() - }(&wg, i, *pvcClone, *appClone) - } - wg.Wait() - - for i, err := range wgErrs { - if err != nil { - // not using Failf() as it aborts the test and does not log other errors - e2elog.Logf("failed to delete PVC and application (%s%d): %v", f.UniqueName, i, err) - failed++ - } - } - if failed != 0 { - e2elog.Failf("deleting PVCs and applications failed, %d errors were logged", failed) - } - - // validate created backend rbd images - validateRBDImageCount(f, 0) + validatePVCSnapshot(pvcPath, appPath, snapshotPath, pvcClonePath, appClonePath, false, f) } }) @@ -749,6 +507,36 @@ var _ = Describe("RBD", func() { } }) + By("create an encrypted PVC snapshot and restore it for an app", func() { + if !k8sVersionGreaterEquals(f.ClientSet, 1, 16) { + Skip("pvc clone is only supported from v1.16+") + } + + err := deleteResource(rbdExamplePath + "storageclass.yaml") + if err != nil { + e2elog.Failf("failed to delete storageclass with error %v", err) + } + scOpts := map[string]string{ + "encrypted": "true", + "encryptionKMSID": "secrets-metadata-test", + } + err = createRBDStorageClass(f.ClientSet, f, nil, scOpts, deletePolicy) + if err != nil { + e2elog.Failf("failed to create storageclass with error %v", err) + } + + validatePVCSnapshot(pvcPath, appPath, snapshotPath, pvcClonePath, appClonePath, true, f) + + err = deleteResource(rbdExamplePath + "storageclass.yaml") + if err != nil { + e2elog.Failf("failed to delete storageclass with error %v", err) + } + err = createRBDStorageClass(f.ClientSet, f, nil, nil, deletePolicy) + if err != nil { + e2elog.Failf("failed to create storageclass with error %v", err) + } + }) + By("create an encrypted PVC-PVC clone and bind it to an app", func() { if !k8sVersionGreaterEquals(f.ClientSet, 1, 16) { Skip("pvc clone is only supported from v1.16+") @@ -1169,6 +957,17 @@ var _ = Describe("RBD", func() { By("create ROX PVC clone and mount it to multiple pods", func() { // snapshot beta is only supported from v1.17+ if k8sVersionGreaterEquals(f.ClientSet, 1, 17) { + err := createRBDSnapshotClass(f) + if err != nil { + e2elog.Failf("failed to create storageclass with error %v", err) + } + defer func() { + err = deleteRBDSnapshotClass() + if err != nil { + e2elog.Failf("failed to delete VolumeSnapshotClass: %v", err) + } + }() + // create PVC and bind it to an app pvc, err := loadPVC(pvcPath) if err != nil { @@ -1392,6 +1191,17 @@ var _ = Describe("RBD", func() { // Create a PVC clone and bind it to an app within the namespace // snapshot beta is only supported from v1.17+ if k8sVersionGreaterEquals(f.ClientSet, 1, 17) { + err = createRBDSnapshotClass(f) + if err != nil { + e2elog.Failf("failed to create storageclass with error %v", err) + } + defer func() { + err = deleteRBDSnapshotClass() + if err != nil { + e2elog.Failf("failed to delete VolumeSnapshotClass: %v", err) + } + }() + var pvc = &v1.PersistentVolumeClaim{} pvc, err = loadPVC(pvcPath) if err != nil { diff --git a/e2e/snapshot.go b/e2e/snapshot.go index 709b0a800..91e0b7835 100644 --- a/e2e/snapshot.go +++ b/e2e/snapshot.go @@ -136,6 +136,17 @@ func createRBDSnapshotClass(f *framework.Framework) error { return err } +func deleteRBDSnapshotClass() error { + scPath := fmt.Sprintf("%s/%s", rbdExamplePath, "snapshotclass.yaml") + sc := getSnapshotClass(scPath) + + sclient, err := newSnapshotClient() + if err != nil { + return err + } + return sclient.SnapshotV1beta1().VolumeSnapshotClasses().Delete(context.TODO(), sc.Name, metav1.DeleteOptions{}) +} + func createCephFSSnapshotClass(f *framework.Framework) error { scPath := fmt.Sprintf("%s/%s", cephfsExamplePath, "snapshotclass.yaml") sc := getSnapshotClass(scPath) diff --git a/e2e/utils.go b/e2e/utils.go index 8a8f023c3..f1b7bc2ab 100644 --- a/e2e/utils.go +++ b/e2e/utils.go @@ -12,6 +12,7 @@ import ( "sync" "time" + "github.com/kubernetes-csi/external-snapshotter/v2/pkg/apis/volumesnapshot/v1beta1" v1 "k8s.io/api/core/v1" scv1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -651,6 +652,262 @@ func validatePVCClone(sourcePvcPath, sourceAppPath, clonePvcPath, clonePvcAppPat validateRBDImageCount(f, 0) } +// nolint:gocyclo,gocognit // reduce complexity +func validatePVCSnapshot(pvcPath, appPath, snapshotPath, pvcClonePath, appClonePath string, validateEncryption bool, f *framework.Framework) { + var wg sync.WaitGroup + totalCount := 10 + wgErrs := make([]error, totalCount) + chErrs := make([]error, totalCount) + wg.Add(totalCount) + + err := createRBDSnapshotClass(f) + if err != nil { + e2elog.Failf("failed to create storageclass with error %v", err) + } + defer func() { + err = deleteRBDSnapshotClass() + if err != nil { + e2elog.Failf("failed to delete VolumeSnapshotClass: %v", err) + } + }() + + pvc, err := loadPVC(pvcPath) + if err != nil { + e2elog.Failf("failed to load PVC with error %v", err) + } + label := make(map[string]string) + pvc.Namespace = f.UniqueName + err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout) + if err != nil { + e2elog.Failf("failed to create PVC with error %v", err) + } + app, err := loadApp(appPath) + if err != nil { + e2elog.Failf("failed to load app with error %v", err) + } + // write data in PVC + label[appKey] = appLabel + app.Namespace = f.UniqueName + app.Labels = label + opt := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", appKey, label[appKey]), + } + app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvc.Name + checkSum, err := writeDataAndCalChecksum(app, &opt, f) + if err != nil { + e2elog.Failf("failed to calculate checksum with error %v", err) + } + validateRBDImageCount(f, 1) + snap := getSnapshot(snapshotPath) + snap.Namespace = f.UniqueName + snap.Spec.Source.PersistentVolumeClaimName = &pvc.Name + // create snapshot + for i := 0; i < totalCount; i++ { + go func(w *sync.WaitGroup, n int, s v1beta1.VolumeSnapshot) { + s.Name = fmt.Sprintf("%s%d", f.UniqueName, n) + wgErrs[n] = createSnapshot(&s, deployTimeout) + w.Done() + }(&wg, i, snap) + } + wg.Wait() + + failed := 0 + for i, err := range wgErrs { + if err != nil { + // not using Failf() as it aborts the test and does not log other errors + e2elog.Logf("failed to create snapshot (%s%d): %v", f.UniqueName, i, err) + failed++ + } + } + if failed != 0 { + e2elog.Failf("creating snapshots failed, %d errors were logged", failed) + } + + // total images in cluster is 1 parent rbd image+ total snaps + validateRBDImageCount(f, totalCount+1) + pvcClone, err := loadPVC(pvcClonePath) + if err != nil { + e2elog.Failf("failed to load PVC with error %v", err) + } + appClone, err := loadApp(appClonePath) + if err != nil { + e2elog.Failf("failed to load application with error %v", err) + } + pvcClone.Namespace = f.UniqueName + appClone.Namespace = f.UniqueName + pvcClone.Spec.DataSource.Name = fmt.Sprintf("%s%d", f.UniqueName, 0) + + // create multiple PVC from same snapshot + wg.Add(totalCount) + for i := 0; i < totalCount; i++ { + go func(w *sync.WaitGroup, n int, p v1.PersistentVolumeClaim, a v1.Pod) { + name := fmt.Sprintf("%s%d", f.UniqueName, n) + label := make(map[string]string) + label[appKey] = name + a.Labels = label + opt := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", appKey, label[appKey]), + } + wgErrs[n] = createPVCAndApp(name, f, &p, &a, deployTimeout) + if wgErrs[n] == nil { + filePath := a.Spec.Containers[0].VolumeMounts[0].MountPath + "/test" + checkSumClone := "" + e2elog.Logf("calculating checksum clone for filepath %s", filePath) + checkSumClone, chErrs[n] = calculateSHA512sum(f, &a, filePath, &opt) + e2elog.Logf("checksum value for the clone is %s with pod name %s", checkSumClone, name) + if chErrs[n] != nil { + e2elog.Logf("failed to calculte checksum for clone with error %s", chErrs[n]) + } + if checkSumClone != checkSum { + e2elog.Logf("checksum value didn't match. checksum=%s and checksumclone=%s", checkSum, checkSumClone) + } + } + if wgErrs[n] == nil && validateEncryption { + wgErrs[n] = validateEncryptedPVC(f, &p, &a) + } + w.Done() + }(&wg, i, *pvcClone, *appClone) + } + wg.Wait() + + for i, err := range wgErrs { + if err != nil { + // not using Failf() as it aborts the test and does not log other errors + e2elog.Logf("failed to create PVC and application (%s%d): %v", f.UniqueName, i, err) + failed++ + } + } + if failed != 0 { + e2elog.Failf("creating PVCs and applications failed, %d errors were logged", failed) + } + + for i, err := range chErrs { + if err != nil { + // not using Failf() as it aborts the test and does not log other errors + e2elog.Logf("failed to calculate checksum (%s%d): %v", f.UniqueName, i, err) + failed++ + } + } + if failed != 0 { + e2elog.Failf("calculating checksum failed, %d errors were logged", failed) + } + // total images in cluster is 1 parent rbd image+ total + // snaps+ total clones + totalCloneCount := totalCount + totalCount + 1 + validateRBDImageCount(f, totalCloneCount) + wg.Add(totalCount) + // delete clone and app + for i := 0; i < totalCount; i++ { + go func(w *sync.WaitGroup, n int, p v1.PersistentVolumeClaim, a v1.Pod) { + name := fmt.Sprintf("%s%d", f.UniqueName, n) + p.Spec.DataSource.Name = name + wgErrs[n] = deletePVCAndApp(name, f, &p, &a) + w.Done() + }(&wg, i, *pvcClone, *appClone) + } + wg.Wait() + + for i, err := range wgErrs { + if err != nil { + // not using Failf() as it aborts the test and does not log other errors + e2elog.Logf("failed to delete PVC and application (%s%d): %v", f.UniqueName, i, err) + failed++ + } + } + if failed != 0 { + e2elog.Failf("deleting PVCs and applications failed, %d errors were logged", failed) + } + + // total images in cluster is 1 parent rbd image+ total + // snaps + validateRBDImageCount(f, totalCount+1) + // create clones from different snapshots and bind it to an + // app + wg.Add(totalCount) + for i := 0; i < totalCount; i++ { + go func(w *sync.WaitGroup, n int, p v1.PersistentVolumeClaim, a v1.Pod) { + name := fmt.Sprintf("%s%d", f.UniqueName, n) + p.Spec.DataSource.Name = name + wgErrs[n] = createPVCAndApp(name, f, &p, &a, deployTimeout) + w.Done() + }(&wg, i, *pvcClone, *appClone) + } + wg.Wait() + + for i, err := range wgErrs { + if err != nil { + // not using Failf() as it aborts the test and does not log other errors + e2elog.Logf("failed to create PVC and application (%s%d): %v", f.UniqueName, i, err) + failed++ + } + } + if failed != 0 { + e2elog.Failf("creating PVCs and applications failed, %d errors were logged", failed) + } + + // total images in cluster is 1 parent rbd image+ total + // snaps+ total clones + totalCloneCount = totalCount + totalCount + 1 + validateRBDImageCount(f, totalCloneCount) + // delete parent pvc + err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout) + if err != nil { + e2elog.Failf("failed to delete PVC with error %v", err) + } + + // total images in cluster is total snaps+ total clones + totalSnapCount := totalCount + totalCount + validateRBDImageCount(f, totalSnapCount) + wg.Add(totalCount) + // delete snapshot + for i := 0; i < totalCount; i++ { + go func(w *sync.WaitGroup, n int, s v1beta1.VolumeSnapshot) { + s.Name = fmt.Sprintf("%s%d", f.UniqueName, n) + wgErrs[n] = deleteSnapshot(&s, deployTimeout) + w.Done() + }(&wg, i, snap) + } + wg.Wait() + + for i, err := range wgErrs { + if err != nil { + // not using Failf() as it aborts the test and does not log other errors + e2elog.Logf("failed to delete snapshot (%s%d): %v", f.UniqueName, i, err) + failed++ + } + } + if failed != 0 { + e2elog.Failf("deleting snapshots failed, %d errors were logged", failed) + } + + validateRBDImageCount(f, totalCount) + wg.Add(totalCount) + // delete clone and app + for i := 0; i < totalCount; i++ { + go func(w *sync.WaitGroup, n int, p v1.PersistentVolumeClaim, a v1.Pod) { + name := fmt.Sprintf("%s%d", f.UniqueName, n) + p.Spec.DataSource.Name = name + wgErrs[n] = deletePVCAndApp(name, f, &p, &a) + w.Done() + }(&wg, i, *pvcClone, *appClone) + } + wg.Wait() + + for i, err := range wgErrs { + if err != nil { + // not using Failf() as it aborts the test and does not log other errors + e2elog.Logf("failed to delete PVC and application (%s%d): %v", f.UniqueName, i, err) + failed++ + } + } + if failed != 0 { + e2elog.Failf("deleting PVCs and applications failed, %d errors were logged", failed) + } + + // validate created backend rbd images + validateRBDImageCount(f, 0) +} + // validateController simulates the required operations to validate the // controller. // Controller will generates the omap data when the PV is created.