From 34c37663c2380ded00374bc77dc6f90870a55599 Mon Sep 17 00:00:00 2001 From: Yug Gupta Date: Wed, 27 Oct 2021 10:42:09 +0530 Subject: [PATCH 01/16] e2e: Verify if app can mount ec pvc Verify if an application is able to mount a pvc created via an erasure coded pool. Signed-off-by: Yug Gupta --- e2e/rbd.go | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/e2e/rbd.go b/e2e/rbd.go index 27f1931ea..cce6ed5fb 100644 --- a/e2e/rbd.go +++ b/e2e/rbd.go @@ -37,6 +37,7 @@ var ( rbdDeploymentName = "csi-rbdplugin-provisioner" rbdDaemonsetName = "csi-rbdplugin" defaultRBDPool = "replicapool" + erasureCodedPool = "ec-pool" // Topology related variables. nodeRegionLabel = "test.failure-domain/region" regionValue = "testregion" @@ -465,6 +466,48 @@ var _ = Describe("RBD", func() { validateRBDImageCount(f, 0, defaultRBDPool) }) + By("create an erasure coded PVC and bind it to an app", func() { + err := deleteResource(rbdExamplePath + "storageclass.yaml") + if err != nil { + e2elog.Failf("failed to delete storageclass: %v", err) + } + err = createRBDStorageClass( + f.ClientSet, + f, + defaultSCName, + nil, + map[string]string{ + "dataPool": erasureCodedPool, + "pool": defaultRBDPool, + }, + deletePolicy) + if err != nil { + e2elog.Failf("failed to create storageclass: %v", err) + } + pvc, app, err := createPVCAndAppBinding(pvcPath, appPath, f, deployTimeout) + if err != nil { + e2elog.Failf("failed to create pvc and application binding: %v", err) + } + err = checkPVCDataPoolForImageInPool(f, pvc, defaultRBDPool, "ec-pool") + if err != nil { + e2elog.Failf("failed to check data pool for image: %v", err) + } + err = deletePVCAndApp("", f, pvc, app) + if err != nil { + e2elog.Failf("failed to delete pvc and application : %v", err) + } + // validate created backend rbd images + validateRBDImageCount(f, 0, defaultRBDPool) + err = deleteResource(rbdExamplePath + "storageclass.yaml") + if err != nil { + e2elog.Failf("failed to delete storageclass with error %v", err) + } + err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy) + if err != nil { + e2elog.Failf("failed to create storageclass with error %v", err) + } + }) + By("create a PVC and bind it to an app with ext4 as the FS ", func() { err := deleteResource(rbdExamplePath + "storageclass.yaml") if err != nil { From 938051463fc3f22c09020d7fe08aac18e8d958d2 Mon Sep 17 00:00:00 2001 From: Yug Gupta Date: Wed, 27 Oct 2021 13:52:50 +0530 Subject: [PATCH 02/16] e2e: validate snapshot restore in ec pool validate snapshot restore in erasure coded pool. Signed-off-by: Yug Gupta --- e2e/rbd.go | 30 +++++++++++++++++++----------- e2e/utils.go | 9 ++++++--- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/e2e/rbd.go b/e2e/rbd.go index cce6ed5fb..3c703c00e 100644 --- a/e2e/rbd.go +++ b/e2e/rbd.go @@ -38,6 +38,7 @@ var ( rbdDaemonsetName = "csi-rbdplugin" defaultRBDPool = "replicapool" erasureCodedPool = "ec-pool" + noDataPool = "" // Topology related variables. nodeRegionLabel = "test.failure-domain/region" regionValue = "testregion" @@ -498,14 +499,20 @@ var _ = Describe("RBD", func() { } // validate created backend rbd images validateRBDImageCount(f, 0, defaultRBDPool) - err = deleteResource(rbdExamplePath + "storageclass.yaml") - if err != nil { - e2elog.Failf("failed to delete storageclass with error %v", err) - } - err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy) - if err != nil { - e2elog.Failf("failed to create storageclass with error %v", err) - } + }) + + By("create an erasure coded PVC and validate snapshot restore", func() { + validatePVCSnapshot( + defaultCloneCount, + pvcPath, + appPath, + snapshotPath, + pvcClonePath, + appClonePath, + noKMS, noKMS, + defaultSCName, + erasureCodedPool, + f) }) By("create a PVC and bind it to an app with ext4 as the FS ", func() { @@ -1518,6 +1525,7 @@ var _ = Describe("RBD", func() { appClonePath, noKMS, noKMS, defaultSCName, + noDataPool, f) }) @@ -1574,7 +1582,7 @@ var _ = Describe("RBD", func() { validatePVCSnapshot(1, pvcPath, appPath, snapshotPath, pvcClonePath, appClonePath, vaultKMS, vaultKMS, - defaultSCName, + defaultSCName, noDataPool, f) err = deleteResource(rbdExamplePath + "storageclass.yaml") @@ -1620,7 +1628,7 @@ var _ = Describe("RBD", func() { validatePVCSnapshot(1, pvcPath, appPath, snapshotPath, pvcClonePath, appClonePath, vaultKMS, vaultTenantSAKMS, - restoreSCName, f) + restoreSCName, noDataPool, f) err = retryKubectlArgs(cephCSINamespace, kubectlDelete, deployTimeout, "storageclass", restoreSCName) if err != nil { @@ -1679,7 +1687,7 @@ var _ = Describe("RBD", func() { validatePVCSnapshot(1, pvcPath, appPath, snapshotPath, pvcClonePath, appClonePath, vaultKMS, secretsMetadataKMS, - restoreSCName, f) + restoreSCName, noDataPool, f) // delete user secret err = retryKubectlFile(namespace, diff --git a/e2e/utils.go b/e2e/utils.go index 9af1a42a6..ff295f044 100644 --- a/e2e/utils.go +++ b/e2e/utils.go @@ -802,9 +802,8 @@ func validatePVCClone( func validatePVCSnapshot( totalCount int, pvcPath, appPath, snapshotPath, pvcClonePath, appClonePath string, - kms, restoreKMS kmsConfig, - restoreSCName string, - f *framework.Framework) { + kms, restoreKMS kmsConfig, restoreSCName, + dataPool string, f *framework.Framework) { var wg sync.WaitGroup wgErrs := make([]error, totalCount) chErrs := make([]error, totalCount) @@ -1020,6 +1019,10 @@ func validatePVCSnapshot( name := fmt.Sprintf("%s%d", f.UniqueName, n) p.Spec.DataSource.Name = name wgErrs[n] = createPVCAndApp(name, f, &p, &a, deployTimeout) + if wgErrs[n] == nil && dataPool != noDataPool { + wgErrs[n] = checkPVCDataPoolForImageInPool(f, &p, defaultRBDPool, dataPool) + } + wg.Done() }(i, *pvcClone, *appClone) } From 2bf17007d9e3de3b2577f8b8359b47551e52c955 Mon Sep 17 00:00:00 2001 From: Yug Gupta Date: Wed, 27 Oct 2021 13:55:23 +0530 Subject: [PATCH 03/16] e2e: validate pvc-pvc clone in ec pool Validate pvc to pvc clone in erasure coded pools. Signed-off-by: Yug Gupta --- e2e/rbd.go | 53 +++++++++++++++++++++++++++++++++++++++++++++++++--- e2e/utils.go | 6 +++++- 2 files changed, 55 insertions(+), 4 deletions(-) diff --git a/e2e/rbd.go b/e2e/rbd.go index 3c703c00e..31e68f95b 100644 --- a/e2e/rbd.go +++ b/e2e/rbd.go @@ -515,6 +515,27 @@ var _ = Describe("RBD", func() { f) }) + By("create an erasure coded PVC and validate PVC-PVC clone", func() { + validatePVCClone( + defaultCloneCount, + pvcPath, + appPath, + pvcSmartClonePath, + appSmartClonePath, + erasureCodedPool, + noKMS, + noPVCValidation, + f) + err := deleteResource(rbdExamplePath + "storageclass.yaml") + if err != nil { + e2elog.Failf("failed to delete storageclass: %v", err) + } + err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy) + if err != nil { + e2elog.Failf("failed to create storageclass: %v", err) + } + }) + By("create a PVC and bind it to an app with ext4 as the FS ", func() { err := deleteResource(rbdExamplePath + "storageclass.yaml") if err != nil { @@ -1536,6 +1557,7 @@ var _ = Describe("RBD", func() { appPath, pvcSmartClonePath, appSmartClonePath, + noDataPool, noKMS, noPVCValidation, f) @@ -1553,7 +1575,15 @@ var _ = Describe("RBD", func() { e2elog.Failf("failed to create storageclass: %v", err) } - validatePVCClone(1, pvcPath, appPath, pvcSmartClonePath, appSmartClonePath, noKMS, isThickPVC, f) + validatePVCClone(1, + pvcPath, + appPath, + pvcSmartClonePath, + appSmartClonePath, + noDataPool, + noKMS, + isThickPVC, + f) err = deleteResource(rbdExamplePath + "storageclass.yaml") if err != nil { @@ -1732,7 +1762,15 @@ var _ = Describe("RBD", func() { e2elog.Failf("failed to create storageclass: %v", err) } - validatePVCClone(1, pvcPath, appPath, pvcSmartClonePath, appSmartClonePath, secretsMetadataKMS, isEncryptedPVC, f) + validatePVCClone(1, + pvcPath, + appPath, + pvcSmartClonePath, + appSmartClonePath, + noDataPool, + secretsMetadataKMS, + isEncryptedPVC, + f) err = deleteResource(rbdExamplePath + "storageclass.yaml") if err != nil { @@ -1758,7 +1796,15 @@ var _ = Describe("RBD", func() { e2elog.Failf("failed to create storageclass: %v", err) } - validatePVCClone(1, pvcPath, appPath, pvcSmartClonePath, appSmartClonePath, vaultKMS, isEncryptedPVC, f) + validatePVCClone(1, + pvcPath, + appPath, + pvcSmartClonePath, + appSmartClonePath, + noDataPool, + vaultKMS, + isEncryptedPVC, + f) err = deleteResource(rbdExamplePath + "storageclass.yaml") if err != nil { @@ -1787,6 +1833,7 @@ var _ = Describe("RBD", func() { rawAppPath, pvcBlockSmartClonePath, appBlockSmartClonePath, + noDataPool, noKMS, noPVCValidation, f) diff --git a/e2e/utils.go b/e2e/utils.go index ff295f044..2c6b3ece9 100644 --- a/e2e/utils.go +++ b/e2e/utils.go @@ -594,7 +594,8 @@ func writeDataAndCalChecksum(app *v1.Pod, opt *metav1.ListOptions, f *framework. // nolint:gocyclo,gocognit,nestif,cyclop // reduce complexity func validatePVCClone( totalCount int, - sourcePvcPath, sourceAppPath, clonePvcPath, clonePvcAppPath string, + sourcePvcPath, sourceAppPath, clonePvcPath, clonePvcAppPath, + dataPool string, kms kmsConfig, validatePVC validateFunc, f *framework.Framework) { @@ -662,6 +663,9 @@ func validatePVCClone( LabelSelector: fmt.Sprintf("%s=%s", appKey, label[appKey]), } wgErrs[n] = createPVCAndApp(name, f, &p, &a, deployTimeout) + if wgErrs[n] == nil && dataPool != noDataPool { + wgErrs[n] = checkPVCDataPoolForImageInPool(f, &p, defaultRBDPool, dataPool) + } if wgErrs[n] == nil && kms != noKMS { if kms.canGetPassphrase() { imageData, sErr := getImageInfoFromPVC(p.Namespace, name, f) From 4ebfeaaa75d425f465e67978b89ac751b0eee4b4 Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Tue, 7 Dec 2021 13:35:31 +0530 Subject: [PATCH 04/16] e2e: add e2e testcase of new image features adding e2e testcase to validate the workflow of pvc creation and attaching to pod works for new image features like fast-diff,obj-map,exclusive-lock and layering. fixes: #2695 Signed-off-by: Madhu Rajanna --- e2e/rbd.go | 74 +++++++++++++++++++++++++++++++++++++++++++++++ e2e/rbd_helper.go | 12 ++++++++ 2 files changed, 86 insertions(+) diff --git a/e2e/rbd.go b/e2e/rbd.go index 31e68f95b..42fd288f6 100644 --- a/e2e/rbd.go +++ b/e2e/rbd.go @@ -649,6 +649,80 @@ var _ = Describe("RBD", func() { } }) + By("create PVC with layering,fast-diff image-features and bind it to an app", + func() { + if util.CheckKernelSupport(kernelRelease, fastDiffSupport) { + err := deleteResource(rbdExamplePath + "storageclass.yaml") + if err != nil { + e2elog.Failf("failed to delete storageclass: %v", err) + } + err = createRBDStorageClass( + f.ClientSet, + f, + defaultSCName, + nil, + map[string]string{ + "imageFeatures": "layering,exclusive-lock,object-map,fast-diff", + }, + deletePolicy) + if err != nil { + e2elog.Failf("failed to create storageclass: %v", err) + } + err = validatePVCAndAppBinding(pvcPath, appPath, f) + if err != nil { + e2elog.Failf("failed to validate CephFS pvc and application binding: %v", err) + } + // validate created backend rbd images + validateRBDImageCount(f, 0, defaultRBDPool) + err = deleteResource(rbdExamplePath + "storageclass.yaml") + if err != nil { + e2elog.Failf("failed to delete storageclass: %v", err) + } + err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy) + if err != nil { + e2elog.Failf("failed to create storageclass: %v", err) + } + } + }) + + By("create PVC with journaling,fast-diff image-features and bind it to an app using rbd-nbd mounter", + func() { + if util.CheckKernelSupport(kernelRelease, fastDiffSupport) { + err := deleteResource(rbdExamplePath + "storageclass.yaml") + if err != nil { + e2elog.Failf("failed to delete storageclass: %v", err) + } + // Storage class with rbd-nbd mounter + err = createRBDStorageClass( + f.ClientSet, + f, + defaultSCName, + nil, + map[string]string{ + "mounter": "rbd-nbd", + "imageFeatures": "layering,journaling,exclusive-lock,object-map,fast-diff", + }, + deletePolicy) + if err != nil { + e2elog.Failf("failed to create storageclass: %v", err) + } + err = validatePVCAndAppBinding(pvcPath, appPath, f) + if err != nil { + e2elog.Failf("failed to validate CephFS pvc and application binding: %v", err) + } + // validate created backend rbd images + validateRBDImageCount(f, 0, defaultRBDPool) + err = deleteResource(rbdExamplePath + "storageclass.yaml") + if err != nil { + e2elog.Failf("failed to delete storageclass: %v", err) + } + err = createRBDStorageClass(f.ClientSet, f, defaultSCName, nil, nil, deletePolicy) + if err != nil { + e2elog.Failf("failed to create storageclass: %v", err) + } + } + }) + // NOTE: RWX is restricted for FileSystem VolumeMode at ceph-csi, // see pull#261 for more details. By("Create RWX+Block Mode PVC and bind to multiple pods via deployment using rbd-nbd mounter", func() { diff --git a/e2e/rbd_helper.go b/e2e/rbd_helper.go index 6d6950fb4..8ebcfc4b1 100644 --- a/e2e/rbd_helper.go +++ b/e2e/rbd_helper.go @@ -40,6 +40,18 @@ var nbdResizeSupport = []util.KernelVersion{ }, // standard 5.3+ versions } +// nolint:gomnd // numbers specify Kernel versions. +var fastDiffSupport = []util.KernelVersion{ + { + Version: 5, + PatchLevel: 3, + SubLevel: 0, + ExtraVersion: 0, + Distribution: "", + Backport: false, + }, // standard 5.3+ versions +} + // To use `io-timeout=0` we need // www.mail-archive.com/linux-block@vger.kernel.org/msg38060.html // nolint:gomnd // numbers specify Kernel versions. From e76bffe35341f94fc514581110502098dead6e19 Mon Sep 17 00:00:00 2001 From: Niels de Vos Date: Thu, 9 Dec 2021 08:52:39 +0100 Subject: [PATCH 05/16] cleanup: import k8s.io/mount-utils instead of k8s.io/utils/mount k8s.io/utils/mount has moved to k8s.io/mount-utils, and Ceph-CSI uses that already in most locations. Only internal/util/util.go still imports the old path. Signed-off-by: Niels de Vos --- internal/util/util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/util/util.go b/internal/util/util.go index 33dd04872..6bf522713 100644 --- a/internal/util/util.go +++ b/internal/util/util.go @@ -31,7 +31,7 @@ import ( "golang.org/x/sys/unix" "k8s.io/apimachinery/pkg/util/validation" "k8s.io/cloud-provider/volume/helpers" - "k8s.io/utils/mount" + mount "k8s.io/mount-utils" ) // RoundOffVolSize rounds up given quantity up to chunks of MiB/GiB. From 8d09134125180d1fbe26df925913de20e27d93f5 Mon Sep 17 00:00:00 2001 From: Niels de Vos Date: Thu, 9 Dec 2021 09:00:52 +0100 Subject: [PATCH 06/16] rbd: export GenVolFromVolID() for consumption by csi-addons genVolFromVolID() is used by the CSI Controller service to create an rbdVolume object from a CSI volume_id. This function is useful for CSI-Addons Services as well, so rename it to GenVolFromVolID(). Signed-off-by: Niels de Vos --- internal/rbd/controllerserver.go | 12 ++++++------ internal/rbd/rbd_util.go | 6 ++++-- internal/rbd/replicationcontrollerserver.go | 10 +++++----- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/internal/rbd/controllerserver.go b/internal/rbd/controllerserver.go index 50d7c4f73..04c5496ab 100644 --- a/internal/rbd/controllerserver.go +++ b/internal/rbd/controllerserver.go @@ -734,7 +734,7 @@ func checkContentSource( if volID == "" { return nil, nil, status.Errorf(codes.NotFound, "volume ID cannot be empty") } - rbdvol, err := genVolFromVolID(ctx, volID, cr, req.GetSecrets()) + rbdvol, err := GenVolFromVolID(ctx, volID, cr, req.GetSecrets()) if err != nil { log.ErrorLog(ctx, "failed to get backend image for %s: %v", volID, err) if !errors.Is(err, ErrImageNotFound) { @@ -750,9 +750,9 @@ func checkContentSource( return nil, nil, status.Errorf(codes.InvalidArgument, "not a proper volume source") } -// checkErrAndUndoReserve work on error from genVolFromVolID() and undo omap reserve. +// checkErrAndUndoReserve work on error from GenVolFromVolID() and undo omap reserve. // Even-though volumeID is part of rbdVolume struct we take it as an arg here, the main reason -// being, the volume id is getting filled from `genVolFromVolID->generateVolumeFromVolumeID` call path, +// being, the volume id is getting filled from `GenVolFromVolID->generateVolumeFromVolumeID` call path, // and this function is operating on the error case/scenario of above call chain, so we can not rely // on the 'rbdvol->rbdimage->voldID' field. @@ -865,7 +865,7 @@ func (cs *ControllerServer) DeleteVolume( return &csi.DeleteVolumeResponse{}, nil } - rbdVol, err := genVolFromVolID(ctx, volumeID, cr, secrets) + rbdVol, err := GenVolFromVolID(ctx, volumeID, cr, secrets) defer rbdVol.Destroy() if err != nil { return cs.checkErrAndUndoReserve(ctx, err, volumeID, rbdVol, cr) @@ -1016,7 +1016,7 @@ func (cs *ControllerServer) CreateSnapshot( defer cr.DeleteCredentials() // Fetch source volume information - rbdVol, err := genVolFromVolID(ctx, req.GetSourceVolumeId(), cr, req.GetSecrets()) + rbdVol, err := GenVolFromVolID(ctx, req.GetSourceVolumeId(), cr, req.GetSecrets()) defer rbdVol.Destroy() if err != nil { switch { @@ -1462,7 +1462,7 @@ func (cs *ControllerServer) ControllerExpandVolume( } defer cr.DeleteCredentials() - rbdVol, err := genVolFromVolID(ctx, volID, cr, req.GetSecrets()) + rbdVol, err := GenVolFromVolID(ctx, volID, cr, req.GetSecrets()) defer rbdVol.Destroy() if err != nil { switch { diff --git a/internal/rbd/rbd_util.go b/internal/rbd/rbd_util.go index 550ece80c..545f3365d 100644 --- a/internal/rbd/rbd_util.go +++ b/internal/rbd/rbd_util.go @@ -1145,9 +1145,11 @@ func generateVolumeFromVolumeID( return rbdVol, err } -// genVolFromVolID generates a rbdVolume structure from the provided identifier, updating +// GenVolFromVolID generates a rbdVolume structure from the provided identifier, updating // the structure with elements from on-disk image metadata as well. -func genVolFromVolID( +// +// nolint // returns non-exported *rbdVolume, which is fine +func GenVolFromVolID( ctx context.Context, volumeID string, cr *util.Credentials, diff --git a/internal/rbd/replicationcontrollerserver.go b/internal/rbd/replicationcontrollerserver.go index e99db7847..eaf79d7ba 100644 --- a/internal/rbd/replicationcontrollerserver.go +++ b/internal/rbd/replicationcontrollerserver.go @@ -235,7 +235,7 @@ func (rs *ReplicationServer) EnableVolumeReplication(ctx context.Context, } defer rs.VolumeLocks.Release(volumeID) - rbdVol, err := genVolFromVolID(ctx, volumeID, cr, req.GetSecrets()) + rbdVol, err := GenVolFromVolID(ctx, volumeID, cr, req.GetSecrets()) defer rbdVol.Destroy() if err != nil { switch { @@ -410,7 +410,7 @@ func (rs *ReplicationServer) DisableVolumeReplication(ctx context.Context, } defer rs.VolumeLocks.Release(volumeID) - rbdVol, err := genVolFromVolID(ctx, volumeID, cr, req.GetSecrets()) + rbdVol, err := GenVolFromVolID(ctx, volumeID, cr, req.GetSecrets()) defer rbdVol.Destroy() if err != nil { switch { @@ -522,7 +522,7 @@ func (rs *ReplicationServer) PromoteVolume(ctx context.Context, } defer rs.VolumeLocks.Release(volumeID) - rbdVol, err := genVolFromVolID(ctx, volumeID, cr, req.GetSecrets()) + rbdVol, err := GenVolFromVolID(ctx, volumeID, cr, req.GetSecrets()) defer rbdVol.Destroy() if err != nil { switch { @@ -658,7 +658,7 @@ func (rs *ReplicationServer) DemoteVolume(ctx context.Context, } defer rs.VolumeLocks.Release(volumeID) - rbdVol, err := genVolFromVolID(ctx, volumeID, cr, req.GetSecrets()) + rbdVol, err := GenVolFromVolID(ctx, volumeID, cr, req.GetSecrets()) defer rbdVol.Destroy() if err != nil { switch { @@ -745,7 +745,7 @@ func (rs *ReplicationServer) ResyncVolume(ctx context.Context, return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, volumeID) } defer rs.VolumeLocks.Release(volumeID) - rbdVol, err := genVolFromVolID(ctx, volumeID, cr, req.GetSecrets()) + rbdVol, err := GenVolFromVolID(ctx, volumeID, cr, req.GetSecrets()) defer rbdVol.Destroy() if err != nil { switch { From 5baf9811f9ebb7d2d773b8fe0ec38fd24b612bfb Mon Sep 17 00:00:00 2001 From: Niels de Vos Date: Thu, 9 Dec 2021 09:04:48 +0100 Subject: [PATCH 07/16] rbd: export NodeServer.mounter outside of the rbd package NodeServer.mounter is internal to the NodeServer type, but it needs to be initialized by the rbd-driver. The rbd-driver is moved to its own package, so .Mounter needs to be available from there in order to set it. Signed-off-by: Niels de Vos --- internal/rbd/nodeserver.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/internal/rbd/nodeserver.go b/internal/rbd/nodeserver.go index 8242751cd..e310ba6d0 100644 --- a/internal/rbd/nodeserver.go +++ b/internal/rbd/nodeserver.go @@ -42,7 +42,7 @@ import ( // node server spec. type NodeServer struct { *csicommon.DefaultNodeServer - mounter mount.Interface + Mounter mount.Interface // A map storing all volumes with ongoing operations so that additional operations // for that same volume (as defined by VolumeID) return an Aborted error VolumeLocks *util.VolumeLocks @@ -297,7 +297,7 @@ func (ns *NodeServer) NodeStageVolume( if !isHealer { var isNotMnt bool // check if stagingPath is already mounted - isNotMnt, err = isNotMountPoint(ns.mounter, stagingTargetPath) + isNotMnt, err = isNotMountPoint(ns.Mounter, stagingTargetPath) if err != nil { return nil, status.Error(codes.Internal, err.Error()) } else if !isNotMnt { @@ -507,7 +507,7 @@ func (ns *NodeServer) undoStagingTransaction( stagingTargetPath := getStagingTargetPath(req) if transaction.isMounted { - err = ns.mounter.Unmount(stagingTargetPath) + err = ns.Mounter.Unmount(stagingTargetPath) if err != nil { log.ErrorLog(ctx, "failed to unmount stagingtargetPath: %s with error: %v", stagingTargetPath, err) @@ -626,7 +626,7 @@ func (ns *NodeServer) mountVolumeToStagePath( stagingPath, devicePath string) (bool, error) { readOnly := false fsType := req.GetVolumeCapability().GetMount().GetFsType() - diskMounter := &mount.SafeFormatAndMount{Interface: ns.mounter, Exec: utilexec.New()} + diskMounter := &mount.SafeFormatAndMount{Interface: ns.Mounter, Exec: utilexec.New()} // rbd images are thin-provisioned and return zeros for unwritten areas. A freshly created // image will not benefit from discard and we also want to avoid as much unnecessary zeroing // as possible. Open-code mkfs here because FormatAndMount() doesn't accept custom mkfs @@ -730,7 +730,7 @@ func (ns *NodeServer) mountVolume(ctx context.Context, stagingPath string, req * func (ns *NodeServer) createTargetMountPath(ctx context.Context, mountPath string, isBlock bool) (bool, error) { // Check if that mount path exists properly - notMnt, err := mount.IsNotMountPoint(ns.mounter, mountPath) + notMnt, err := mount.IsNotMountPoint(ns.Mounter, mountPath) if err == nil { return notMnt, nil } @@ -773,7 +773,7 @@ func (ns *NodeServer) NodeUnpublishVolume( targetPath := req.GetTargetPath() // considering kubelet make sure node operations like unpublish/unstage...etc can not be called // at same time, an explicit locking at time of nodeunpublish is not required. - notMnt, err := mount.IsNotMountPoint(ns.mounter, targetPath) + notMnt, err := mount.IsNotMountPoint(ns.Mounter, targetPath) if err != nil { if os.IsNotExist(err) { // targetPath has already been deleted @@ -792,7 +792,7 @@ func (ns *NodeServer) NodeUnpublishVolume( return &csi.NodeUnpublishVolumeResponse{}, nil } - if err = ns.mounter.Unmount(targetPath); err != nil { + if err = ns.Mounter.Unmount(targetPath); err != nil { return nil, status.Error(codes.Internal, err.Error()) } @@ -839,7 +839,7 @@ func (ns *NodeServer) NodeUnstageVolume( stagingParentPath := req.GetStagingTargetPath() stagingTargetPath := getStagingTargetPath(req) - notMnt, err := mount.IsNotMountPoint(ns.mounter, stagingTargetPath) + notMnt, err := mount.IsNotMountPoint(ns.Mounter, stagingTargetPath) if err != nil { if !os.IsNotExist(err) { return nil, status.Error(codes.NotFound, err.Error()) @@ -849,7 +849,7 @@ func (ns *NodeServer) NodeUnstageVolume( } if !notMnt { // Unmounting the image - err = ns.mounter.Unmount(stagingTargetPath) + err = ns.Mounter.Unmount(stagingTargetPath) if err != nil { log.ExtendedLog(ctx, "failed to unmount targetPath: %s with error: %v", stagingTargetPath, err) @@ -1061,7 +1061,7 @@ func (ns *NodeServer) processEncryptedDevice( // make sure we continue with the encrypting of the device fallthrough case encrypted == rbdImageEncryptionPrepared: - diskMounter := &mount.SafeFormatAndMount{Interface: ns.mounter, Exec: utilexec.New()} + diskMounter := &mount.SafeFormatAndMount{Interface: ns.Mounter, Exec: utilexec.New()} // TODO: update this when adding support for static (pre-provisioned) PVs var existingFormat string existingFormat, err = diskMounter.GetDiskFormat(devicePath) @@ -1109,7 +1109,7 @@ func (ns *NodeServer) xfsSupportsReflink() bool { // run mkfs.xfs in the same namespace as formatting would be done in // mountVolumeToStagePath() - diskMounter := &mount.SafeFormatAndMount{Interface: ns.mounter, Exec: utilexec.New()} + diskMounter := &mount.SafeFormatAndMount{Interface: ns.Mounter, Exec: utilexec.New()} out, err := diskMounter.Exec.Command("mkfs.xfs").CombinedOutput() if err != nil { // mkfs.xfs should fail with an error message (and help text) From 3eeac3d36c64eeddb298660f2f489996779e17ca Mon Sep 17 00:00:00 2001 From: Niels de Vos Date: Thu, 9 Dec 2021 09:07:29 +0100 Subject: [PATCH 08/16] rbd: export RunVolumeHealer() so that rbd/driver can start it The rbd-driver calls rbd.runVolumeHealer() which is not available outside the rbd package. By moving the rbd-driver into its own package, RunVolumeHealer() needs to be exported. Signed-off-by: Niels de Vos --- internal/rbd/rbd_healer.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/rbd/rbd_healer.go b/internal/rbd/rbd_healer.go index dedd5a678..91a54535b 100644 --- a/internal/rbd/rbd_healer.go +++ b/internal/rbd/rbd_healer.go @@ -127,8 +127,8 @@ func callNodeStageVolume(ns *NodeServer, c *k8s.Clientset, pv *v1.PersistentVolu return nil } -// runVolumeHealer heal the volumes attached on a node. -func runVolumeHealer(ns *NodeServer, conf *util.Config) error { +// RunVolumeHealer heal the volumes attached on a node. +func RunVolumeHealer(ns *NodeServer, conf *util.Config) error { c, err := kubeclient.NewK8sClient() if err != nil { log.ErrorLogMsg("failed to connect to Kubernetes: %v", err) From 8b531f337e85f43c30cf1ea7ca5da1ffe8547d08 Mon Sep 17 00:00:00 2001 From: Niels de Vos Date: Thu, 9 Dec 2021 09:12:37 +0100 Subject: [PATCH 09/16] rbd: add functions for initializing global variables When the rbd-driver starts, it initializes some global (yuck!) variables in the rbd package. Because the rbd-driver is moved out into its own package, these variables can not easily be set anymore. Introcude SetGlobalInt(), SetGlobalBool() and InitJournals() so that the rbd-driver can configure the rbd package. Signed-off-by: Niels de Vos --- internal/rbd/globals.go | 106 +++++++++++++++++++++++++++++++++++++++ internal/rbd/rbd_util.go | 48 ++++++++---------- 2 files changed, 128 insertions(+), 26 deletions(-) create mode 100644 internal/rbd/globals.go diff --git a/internal/rbd/globals.go b/internal/rbd/globals.go new file mode 100644 index 000000000..22e72c9d7 --- /dev/null +++ b/internal/rbd/globals.go @@ -0,0 +1,106 @@ +/* +Copyright 2021 The Ceph-CSI Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rbd + +import ( + "fmt" + + "github.com/ceph/ceph-csi/internal/journal" +) + +const ( + // volIDVersion is the version number of volume ID encoding scheme. + volIDVersion uint16 = 1 +) + +var ( + // CSIInstanceID is the instance ID that is unique to an instance of CSI, used when sharing + // ceph clusters across CSI instances, to differentiate omap names per CSI instance. + CSIInstanceID = "default" + + // volJournal and snapJournal are used to maintain RADOS based journals for CO generated + // VolumeName to backing RBD images. + volJournal *journal.Config + snapJournal *journal.Config + // rbdHardMaxCloneDepth is the hard limit for maximum number of nested volume clones that are taken before flatten + // occurs. + rbdHardMaxCloneDepth uint + + // rbdSoftMaxCloneDepth is the soft limit for maximum number of nested volume clones that are taken before flatten + // occurs. + rbdSoftMaxCloneDepth uint + maxSnapshotsOnImage uint + minSnapshotsOnImageToStartFlatten uint + skipForceFlatten bool + + // krbd features supported by the loaded driver. + krbdFeatures uint +) + +// SetGlobalInt provides a way for the rbd-driver to configure global variables +// in the rbd package. +// +// TODO: these global variables should be set in the ControllerService and +// NodeService where appropriate. Using global variables limits the ability to +// configure these options based on the Ceph cluster or StorageClass. +func SetGlobalInt(name string, value uint) { + switch name { + case "rbdHardMaxCloneDepth": + rbdHardMaxCloneDepth = value + case "rbdSoftMaxCloneDepth": + rbdSoftMaxCloneDepth = value + case "maxSnapshotsOnImage": + maxSnapshotsOnImage = value + case "minSnapshotsOnImageToStartFlatten": + minSnapshotsOnImageToStartFlatten = value + case "krbdFeatures": + krbdFeatures = value + default: + panic(fmt.Sprintf("BUG: can not set unknown variable %q", name)) + } +} + +// SetGlobalBool provides a way for the rbd-driver to configure global +// variables in the rbd package. +// +// TODO: these global variables should be set in the ControllerService and +// NodeService where appropriate. Using global variables limits the ability to +// configure these options based on the Ceph cluster or StorageClass. +func SetGlobalBool(name string, value bool) { + switch name { + case "skipForceFlatten": + skipForceFlatten = value + default: + panic(fmt.Sprintf("BUG: can not set unknown variable %q", name)) + } +} + +// InitJournals initializes the global journals that are used by the rbd +// package. This is called from the rbd-driver on startup. +// +// TODO: these global journals should be set in the ControllerService and +// NodeService where appropriate. Using global journals limits the ability to +// configure these options based on the Ceph cluster or StorageClass. +func InitJournals(instance string) { + // Use passed in instance ID, if provided for omap suffix naming + if instance != "" { + CSIInstanceID = instance + } + + volJournal = journal.NewCSIVolumeJournal(CSIInstanceID) + snapJournal = journal.NewCSISnapshotJournal(CSIInstanceID) +} diff --git a/internal/rbd/rbd_util.go b/internal/rbd/rbd_util.go index 545f3365d..471e1b7d0 100644 --- a/internal/rbd/rbd_util.go +++ b/internal/rbd/rbd_util.go @@ -200,34 +200,30 @@ type migrationVolID struct { clusterID string } -var ( - supportedFeatures = map[string]imageFeature{ - librbd.FeatureNameLayering: { - needRbdNbd: false, - }, - librbd.FeatureNameExclusiveLock: { - needRbdNbd: false, - }, - librbd.FeatureNameObjectMap: { - needRbdNbd: false, - dependsOn: []string{librbd.FeatureNameExclusiveLock}, - }, - librbd.FeatureNameFastDiff: { - needRbdNbd: false, - dependsOn: []string{librbd.FeatureNameObjectMap}, - }, - librbd.FeatureNameJournaling: { - needRbdNbd: true, - dependsOn: []string{librbd.FeatureNameExclusiveLock}, - }, - } +var supportedFeatures = map[string]imageFeature{ + librbd.FeatureNameLayering: { + needRbdNbd: false, + }, + librbd.FeatureNameExclusiveLock: { + needRbdNbd: false, + }, + librbd.FeatureNameObjectMap: { + needRbdNbd: false, + dependsOn: []string{librbd.FeatureNameExclusiveLock}, + }, + librbd.FeatureNameFastDiff: { + needRbdNbd: false, + dependsOn: []string{librbd.FeatureNameObjectMap}, + }, + librbd.FeatureNameJournaling: { + needRbdNbd: true, + dependsOn: []string{librbd.FeatureNameExclusiveLock}, + }, +} - krbdFeatures uint64 // krbd features supported by the loaded driver. -) - -// getKrbdSupportedFeatures load the module if needed and return supported +// GetKrbdSupportedFeatures load the module if needed and return supported // features attribute as a string. -func getKrbdSupportedFeatures() (string, error) { +func GetKrbdSupportedFeatures() (string, error) { // check if the module is loaded or compiled in _, err := os.Stat(krbdSupportedFeaturesFile) if err != nil { From 44d69502bc715b8485f5d1888320e2558ece06d2 Mon Sep 17 00:00:00 2001 From: Niels de Vos Date: Thu, 9 Dec 2021 09:16:32 +0100 Subject: [PATCH 10/16] rbd: export HexStringToInteger() HexStringToInteger() used to return a uint64, but everywhere else uint is used. Having HexStringToInteger() return a uint as well makes it a little easier to use when setting it with SetGlobalInt(). Signed-off-by: Niels de Vos --- internal/rbd/rbd_util.go | 8 ++++---- internal/rbd/rbd_util_test.go | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/rbd/rbd_util.go b/internal/rbd/rbd_util.go index 471e1b7d0..542f84e15 100644 --- a/internal/rbd/rbd_util.go +++ b/internal/rbd/rbd_util.go @@ -250,8 +250,8 @@ func GetKrbdSupportedFeatures() (string, error) { return strings.TrimSuffix(string(val), "\n"), nil } -// hexStringToInteger convert hex value to uint. -func hexStringToInteger(hexString string) (uint64, error) { +// HexStringToInteger convert hex value to uint. +func HexStringToInteger(hexString string) (uint, error) { // trim 0x prefix numberStr := strings.TrimPrefix(strings.ToLower(hexString), "0x") @@ -262,7 +262,7 @@ func hexStringToInteger(hexString string) (uint64, error) { return 0, err } - return output, nil + return uint(output), nil } // isKrbdFeatureSupported checks if a given Image Feature is supported by krbd @@ -274,7 +274,7 @@ func isKrbdFeatureSupported(ctx context.Context, imageFeatures string) bool { supported := true for _, featureName := range imageFeatureSet.Names() { - if (uint64(librbd.FeatureSetFromNames(strings.Split(featureName, " "))) & krbdFeatures) == 0 { + if (uint(librbd.FeatureSetFromNames(strings.Split(featureName, " "))) & krbdFeatures) == 0 { supported = false log.ErrorLog(ctx, "krbd feature %q not supported", featureName) diff --git a/internal/rbd/rbd_util_test.go b/internal/rbd/rbd_util_test.go index 331eb693d..2174ca2ea 100644 --- a/internal/rbd/rbd_util_test.go +++ b/internal/rbd/rbd_util_test.go @@ -334,9 +334,9 @@ func TestIsKrbdFeatureSupported(t *testing.T) { t.Parallel() var err error krbdSupportedFeaturesAttr := "0x1" - krbdFeatures, err = hexStringToInteger(krbdSupportedFeaturesAttr) // initialize krbdFeatures + krbdFeatures, err = HexStringToInteger(krbdSupportedFeaturesAttr) // initialize krbdFeatures if err != nil { - t.Errorf("hexStringToInteger(%s) failed", krbdSupportedFeaturesAttr) + t.Errorf("HexStringToInteger(%s) failed", krbdSupportedFeaturesAttr) } supported := isKrbdFeatureSupported(ctx, tc.featureName) if supported != tc.isSupported { From 203920d8f4d386606b177a7a7fb240365bd3d6bd Mon Sep 17 00:00:00 2001 From: Niels de Vos Date: Thu, 9 Dec 2021 09:18:39 +0100 Subject: [PATCH 11/16] rbd: move driver component into the rbd/driver package The rbd package contains several functions that can be used by CSI-Addons Service implmentations. Unfortunately it is not possible to do this, as the rbd-driver needs to import the csi-addons/rbd package to provide the CSI-Addons server. This causes a circular import when services use the rbd package: - rbd/driver.go import csi-addons/rbd - csi-addons/rbd import rbd (including the driver) By moving rbd/driver.go into its own package, the circular import can be prevented. Signed-off-by: Niels de Vos --- cmd/cephcsi.go | 4 +- internal/rbd/{ => driver}/driver.go | 83 ++++++++---------------- internal/rbd/{ => driver}/driver_test.go | 2 +- 3 files changed, 29 insertions(+), 60 deletions(-) rename internal/rbd/{ => driver}/driver.go (74%) rename internal/rbd/{ => driver}/driver_test.go (98%) diff --git a/cmd/cephcsi.go b/cmd/cephcsi.go index 2d1606c2a..3bbd67770 100644 --- a/cmd/cephcsi.go +++ b/cmd/cephcsi.go @@ -27,7 +27,7 @@ import ( "github.com/ceph/ceph-csi/internal/controller" "github.com/ceph/ceph-csi/internal/controller/persistentvolume" "github.com/ceph/ceph-csi/internal/liveness" - "github.com/ceph/ceph-csi/internal/rbd" + rbddriver "github.com/ceph/ceph-csi/internal/rbd/driver" "github.com/ceph/ceph-csi/internal/util" "github.com/ceph/ceph-csi/internal/util/log" @@ -222,7 +222,7 @@ func main() { case rbdType: validateCloneDepthFlag(&conf) validateMaxSnaphostFlag(&conf) - driver := rbd.NewDriver() + driver := rbddriver.NewDriver() driver.Run(&conf) case cephFSType: diff --git a/internal/rbd/driver.go b/internal/rbd/driver/driver.go similarity index 74% rename from internal/rbd/driver.go rename to internal/rbd/driver/driver.go index 1dae0a58c..b4840da7a 100644 --- a/internal/rbd/driver.go +++ b/internal/rbd/driver/driver.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package rbd +package rbddriver import ( "fmt" @@ -22,7 +22,7 @@ import ( casrbd "github.com/ceph/ceph-csi/internal/csi-addons/rbd" csiaddons "github.com/ceph/ceph-csi/internal/csi-addons/server" csicommon "github.com/ceph/ceph-csi/internal/csi-common" - "github.com/ceph/ceph-csi/internal/journal" + "github.com/ceph/ceph-csi/internal/rbd" "github.com/ceph/ceph-csi/internal/util" "github.com/ceph/ceph-csi/internal/util/log" @@ -30,61 +30,34 @@ import ( mount "k8s.io/mount-utils" ) -const ( - // volIDVersion is the version number of volume ID encoding scheme. - volIDVersion uint16 = 1 -) - // Driver contains the default identity,node and controller struct. type Driver struct { cd *csicommon.CSIDriver - ids *IdentityServer - ns *NodeServer - cs *ControllerServer - rs *ReplicationServer + ids *rbd.IdentityServer + ns *rbd.NodeServer + cs *rbd.ControllerServer + rs *rbd.ReplicationServer // cas is the CSIAddonsServer where CSI-Addons services are handled cas *csiaddons.CSIAddonsServer } -var ( - - // CSIInstanceID is the instance ID that is unique to an instance of CSI, used when sharing - // ceph clusters across CSI instances, to differentiate omap names per CSI instance. - CSIInstanceID = "default" - - // volJournal and snapJournal are used to maintain RADOS based journals for CO generated - // VolumeName to backing RBD images. - volJournal *journal.Config - snapJournal *journal.Config - // rbdHardMaxCloneDepth is the hard limit for maximum number of nested volume clones that are taken before flatten - // occurs. - rbdHardMaxCloneDepth uint - - // rbdSoftMaxCloneDepth is the soft limit for maximum number of nested volume clones that are taken before flatten - // occurs. - rbdSoftMaxCloneDepth uint - maxSnapshotsOnImage uint - minSnapshotsOnImageToStartFlatten uint - skipForceFlatten bool -) - // NewDriver returns new rbd driver. func NewDriver() *Driver { return &Driver{} } // NewIdentityServer initialize a identity server for rbd CSI driver. -func NewIdentityServer(d *csicommon.CSIDriver) *IdentityServer { - return &IdentityServer{ +func NewIdentityServer(d *csicommon.CSIDriver) *rbd.IdentityServer { + return &rbd.IdentityServer{ DefaultIdentityServer: csicommon.NewDefaultIdentityServer(d), } } // NewControllerServer initialize a controller server for rbd CSI driver. -func NewControllerServer(d *csicommon.CSIDriver) *ControllerServer { - return &ControllerServer{ +func NewControllerServer(d *csicommon.CSIDriver) *rbd.ControllerServer { + return &rbd.ControllerServer{ DefaultControllerServer: csicommon.NewDefaultControllerServer(d), VolumeLocks: util.NewVolumeLocks(), SnapshotLocks: util.NewVolumeLocks(), @@ -92,17 +65,17 @@ func NewControllerServer(d *csicommon.CSIDriver) *ControllerServer { } } -func NewReplicationServer(c *ControllerServer) *ReplicationServer { - return &ReplicationServer{ControllerServer: c} +func NewReplicationServer(c *rbd.ControllerServer) *rbd.ReplicationServer { + return &rbd.ReplicationServer{ControllerServer: c} } // NewNodeServer initialize a node server for rbd CSI driver. -func NewNodeServer(d *csicommon.CSIDriver, t string, topology map[string]string) (*NodeServer, error) { +func NewNodeServer(d *csicommon.CSIDriver, t string, topology map[string]string) (*rbd.NodeServer, error) { mounter := mount.New("") - return &NodeServer{ + return &rbd.NodeServer{ DefaultNodeServer: csicommon.NewDefaultNodeServer(d, t, topology), - mounter: mounter, + Mounter: mounter, VolumeLocks: util.NewVolumeLocks(), }, nil } @@ -116,20 +89,14 @@ func (r *Driver) Run(conf *util.Config) { var err error var topology map[string]string - // Use passed in instance ID, if provided for omap suffix naming - if conf.InstanceID != "" { - CSIInstanceID = conf.InstanceID - } - // update clone soft and hard limit - rbdHardMaxCloneDepth = conf.RbdHardMaxCloneDepth - rbdSoftMaxCloneDepth = conf.RbdSoftMaxCloneDepth - skipForceFlatten = conf.SkipForceFlatten - maxSnapshotsOnImage = conf.MaxSnapshotsOnImage - minSnapshotsOnImageToStartFlatten = conf.MinSnapshotsOnImage + rbd.SetGlobalInt("rbdHardMaxCloneDepth", conf.RbdHardMaxCloneDepth) + rbd.SetGlobalInt("rbdSoftMaxCloneDepth", conf.RbdSoftMaxCloneDepth) + rbd.SetGlobalBool("skipForceFlatten", conf.SkipForceFlatten) + rbd.SetGlobalInt("maxSnapshotsOnImage", conf.MaxSnapshotsOnImage) + rbd.SetGlobalInt("minSnapshotsOnImageToStartFlatten", conf.MinSnapshotsOnImage) // Create instances of the volume and snapshot journal - volJournal = journal.NewCSIVolumeJournal(CSIInstanceID) - snapJournal = journal.NewCSISnapshotJournal(CSIInstanceID) + rbd.InitJournals(conf.InstanceID) // configre CSI-Addons server and components err = r.setupCSIAddonsServer(conf) @@ -174,14 +141,16 @@ func (r *Driver) Run(conf *util.Config) { log.FatalLogMsg("failed to start node server, err %v\n", err) } var attr string - attr, err = getKrbdSupportedFeatures() + attr, err = rbd.GetKrbdSupportedFeatures() if err != nil { log.FatalLogMsg(err.Error()) } - krbdFeatures, err = hexStringToInteger(attr) + var krbdFeatures uint + krbdFeatures, err = rbd.HexStringToInteger(attr) if err != nil { log.FatalLogMsg(err.Error()) } + rbd.SetGlobalInt("krbdFeatures", krbdFeatures) } if conf.IsControllerServer { @@ -220,7 +189,7 @@ func (r *Driver) Run(conf *util.Config) { if conf.IsNodeServer { go func() { // TODO: move the healer to csi-addons - err := runVolumeHealer(r.ns, conf) + err := rbd.RunVolumeHealer(r.ns, conf) if err != nil { log.ErrorLogMsg("healer had failures, err %v\n", err) } diff --git a/internal/rbd/driver_test.go b/internal/rbd/driver/driver_test.go similarity index 98% rename from internal/rbd/driver_test.go rename to internal/rbd/driver/driver_test.go index baabb67ae..b1c5d884f 100644 --- a/internal/rbd/driver_test.go +++ b/internal/rbd/driver/driver_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package rbd +package rbddriver import ( "os" From 0bcaffc4332571396fda65cf677f9918964c5024 Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Fri, 10 Dec 2021 11:03:40 +0530 Subject: [PATCH 12/16] ci: remove release-v3.3 rules as we are no longer supporting the release 3.3.x removing the mergify rules for the same. Signed-off-by: Madhu Rajanna --- .mergify.yml | 38 -------------------------------------- 1 file changed, 38 deletions(-) diff --git a/.mergify.yml b/.mergify.yml index 5ce8e8fca..bd45eb08b 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -122,44 +122,6 @@ pull_request_rules: merge: {} dismiss_reviews: {} delete_head_branch: {} - - name: backport patches to release-v3.3 branch - conditions: - - base=devel - - label=backport-to-release-v3.3 - actions: - backport: - branches: - - release-v3.3 - # automerge backports if CI successfully ran - - name: automerge backport release-v3.3 - conditions: - - author=mergify[bot] - - base=release-v3.3 - - label!=DNM - - "#approved-reviews-by>=2" - - "approved-reviews-by=@ceph/ceph-csi-contributors" - - "approved-reviews-by=@ceph/ceph-csi-maintainers" - - "status-success=codespell" - - "status-success=multi-arch-build" - - "status-success=go-test" - - "status-success=commitlint" - - "status-success=golangci-lint" - - "status-success=mod-check" - - "status-success=lint-extras" - - "#changes-requested-reviews-by=0" - - "status-success=ci/centos/mini-e2e-helm/k8s-1.20" - - "status-success=ci/centos/mini-e2e-helm/k8s-1.21" - - "status-success=ci/centos/mini-e2e-helm/k8s-1.22" - - "status-success=ci/centos/mini-e2e/k8s-1.20" - - "status-success=ci/centos/mini-e2e/k8s-1.21" - - "status-success=ci/centos/mini-e2e/k8s-1.22" - - "status-success=ci/centos/upgrade-tests-cephfs" - - "status-success=ci/centos/upgrade-tests-rbd" - - "status-success=DCO" - actions: - merge: {} - dismiss_reviews: {} - delete_head_branch: {} - name: backport patches to release-v3.4 branch conditions: - base=devel From 8b48ded21f08c7d4ad4af5377d3dafa9121df352 Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Fri, 10 Dec 2021 11:06:39 +0530 Subject: [PATCH 13/16] ci: add mergify condition for kubernetes 1.23 as we are running tests on kubernetes 1.23 by default, adding mergify condition to check test passed on kubernetes 1.23 also. Signed-off-by: Madhu Rajanna --- .mergify.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.mergify.yml b/.mergify.yml index bd45eb08b..88ea1281d 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -50,12 +50,15 @@ pull_request_rules: - "status-success=lint-extras" - "status-success=ci/centos/k8s-e2e-external-storage/1.21" - "status-success=ci/centos/k8s-e2e-external-storage/1.22" + - "status-success=ci/centos/k8s-e2e-external-storage/1.23" - "status-success=ci/centos/mini-e2e-helm/k8s-1.20" - "status-success=ci/centos/mini-e2e-helm/k8s-1.21" - "status-success=ci/centos/mini-e2e-helm/k8s-1.22" + - "status-success=ci/centos/mini-e2e-helm/k8s-1.23" - "status-success=ci/centos/mini-e2e/k8s-1.20" - "status-success=ci/centos/mini-e2e/k8s-1.21" - "status-success=ci/centos/mini-e2e/k8s-1.22" + - "status-success=ci/centos/mini-e2e/k8s-1.23" - "status-success=ci/centos/upgrade-tests-cephfs" - "status-success=ci/centos/upgrade-tests-rbd" - "status-success=DCO" @@ -80,12 +83,15 @@ pull_request_rules: - "status-success=lint-extras" - "status-success=ci/centos/k8s-e2e-external-storage/1.21" - "status-success=ci/centos/k8s-e2e-external-storage/1.22" + - "status-success=ci/centos/k8s-e2e-external-storage/1.23" - "status-success=ci/centos/mini-e2e-helm/k8s-1.20" - "status-success=ci/centos/mini-e2e-helm/k8s-1.21" - "status-success=ci/centos/mini-e2e-helm/k8s-1.22" + - "status-success=ci/centos/mini-e2e-helm/k8s-1.23" - "status-success=ci/centos/mini-e2e/k8s-1.20" - "status-success=ci/centos/mini-e2e/k8s-1.21" - "status-success=ci/centos/mini-e2e/k8s-1.22" + - "status-success=ci/centos/mini-e2e/k8s-1.23" - "status-success=ci/centos/upgrade-tests-cephfs" - "status-success=ci/centos/upgrade-tests-rbd" - "status-success=DCO" @@ -109,12 +115,15 @@ pull_request_rules: - "#changes-requested-reviews-by=0" - "status-success=ci/centos/k8s-e2e-external-storage/1.21" - "status-success=ci/centos/k8s-e2e-external-storage/1.22" + - "status-success=ci/centos/k8s-e2e-external-storage/1.23" - "status-success=ci/centos/mini-e2e-helm/k8s-1.20" - "status-success=ci/centos/mini-e2e-helm/k8s-1.21" - "status-success=ci/centos/mini-e2e-helm/k8s-1.22" + - "status-success=ci/centos/mini-e2e-helm/k8s-1.23" - "status-success=ci/centos/mini-e2e/k8s-1.20" - "status-success=ci/centos/mini-e2e/k8s-1.21" - "status-success=ci/centos/mini-e2e/k8s-1.22" + - "status-success=ci/centos/mini-e2e/k8s-1.23" - "status-success=ci/centos/upgrade-tests-cephfs" - "status-success=ci/centos/upgrade-tests-rbd" - "status-success=DCO" @@ -150,9 +159,11 @@ pull_request_rules: - "status-success=ci/centos/mini-e2e-helm/k8s-1.20" - "status-success=ci/centos/mini-e2e-helm/k8s-1.21" - "status-success=ci/centos/mini-e2e-helm/k8s-1.22" + - "status-success=ci/centos/mini-e2e-helm/k8s-1.23" - "status-success=ci/centos/mini-e2e/k8s-1.20" - "status-success=ci/centos/mini-e2e/k8s-1.21" - "status-success=ci/centos/mini-e2e/k8s-1.22" + - "status-success=ci/centos/mini-e2e/k8s-1.23" - "status-success=ci/centos/upgrade-tests-cephfs" - "status-success=ci/centos/upgrade-tests-rbd" - "status-success=DCO" From 99b814fe94288b4f3f8f30df77a037f3ac6fb111 Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Fri, 10 Dec 2021 11:08:10 +0530 Subject: [PATCH 14/16] ci: remove kubernetes 1.20 from condition as we added kubernetes 1.23 for mergify, removing older version i.e 1.20 as we will run tests only on latest 3 kubernetes releases. Signed-off-by: Madhu Rajanna --- .mergify.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.mergify.yml b/.mergify.yml index 88ea1281d..b8e56d7b3 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -51,11 +51,9 @@ pull_request_rules: - "status-success=ci/centos/k8s-e2e-external-storage/1.21" - "status-success=ci/centos/k8s-e2e-external-storage/1.22" - "status-success=ci/centos/k8s-e2e-external-storage/1.23" - - "status-success=ci/centos/mini-e2e-helm/k8s-1.20" - "status-success=ci/centos/mini-e2e-helm/k8s-1.21" - "status-success=ci/centos/mini-e2e-helm/k8s-1.22" - "status-success=ci/centos/mini-e2e-helm/k8s-1.23" - - "status-success=ci/centos/mini-e2e/k8s-1.20" - "status-success=ci/centos/mini-e2e/k8s-1.21" - "status-success=ci/centos/mini-e2e/k8s-1.22" - "status-success=ci/centos/mini-e2e/k8s-1.23" @@ -84,11 +82,9 @@ pull_request_rules: - "status-success=ci/centos/k8s-e2e-external-storage/1.21" - "status-success=ci/centos/k8s-e2e-external-storage/1.22" - "status-success=ci/centos/k8s-e2e-external-storage/1.23" - - "status-success=ci/centos/mini-e2e-helm/k8s-1.20" - "status-success=ci/centos/mini-e2e-helm/k8s-1.21" - "status-success=ci/centos/mini-e2e-helm/k8s-1.22" - "status-success=ci/centos/mini-e2e-helm/k8s-1.23" - - "status-success=ci/centos/mini-e2e/k8s-1.20" - "status-success=ci/centos/mini-e2e/k8s-1.21" - "status-success=ci/centos/mini-e2e/k8s-1.22" - "status-success=ci/centos/mini-e2e/k8s-1.23" @@ -116,11 +112,9 @@ pull_request_rules: - "status-success=ci/centos/k8s-e2e-external-storage/1.21" - "status-success=ci/centos/k8s-e2e-external-storage/1.22" - "status-success=ci/centos/k8s-e2e-external-storage/1.23" - - "status-success=ci/centos/mini-e2e-helm/k8s-1.20" - "status-success=ci/centos/mini-e2e-helm/k8s-1.21" - "status-success=ci/centos/mini-e2e-helm/k8s-1.22" - "status-success=ci/centos/mini-e2e-helm/k8s-1.23" - - "status-success=ci/centos/mini-e2e/k8s-1.20" - "status-success=ci/centos/mini-e2e/k8s-1.21" - "status-success=ci/centos/mini-e2e/k8s-1.22" - "status-success=ci/centos/mini-e2e/k8s-1.23" @@ -156,11 +150,9 @@ pull_request_rules: - "status-success=mod-check" - "status-success=lint-extras" - "#changes-requested-reviews-by=0" - - "status-success=ci/centos/mini-e2e-helm/k8s-1.20" - "status-success=ci/centos/mini-e2e-helm/k8s-1.21" - "status-success=ci/centos/mini-e2e-helm/k8s-1.22" - "status-success=ci/centos/mini-e2e-helm/k8s-1.23" - - "status-success=ci/centos/mini-e2e/k8s-1.20" - "status-success=ci/centos/mini-e2e/k8s-1.21" - "status-success=ci/centos/mini-e2e/k8s-1.22" - "status-success=ci/centos/mini-e2e/k8s-1.23" From b8b30eb6686b7d7b0c4a7a02b84618abcb2b6980 Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Fri, 10 Dec 2021 10:36:19 +0530 Subject: [PATCH 15/16] rebase: update golang version to 1.17.5 This commit updates the golang to the latest available release i.e 1.17.5 Signed-off-by: Madhu Rajanna --- build.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.env b/build.env index d2af354e8..441140ed8 100644 --- a/build.env +++ b/build.env @@ -16,7 +16,7 @@ BASE_IMAGE=docker.io/ceph/ceph:v16 CEPH_VERSION=octopus # standard Golang options -GOLANG_VERSION=1.16.4 +GOLANG_VERSION=1.17.5 GO111MODULE=on # commitlint version From 42403e2ba73debf28b4c504988f0739a11a37a1a Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Fri, 10 Dec 2021 10:41:18 +0530 Subject: [PATCH 16/16] rebase: add protobuf to go.mod adding protobuf to go.mod to fix `from implicitly required module to add missing requirement` error Signed-off-by: Madhu Rajanna --- go.mod | 1 + vendor/modules.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/go.mod b/go.mod index 628c4ce53..5a8a0fe4c 100644 --- a/go.mod +++ b/go.mod @@ -25,6 +25,7 @@ require ( golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 golang.org/x/sys v0.0.0-20210817190340-bfb29a6856f2 google.golang.org/grpc v1.42.0 + google.golang.org/protobuf v1.26.0 k8s.io/api v0.22.4 k8s.io/apimachinery v0.22.4 k8s.io/client-go v12.0.0+incompatible diff --git a/vendor/modules.txt b/vendor/modules.txt index 76034bb28..807f81637 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -543,6 +543,7 @@ google.golang.org/grpc/stats google.golang.org/grpc/status google.golang.org/grpc/tap # google.golang.org/protobuf v1.26.0 +## explicit google.golang.org/protobuf/encoding/protojson google.golang.org/protobuf/encoding/prototext google.golang.org/protobuf/encoding/protowire