From d1acae7209c78e5cf6ade35b88e8681a49ac22f6 Mon Sep 17 00:00:00 2001 From: Humble Chirammal Date: Tue, 27 Sep 2022 19:10:26 +0530 Subject: [PATCH 1/3] deploy: update sidecars to latest versions available Below sidecars are updated with this commit. csi-provisioner: v3.3.0 csi-snapshotter: v6.1.0 This commit change the sidecar versions in build.env setup. Signed-off-by: Humble Chirammal --- build.env | 6 +++--- charts/ceph-csi-cephfs/README.md | 4 ++-- charts/ceph-csi-cephfs/values.yaml | 4 ++-- charts/ceph-csi-rbd/README.md | 4 ++-- charts/ceph-csi-rbd/values.yaml | 4 ++-- deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml | 4 ++-- deploy/nfs/kubernetes/csi-nfsplugin-provisioner.yaml | 4 ++-- deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml | 4 ++-- 8 files changed, 17 insertions(+), 17 deletions(-) diff --git a/build.env b/build.env index 1b3d53d9f..c694e325b 100644 --- a/build.env +++ b/build.env @@ -27,7 +27,7 @@ GOLANGCI_VERSION=v1.47.3 # external snapshotter version # Refer: https://github.com/kubernetes-csi/external-snapshotter/releases -SNAPSHOT_VERSION=v6.0.1 +SNAPSHOT_VERSION=v6.1.0 # "go test" configuration # set to stdout or html to enable coverage reporting, disabled by default @@ -49,9 +49,9 @@ ROOK_CEPH_CLUSTER_IMAGE=quay.io/ceph/ceph:v17 # CSI sidecar version CSI_ATTACHER_VERSION=v4.0.0 -CSI_SNAPSHOTTER_VERSION=v6.0.1 +CSI_SNAPSHOTTER_VERSION=v6.1.0 CSI_RESIZER_VERSION=v1.6.0 -CSI_PROVISIONER_VERSION=v3.2.1 +CSI_PROVISIONER_VERSION=v3.3.0 CSI_NODE_DRIVER_REGISTRAR_VERSION=v2.5.1 # e2e settings diff --git a/charts/ceph-csi-cephfs/README.md b/charts/ceph-csi-cephfs/README.md index a3cb3379e..c1da58988 100644 --- a/charts/ceph-csi-cephfs/README.md +++ b/charts/ceph-csi-cephfs/README.md @@ -112,7 +112,7 @@ charts and their default values. | `provisioner.enableHostNetwork` | Specifies whether hostNetwork is enabled for provisioner pod. | `false` | | `provisioner.profiling.enabled` | Specifies whether profiling should be enabled | `false` | | `provisioner.provisioner.image.repository` | Specifies the csi-provisioner image repository URL | `registry.k8s.io/sig-storage/csi-provisioner` | -| `provisioner.provisioner.image.tag` | Specifies image tag | `v3.2.1` | +| `provisioner.provisioner.image.tag` | Specifies image tag | `v3.3.0` | | `provisioner.provisioner.image.pullPolicy` | Specifies pull policy | `IfNotPresent` | | `provisioner.resizer.image.repository` | Specifies the csi-resizer image repository URL | `registry.k8s.io/sig-storage/csi-resizer` | | `provisioner.resizer.image.tag` | Specifies image tag | `v1.6.0` | @@ -120,7 +120,7 @@ charts and their default values. | `provisioner.resizer.name` | Specifies the name of csi-resizer sidecar | `resizer` | | `provisioner.resizer.enabled` | Specifies whether resizer sidecar is enabled | `true` | | `provisioner.snapshotter.image.repository` | Specifies the csi-snapshotter image repository URL | `registry.k8s.io/sig-storage/csi-snapshotter` | -| `provisioner.snapshotter.image.tag` | Specifies image tag | `v6.0.1` | +| `provisioner.snapshotter.image.tag` | Specifies image tag | `v6.1.0` | | `provisioner.snapshotter.image.pullPolicy` | Specifies pull policy | `IfNotPresent` | | `provisioner.nodeSelector` | Specifies the node selector for provisioner deployment | `{}` | | `provisioner.tolerations` | Specifies the tolerations for provisioner deployment | `{}` | diff --git a/charts/ceph-csi-cephfs/values.yaml b/charts/ceph-csi-cephfs/values.yaml index a37a2505c..f06ccbc5d 100644 --- a/charts/ceph-csi-cephfs/values.yaml +++ b/charts/ceph-csi-cephfs/values.yaml @@ -171,7 +171,7 @@ provisioner: provisioner: image: repository: registry.k8s.io/sig-storage/csi-provisioner - tag: v3.2.1 + tag: v3.3.0 pullPolicy: IfNotPresent resources: {} @@ -190,7 +190,7 @@ provisioner: snapshotter: image: repository: registry.k8s.io/sig-storage/csi-snapshotter - tag: v6.0.1 + tag: v6.1.0 pullPolicy: IfNotPresent resources: {} diff --git a/charts/ceph-csi-rbd/README.md b/charts/ceph-csi-rbd/README.md index 35b4c3291..0f3975671 100644 --- a/charts/ceph-csi-rbd/README.md +++ b/charts/ceph-csi-rbd/README.md @@ -118,7 +118,7 @@ charts and their default values. | `provisioner.enableHostNetwork` | Specifies whether hostNetwork is enabled for provisioner pod. | `false` | | `provisioner.profiling.enabled` | Specifies whether profiling should be enabled | `false` | | `provisioner.provisioner.image.repository` | Specifies the csi-provisioner image repository URL | `registry.k8s.io/sig-storage/csi-provisioner` | -| `provisioner.provisioner.image.tag` | Specifies image tag | `v3.2.1` | +| `provisioner.provisioner.image.tag` | Specifies image tag | `v3.3.0` | | `provisioner.provisioner.image.pullPolicy` | Specifies pull policy | `IfNotPresent` | | `provisioner.attacher.image.repository` | Specifies the csi-attacher image repository URL | `registry.k8s.io/sig-storage/csi-attacher` | | `provisioner.attacher.image.tag` | Specifies image tag | `v4.0.0` | @@ -131,7 +131,7 @@ charts and their default values. | `provisioner.resizer.name` | Specifies the name of csi-resizer sidecar | `resizer` | | `provisioner.resizer.enabled` | Specifies whether resizer sidecar is enabled | `true` | | `provisioner.snapshotter.image.repository` | Specifies the csi-snapshotter image repository URL | `registry.k8s.io/sig-storage/csi-snapshotter` | -| `provisioner.snapshotter.image.tag` | Specifies image tag | `v6.0.1` | +| `provisioner.snapshotter.image.tag` | Specifies image tag | `v6.1.0` | | `provisioner.snapshotter.image.pullPolicy` | Specifies pull policy | `IfNotPresent` | | `provisioner.nodeSelector` | Specifies the node selector for provisioner deployment | `{}` | | `provisioner.tolerations` | Specifies the tolerations for provisioner deployment | `{}` | diff --git a/charts/ceph-csi-rbd/values.yaml b/charts/ceph-csi-rbd/values.yaml index 04d5c628a..65745773b 100644 --- a/charts/ceph-csi-rbd/values.yaml +++ b/charts/ceph-csi-rbd/values.yaml @@ -204,7 +204,7 @@ provisioner: provisioner: image: repository: gcr.io/k8s-staging-sig-storage/csi-provisioner - tag: v3.2.1 + tag: v3.3.0 pullPolicy: IfNotPresent resources: {} @@ -232,7 +232,7 @@ provisioner: snapshotter: image: repository: registry.k8s.io/sig-storage/csi-snapshotter - tag: v6.0.1 + tag: v6.1.0 pullPolicy: IfNotPresent resources: {} diff --git a/deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml b/deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml index 2633e36e7..71bc300ec 100644 --- a/deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml +++ b/deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml @@ -43,7 +43,7 @@ spec: priorityClassName: system-cluster-critical containers: - name: csi-provisioner - image: registry.k8s.io/sig-storage/csi-provisioner:v3.2.1 + image: registry.k8s.io/sig-storage/csi-provisioner:v3.3.0 args: - "--csi-address=$(ADDRESS)" - "--v=1" @@ -79,7 +79,7 @@ spec: - name: socket-dir mountPath: /csi - name: csi-snapshotter - image: registry.k8s.io/sig-storage/csi-snapshotter:v6.0.1 + image: registry.k8s.io/sig-storage/csi-snapshotter:v6.1.0 args: - "--csi-address=$(ADDRESS)" - "--v=1" diff --git a/deploy/nfs/kubernetes/csi-nfsplugin-provisioner.yaml b/deploy/nfs/kubernetes/csi-nfsplugin-provisioner.yaml index 84fb70ae1..600b80050 100644 --- a/deploy/nfs/kubernetes/csi-nfsplugin-provisioner.yaml +++ b/deploy/nfs/kubernetes/csi-nfsplugin-provisioner.yaml @@ -40,7 +40,7 @@ spec: topologyKey: "kubernetes.io/hostname" containers: - name: csi-provisioner - image: registry.k8s.io/sig-storage/csi-provisioner:v3.2.1 + image: registry.k8s.io/sig-storage/csi-provisioner:v3.3.0 args: - "--csi-address=$(ADDRESS)" - "--v=1" @@ -73,7 +73,7 @@ spec: - name: socket-dir mountPath: /csi - name: csi-snapshotter - image: registry.k8s.io/sig-storage/csi-snapshotter:v6.0.1 + image: registry.k8s.io/sig-storage/csi-snapshotter:v6.1.0 args: - "--csi-address=$(ADDRESS)" - "--v=1" diff --git a/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml b/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml index 9e88a67a0..7d60095aa 100644 --- a/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml +++ b/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml @@ -47,7 +47,7 @@ spec: priorityClassName: system-cluster-critical containers: - name: csi-provisioner - image: k8s.gcr.io/sig-storage/csi-provisioner:v3.2.1 + image: registry.k8s.io/sig-storage/csi-provisioner:v3.3.0 args: - "--csi-address=$(ADDRESS)" - "--v=1" @@ -69,7 +69,7 @@ spec: - name: socket-dir mountPath: /csi - name: csi-snapshotter - image: registry.k8s.io/sig-storage/csi-snapshotter:v6.0.1 + image: registry.k8s.io/sig-storage/csi-snapshotter:v6.1.0 args: - "--csi-address=$(ADDRESS)" - "--v=1" From 5c8564c69d6e9c96b1c2101642a57fec279d5c54 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Oct 2022 20:26:36 +0000 Subject: [PATCH 2/3] rebase: bump actions/stale from 5 to 6 Bumps [actions/stale](https://github.com/actions/stale) from 5 to 6. - [Release notes](https://github.com/actions/stale/releases) - [Changelog](https://github.com/actions/stale/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/stale/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/stale dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/stale.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stale.yaml b/.github/workflows/stale.yaml index ee26d18bd..3a1d50ce6 100644 --- a/.github/workflows/stale.yaml +++ b/.github/workflows/stale.yaml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest if: github.repository == 'ceph/ceph-csi' steps: - - uses: actions/stale@v5 + - uses: actions/stale@v6 with: repo-token: ${{ secrets.GITHUB_TOKEN }} days-before-issue-stale: 30 From 71e5b3f92241359b97b9eda8747cf291b26137aa Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Wed, 5 Oct 2022 17:09:50 +0200 Subject: [PATCH 3/3] rbd: remove dummy image workaround To address the problem that snapshot schedules are triggered for volumes that are promoted, a dummy image was disabled/enabled for replication. This was done as a workaround, because the promote operation was not triggering the schedules for the image being promoted. The bugs related to the same have been fixed in RBD mirroring functionality and hence the workaround #2656 can be removed from the code base. ceph tracker https://tracker.ceph.com/issues/53914 Signed-off-by: Madhu Rajanna --- internal/rbd/replicationcontrollerserver.go | 141 -------------------- 1 file changed, 141 deletions(-) diff --git a/internal/rbd/replicationcontrollerserver.go b/internal/rbd/replicationcontrollerserver.go index 7b7082c4e..bad710061 100644 --- a/internal/rbd/replicationcontrollerserver.go +++ b/internal/rbd/replicationcontrollerserver.go @@ -24,7 +24,6 @@ import ( "regexp" "strconv" "strings" - "sync" "time" "github.com/ceph/ceph-csi/internal/util" @@ -70,18 +69,6 @@ const ( schedulingStartTimeKey = "schedulingStartTime" ) -type operation string - -var ( - // pool+"/"+key to check dummy image is created. - dummyImageCreated operation = "dummyImageCreated" - // Read write lock to ensure that only one operation is happening at a time. - operationLock = sync.Map{} - - // Lock to serialize operations on the dummy image to tickle RBD snapshot schedule. - dummyImageOpsLock sync.Mutex -) - // ReplicationServer struct of rbd CSI driver with supported methods of Replication // controller server spec. type ReplicationServer struct { @@ -269,11 +256,6 @@ func (rs *ReplicationServer) EnableVolumeReplication(ctx context.Context, return nil, status.Error(codes.Internal, err.Error()) } - err = createDummyImage(ctx, rbdVol) - if err != nil { - return nil, status.Errorf(codes.Internal, "failed to create dummy image %s", err.Error()) - } - if mirroringInfo.State != librbd.MirrorImageEnabled { err = rbdVol.enableImageMirroring(mirroringMode) if err != nil { @@ -286,117 +268,6 @@ func (rs *ReplicationServer) EnableVolumeReplication(ctx context.Context, return &replication.EnableVolumeReplicationResponse{}, nil } -// getDummyImageName returns the csi-vol-dummy+cluster FSID as the image name. -// each cluster should have a unique dummy image created. choosing the cluster -// FSID for the same reason. -func getDummyImageName(conn *util.ClusterConnection) (string, error) { - id, err := conn.GetFSID() - if err != nil { - return "", err - } - - return fmt.Sprintf("csi-vol-dummy-%s", id), nil -} - -// getOperationName returns the operation name for the given operation type -// combined with the pool name. -func getOperationName(poolName string, optName operation) string { - return fmt.Sprintf("%s/%s", poolName, optName) -} - -// createDummyImage creates a dummy image as a workaround for the rbd -// scheduling problem. -func createDummyImage(ctx context.Context, rbdVol *rbdVolume) error { - var err error - var imgName string - - dummyImageOpsLock.Lock() - defer dummyImageOpsLock.Unlock() - optName := getOperationName(rbdVol.Pool, dummyImageCreated) - if _, ok := operationLock.Load(optName); !ok { - // create a dummy image - imgName, err = getDummyImageName(rbdVol.conn) - if err != nil { - return err - } - dummyVol := *rbdVol - dummyVol.RbdImageName = imgName - // dummyVol holds rbdVol details, reset ImageID or else dummy image cannot be - // deleted from trash during repair operation. - dummyVol.ImageID = "" - f := []string{ - librbd.FeatureNameLayering, - librbd.FeatureNameObjectMap, - librbd.FeatureNameExclusiveLock, - librbd.FeatureNameFastDiff, - } - features := librbd.FeatureSetFromNames(f) - dummyVol.ImageFeatureSet = features - // create 1MiB dummy image. 1MiB=1048576 bytes - dummyVol.VolSize = 1048576 - err = createImage(ctx, &dummyVol, dummyVol.conn.Creds) - if err != nil { - if strings.Contains(err.Error(), "File exists") { - err = repairDummyImage(ctx, &dummyVol) - } - } - if err == nil { - operationLock.Store(optName, true) - } - } - - return err -} - -// repairDummyImage deletes and recreates the dummy image. -func repairDummyImage(ctx context.Context, dummyVol *rbdVolume) error { - // instead of checking the images features and than adding missing image - // features, updating the image size to 1Mib. We will delete the image - // and recreate it. - - // deleting and recreating the dummy image will not impact anything as its - // a workaround to fix the scheduling problem. - err := dummyVol.deleteImage(ctx) - if err != nil { - return err - } - - return createImage(ctx, dummyVol, dummyVol.conn.Creds) -} - -// tickleMirroringOnDummyImage disables and reenables mirroring on the dummy image, and sets a -// schedule of a minute for the dummy image, to force a schedule refresh for other mirrored images -// within a minute. -func tickleMirroringOnDummyImage(rbdVol *rbdVolume, mirroringMode librbd.ImageMirrorMode) error { - imgName, err := getDummyImageName(rbdVol.conn) - if err != nil { - return err - } - dummyVol := *rbdVol - dummyVol.RbdImageName = imgName - - dummyImageOpsLock.Lock() - defer dummyImageOpsLock.Unlock() - err = dummyVol.disableImageMirroring(false) - if err != nil { - return err - } - - err = dummyVol.enableImageMirroring(mirroringMode) - if err != nil { - return err - } - - if mirroringMode == librbd.ImageMirrorModeSnapshot { - err = dummyVol.addSnapshotScheduling(admin.Interval("1m"), admin.NoStartTime) - if err != nil { - return err - } - } - - return nil -} - // DisableVolumeReplication extracts the RBD volume information from the // volumeID, If the image is present and the mirroring is enabled on the RBD // image it will disable the mirroring. @@ -588,12 +459,6 @@ func (rs *ReplicationServer) PromoteVolume(ctx context.Context, } } - var mode librbd.ImageMirrorMode - mode, err = getMirroringMode(ctx, req.GetParameters()) - if err != nil { - return nil, status.Errorf(codes.Internal, "failed to get mirroring mode %s", err.Error()) - } - interval, startTime := getSchedulingDetails(req.GetParameters()) if interval != admin.NoInterval { err = rbdVol.addSnapshotScheduling(interval, startTime) @@ -608,12 +473,6 @@ func (rs *ReplicationServer) PromoteVolume(ctx context.Context, rbdVol) } - log.DebugLog(ctx, "attempting to tickle dummy image for restarting RBD schedules") - err = tickleMirroringOnDummyImage(rbdVol, mode) - if err != nil { - return nil, status.Errorf(codes.Internal, "failed to enable mirroring on dummy image %s", err.Error()) - } - return &replication.PromoteVolumeResponse{}, nil }