From ab5ca13586af7af4841f1bc28fb2fc05b074a274 Mon Sep 17 00:00:00 2001 From: Niels de Vos Date: Thu, 7 Apr 2022 17:26:38 +0200 Subject: [PATCH 01/11] e2e: return useful error message when ConfigMap creation fails In case the toolbox pod is not available, the error message lists that no Pods are found, but there is no hint about the toolbox. By mentioning the toolbox in the error message, it suggests a good place to start troubleshooting the environment. Signed-off-by: Niels de Vos --- e2e/configmap.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/configmap.go b/e2e/configmap.go index 932f37fe0..1f90edb85 100644 --- a/e2e/configmap.go +++ b/e2e/configmap.go @@ -47,7 +47,7 @@ func createConfigMap(pluginPath string, c kubernetes.Interface, f *framework.Fra fsID, stdErr, err := execCommandInToolBoxPod(f, "ceph fsid", rookNamespace) if err != nil { - return err + return fmt.Errorf("failed to exec command in toolbox: %w", err) } if stdErr != "" { return fmt.Errorf("error getting fsid %v", stdErr) From 92866f46fdaf4c8c8caaa0055ec4fd7215b08e98 Mon Sep 17 00:00:00 2001 From: Niels de Vos Date: Thu, 7 Apr 2022 17:36:45 +0200 Subject: [PATCH 02/11] e2e: allow passing CephFS filesystem name on CLI A new -filesystem=... option has been added so that the e2e tests can run against environments that do not have a "myfs" CephFS filesystem. Signed-off-by: Niels de Vos --- e2e/README.md | 1 + e2e/cephfs_helper.go | 4 ++-- e2e/e2e_test.go | 1 + e2e/staticpvc.go | 13 ++++++------- e2e/utils.go | 2 +- 5 files changed, 11 insertions(+), 10 deletions(-) diff --git a/e2e/README.md b/e2e/README.md index 8dccda124..22765f706 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -101,6 +101,7 @@ are available while running tests: | kubeconfig | Path to kubeconfig containing embedded authinfo (default: $HOME/.kube/config) | | timeout | Panic test binary after duration d (default 0, timeout disabled) | | v | Verbose: print additional output | +| filesystem | Name of the CephFS filesystem (default: "myfs") | ## E2E for snapshot diff --git a/e2e/cephfs_helper.go b/e2e/cephfs_helper.go index e9d7bfd13..7ceadf6d8 100644 --- a/e2e/cephfs_helper.go +++ b/e2e/cephfs_helper.go @@ -40,7 +40,7 @@ const ( // validateSubvolumegroup validates whether subvolumegroup is present. func validateSubvolumegroup(f *framework.Framework, subvolgrp string) error { - cmd := fmt.Sprintf("ceph fs subvolumegroup getpath myfs %s", subvolgrp) + cmd := fmt.Sprintf("ceph fs subvolumegroup getpath %s %s", fileSystemName, subvolgrp) stdOut, stdErr, err := execCommandInToolBoxPod(f, cmd, rookNamespace) if err != nil { return fmt.Errorf("failed to exec command in toolbox: %w", err) @@ -67,7 +67,7 @@ func createCephfsStorageClass( if err != nil { return err } - sc.Parameters["fsName"] = "myfs" + sc.Parameters["fsName"] = fileSystemName sc.Parameters["csi.storage.k8s.io/provisioner-secret-namespace"] = cephCSINamespace sc.Parameters["csi.storage.k8s.io/provisioner-secret-name"] = cephFSProvisionerSecretName diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index f9dc4a0a6..01b4d0dbb 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -43,6 +43,7 @@ func init() { flag.StringVar(&upgradeVersion, "upgrade-version", "v3.5.1", "target version for upgrade testing") flag.StringVar(&cephCSINamespace, "cephcsi-namespace", defaultNs, "namespace in which cephcsi deployed") flag.StringVar(&rookNamespace, "rook-namespace", "rook-ceph", "namespace in which rook is deployed") + flag.StringVar(&fileSystemName, "filesystem", "myfs", "CephFS filesystem to use") setDefaultKubeconfig() // Register framework flags, then handle flags diff --git a/e2e/staticpvc.go b/e2e/staticpvc.go index 4e75a051d..c3ad075bd 100644 --- a/e2e/staticpvc.go +++ b/e2e/staticpvc.go @@ -331,7 +331,6 @@ func validateCephFsStaticPV(f *framework.Framework, appPath, scPath string) erro var ( cephFsVolName = "testSubVol" groupName = "testGroup" - fsName = "myfs" pvName = "pv-name" pvcName = "pvc-name" namespace = f.UniqueName @@ -361,7 +360,7 @@ func validateCephFsStaticPV(f *framework.Framework, appPath, scPath string) erro size := "4294967296" // create subvolumegroup, command will work even if group is already present. - cmd := fmt.Sprintf("ceph fs subvolumegroup create %s %s", fsName, groupName) + cmd := fmt.Sprintf("ceph fs subvolumegroup create %s %s", fileSystemName, groupName) _, e, err = execCommandInPod(f, cmd, rookNamespace, &listOpt) if err != nil { @@ -372,7 +371,7 @@ func validateCephFsStaticPV(f *framework.Framework, appPath, scPath string) erro } // create subvolume - cmd = fmt.Sprintf("ceph fs subvolume create %s %s %s --size %s", fsName, cephFsVolName, groupName, size) + cmd = fmt.Sprintf("ceph fs subvolume create %s %s %s --size %s", fileSystemName, cephFsVolName, groupName, size) _, e, err = execCommandInPod(f, cmd, rookNamespace, &listOpt) if err != nil { return err @@ -382,7 +381,7 @@ func validateCephFsStaticPV(f *framework.Framework, appPath, scPath string) erro } // get rootpath - cmd = fmt.Sprintf("ceph fs subvolume getpath %s %s %s", fsName, cephFsVolName, groupName) + cmd = fmt.Sprintf("ceph fs subvolume getpath %s %s %s", fileSystemName, cephFsVolName, groupName) rootPath, e, err := execCommandInPod(f, cmd, rookNamespace, &listOpt) if err != nil { return err @@ -415,7 +414,7 @@ func validateCephFsStaticPV(f *framework.Framework, appPath, scPath string) erro } opt["clusterID"] = fsID - opt["fsName"] = fsName + opt["fsName"] = fileSystemName opt["staticVolume"] = strconv.FormatBool(true) opt["rootPath"] = rootPath pv := getStaticPV( @@ -474,7 +473,7 @@ func validateCephFsStaticPV(f *framework.Framework, appPath, scPath string) erro } // delete subvolume - cmd = fmt.Sprintf("ceph fs subvolume rm %s %s %s", fsName, cephFsVolName, groupName) + cmd = fmt.Sprintf("ceph fs subvolume rm %s %s %s", fileSystemName, cephFsVolName, groupName) _, e, err = execCommandInPod(f, cmd, rookNamespace, &listOpt) if err != nil { return err @@ -484,7 +483,7 @@ func validateCephFsStaticPV(f *framework.Framework, appPath, scPath string) erro } // delete subvolume group - cmd = fmt.Sprintf("ceph fs subvolumegroup rm %s %s", fsName, groupName) + cmd = fmt.Sprintf("ceph fs subvolumegroup rm %s %s", fileSystemName, groupName) _, e, err = execCommandInPod(f, cmd, rookNamespace, &listOpt) if err != nil { return err diff --git a/e2e/utils.go b/e2e/utils.go index 8bc873d5f..5167d65d2 100644 --- a/e2e/utils.go +++ b/e2e/utils.go @@ -526,7 +526,7 @@ func pvcDeleteWhenPoolNotFound(pvcPath string, cephFS bool, f *framework.Framewo return err } // delete cephFS filesystem - err = deletePool("myfs", cephFS, f) + err = deletePool(fileSystemName, cephFS, f) if err != nil { return err } From 5bc8584b024e542117b682d7cdb594b7d12a83dc Mon Sep 17 00:00:00 2001 From: Niels de Vos Date: Thu, 7 Apr 2022 17:42:48 +0200 Subject: [PATCH 03/11] e2e: add -is-openshift option to disable certain checks On OpenShift it is not possible for the Rook toolbox to get the metrics from Kubelet (without additional configuration). By passing -is-openshift, the metrics are not checked, and the e2e suite does not fail on that particular piece. Signed-off-by: Niels de Vos --- e2e/README.md | 1 + e2e/e2e_test.go | 1 + e2e/utils.go | 3 ++- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/e2e/README.md b/e2e/README.md index 22765f706..d10783292 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -102,6 +102,7 @@ are available while running tests: | timeout | Panic test binary after duration d (default 0, timeout disabled) | | v | Verbose: print additional output | | filesystem | Name of the CephFS filesystem (default: "myfs") | +| is-openshift | Run in OpenShift compatibility mode, skips certain new feature tests | ## E2E for snapshot diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index 01b4d0dbb..aa5bd3d69 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -44,6 +44,7 @@ func init() { flag.StringVar(&cephCSINamespace, "cephcsi-namespace", defaultNs, "namespace in which cephcsi deployed") flag.StringVar(&rookNamespace, "rook-namespace", "rook-ceph", "namespace in which rook is deployed") flag.StringVar(&fileSystemName, "filesystem", "myfs", "CephFS filesystem to use") + flag.BoolVar(&isOpenShift, "is-openshift", false, "disables certain checks on OpenShift") setDefaultKubeconfig() // Register framework flags, then handle flags diff --git a/e2e/utils.go b/e2e/utils.go index 5167d65d2..2edcb0747 100644 --- a/e2e/utils.go +++ b/e2e/utils.go @@ -78,6 +78,7 @@ var ( rookNamespace string radosNamespace string poll = 2 * time.Second + isOpenShift bool ) func getMons(ns string, c kubernetes.Interface) ([]string, error) { @@ -404,7 +405,7 @@ func validateNormalUserPVCAccess(pvcPath string, f *framework.Framework) error { if pvc.Spec.VolumeMode != nil { isBlockMode = (*pvc.Spec.VolumeMode == v1.PersistentVolumeBlock) } - if !isBlockMode || k8sVersionGreaterEquals(f.ClientSet, 1, 22) { + if (!isBlockMode || k8sVersionGreaterEquals(f.ClientSet, 1, 22)) && !isOpenShift { err = getMetricsForPVC(f, pvc, deployTimeout) if err != nil { return err From 5e23010e2b3aa831c276323b643f3a2b0dd0fcc0 Mon Sep 17 00:00:00 2001 From: Humble Chirammal Date: Mon, 11 Apr 2022 16:51:24 +0530 Subject: [PATCH 04/11] e2e: remove the release check in clone test validation we no longer require the kubernetes validation for clone tests in the e2e tests. This commit remove it for RBD. Signed-off-by: Humble Chirammal --- e2e/upgrade-rbd.go | 243 +++++++++++++++++++++------------------------ 1 file changed, 115 insertions(+), 128 deletions(-) diff --git a/e2e/upgrade-rbd.go b/e2e/upgrade-rbd.go index 3625b827d..1b5e57ea6 100644 --- a/e2e/upgrade-rbd.go +++ b/e2e/upgrade-rbd.go @@ -250,19 +250,17 @@ var _ = Describe("RBD Upgrade Testing", func() { e2elog.Failf("failed to calculate checksum: %v", err) } - // pvc clone is only supported from v1.16+ - if k8sVersionGreaterEquals(f.ClientSet, 1, 16) { - // Create snapshot of the pvc - snapshotPath := rbdExamplePath + "snapshot.yaml" - snap := getSnapshot(snapshotPath) - snap.Name = "rbd-pvc-snapshot" - snap.Namespace = f.UniqueName - snap.Spec.Source.PersistentVolumeClaimName = &pvc.Name - err = createSnapshot(&snap, deployTimeout) - if err != nil { - e2elog.Failf("failed to create snapshot %v", err) - } + // Create snapshot of the pvc + snapshotPath := rbdExamplePath + "snapshot.yaml" + snap := getSnapshot(snapshotPath) + snap.Name = "rbd-pvc-snapshot" + snap.Namespace = f.UniqueName + snap.Spec.Source.PersistentVolumeClaimName = &pvc.Name + err = createSnapshot(&snap, deployTimeout) + if err != nil { + e2elog.Failf("failed to create snapshot %v", err) } + err = deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout) if err != nil { e2elog.Failf("failed to delete application: %v", err) @@ -299,52 +297,48 @@ var _ = Describe("RBD Upgrade Testing", func() { pvcClonePath := rbdExamplePath + "pvc-restore.yaml" appClonePath := rbdExamplePath + "pod-restore.yaml" label := make(map[string]string) - - // pvc clone is only supported from v1.16+ - if k8sVersionGreaterEquals(f.ClientSet, 1, 16) { - pvcClone, err := loadPVC(pvcClonePath) - if err != nil { - e2elog.Failf("failed to load pvc: %v", err) - } - pvcClone.Namespace = f.UniqueName - pvcClone.Spec.Resources.Requests[v1.ResourceStorage] = resource.MustParse(pvcSize) - pvcClone.Spec.DataSource.Name = "rbd-pvc-snapshot" - appClone, err := loadApp(appClonePath) - if err != nil { - e2elog.Failf("failed to load application: %v", err) - } - label[appKey] = "validate-snap-clone" - appClone.Namespace = f.UniqueName - appClone.Name = "app-clone-from-snap" - appClone.Labels = label - err = createPVCAndApp("", f, pvcClone, appClone, deployTimeout) - if err != nil { - e2elog.Failf("failed to create pvc: %v", err) - } - opt := metav1.ListOptions{ - LabelSelector: fmt.Sprintf("%s=%s", appKey, label[appKey]), - } - mountPath := appClone.Spec.Containers[0].VolumeMounts[0].MountPath - testFilePath := filepath.Join(mountPath, "testClone") - newCheckSum, err := calculateSHA512sum(f, appClone, testFilePath, &opt) - if err != nil { - e2elog.Failf("failed to calculate checksum: %v", err) - } - if strings.Compare(newCheckSum, checkSum) != 0 { - e2elog.Failf( - "The checksum of files did not match, expected %s received %s", - checkSum, - newCheckSum) - } - e2elog.Logf("The checksum of files matched") - - // delete cloned pvc and pod - err = deletePVCAndApp("", f, pvcClone, appClone) - if err != nil { - e2elog.Failf("failed to delete pvc and application: %v", err) - } - + pvcClone, err := loadPVC(pvcClonePath) + if err != nil { + e2elog.Failf("failed to load pvc: %v", err) } + pvcClone.Namespace = f.UniqueName + pvcClone.Spec.Resources.Requests[v1.ResourceStorage] = resource.MustParse(pvcSize) + pvcClone.Spec.DataSource.Name = "rbd-pvc-snapshot" + appClone, err := loadApp(appClonePath) + if err != nil { + e2elog.Failf("failed to load application: %v", err) + } + label[appKey] = "validate-snap-clone" + appClone.Namespace = f.UniqueName + appClone.Name = "app-clone-from-snap" + appClone.Labels = label + err = createPVCAndApp("", f, pvcClone, appClone, deployTimeout) + if err != nil { + e2elog.Failf("failed to create pvc: %v", err) + } + opt := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", appKey, label[appKey]), + } + mountPath := appClone.Spec.Containers[0].VolumeMounts[0].MountPath + testFilePath := filepath.Join(mountPath, "testClone") + newCheckSum, err := calculateSHA512sum(f, appClone, testFilePath, &opt) + if err != nil { + e2elog.Failf("failed to calculate checksum: %v", err) + } + if strings.Compare(newCheckSum, checkSum) != 0 { + e2elog.Failf( + "The checksum of files did not match, expected %s received %s", + checkSum, + newCheckSum) + } + e2elog.Logf("The checksum of files matched") + + // delete cloned pvc and pod + err = deletePVCAndApp("", f, pvcClone, appClone) + if err != nil { + e2elog.Failf("failed to delete pvc and application: %v", err) + } + }) By("Create clone from existing PVC", func() { @@ -352,50 +346,46 @@ var _ = Describe("RBD Upgrade Testing", func() { appSmartClonePath := rbdExamplePath + "pod-clone.yaml" label := make(map[string]string) - // pvc clone is only supported from v1.16+ - if k8sVersionGreaterEquals(f.ClientSet, 1, 16) { - pvcClone, err := loadPVC(pvcSmartClonePath) - if err != nil { - e2elog.Failf("failed to load pvc: %v", err) - } - pvcClone.Spec.DataSource.Name = pvc.Name - pvcClone.Namespace = f.UniqueName - pvcClone.Spec.Resources.Requests[v1.ResourceStorage] = resource.MustParse(pvcSize) - appClone, err := loadApp(appSmartClonePath) - if err != nil { - e2elog.Failf("failed to load application: %v", err) - } - label[appKey] = "validate-clone" - appClone.Namespace = f.UniqueName - appClone.Name = "appclone" - appClone.Labels = label - err = createPVCAndApp("", f, pvcClone, appClone, deployTimeout) - if err != nil { - e2elog.Failf("failed to create pvc: %v", err) - } - opt := metav1.ListOptions{ - LabelSelector: fmt.Sprintf("%s=%s", appKey, label[appKey]), - } - mountPath := appClone.Spec.Containers[0].VolumeMounts[0].MountPath - testFilePath := filepath.Join(mountPath, "testClone") - newCheckSum, err := calculateSHA512sum(f, appClone, testFilePath, &opt) - if err != nil { - e2elog.Failf("failed to calculate checksum: %v", err) - } - if strings.Compare(newCheckSum, checkSum) != 0 { - e2elog.Failf( - "The checksum of files did not match, expected %s received %s", - checkSum, - newCheckSum) - } - e2elog.Logf("The checksum of files matched") - - // delete cloned pvc and pod - err = deletePVCAndApp("", f, pvcClone, appClone) - if err != nil { - e2elog.Failf("failed to delete pvc and application: %v", err) - } + pvcClone, err := loadPVC(pvcSmartClonePath) + if err != nil { + e2elog.Failf("failed to load pvc: %v", err) + } + pvcClone.Spec.DataSource.Name = pvc.Name + pvcClone.Namespace = f.UniqueName + pvcClone.Spec.Resources.Requests[v1.ResourceStorage] = resource.MustParse(pvcSize) + appClone, err := loadApp(appSmartClonePath) + if err != nil { + e2elog.Failf("failed to load application: %v", err) + } + label[appKey] = "validate-clone" + appClone.Namespace = f.UniqueName + appClone.Name = "appclone" + appClone.Labels = label + err = createPVCAndApp("", f, pvcClone, appClone, deployTimeout) + if err != nil { + e2elog.Failf("failed to create pvc: %v", err) + } + opt := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", appKey, label[appKey]), + } + mountPath := appClone.Spec.Containers[0].VolumeMounts[0].MountPath + testFilePath := filepath.Join(mountPath, "testClone") + newCheckSum, err := calculateSHA512sum(f, appClone, testFilePath, &opt) + if err != nil { + e2elog.Failf("failed to calculate checksum: %v", err) + } + if strings.Compare(newCheckSum, checkSum) != 0 { + e2elog.Failf( + "The checksum of files did not match, expected %s received %s", + checkSum, + newCheckSum) + } + e2elog.Logf("The checksum of files matched") + // delete cloned pvc and pod + err = deletePVCAndApp("", f, pvcClone, appClone) + if err != nil { + e2elog.Failf("failed to delete pvc and application: %v", err) } }) @@ -403,35 +393,32 @@ var _ = Describe("RBD Upgrade Testing", func() { pvcExpandSize := "5Gi" label := make(map[string]string) - // Resize 0.3.0 is only supported from v1.15+ - if k8sVersionGreaterEquals(f.ClientSet, 1, 15) { - label[appKey] = appLabel - opt := metav1.ListOptions{ - LabelSelector: fmt.Sprintf("%s=%s", appKey, label[appKey]), - } - var err error - pvc, err = f.ClientSet.CoreV1(). - PersistentVolumeClaims(pvc.Namespace). - Get(context.TODO(), pvc.Name, metav1.GetOptions{}) - if err != nil { - e2elog.Failf("failed to get pvc: %v", err) - } + label[appKey] = appLabel + opt := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", appKey, label[appKey]), + } + var err error + pvc, err = f.ClientSet.CoreV1(). + PersistentVolumeClaims(pvc.Namespace). + Get(context.TODO(), pvc.Name, metav1.GetOptions{}) + if err != nil { + e2elog.Failf("failed to get pvc: %v", err) + } - // resize PVC - err = expandPVCSize(f.ClientSet, pvc, pvcExpandSize, deployTimeout) - if err != nil { - e2elog.Failf("failed to expand pvc: %v", err) - } - // wait for application pod to come up after resize - err = waitForPodInRunningState(app.Name, app.Namespace, f.ClientSet, deployTimeout, noError) - if err != nil { - e2elog.Failf("timeout waiting for pod to be in running state: %v", err) - } - // validate if resize is successful. - err = checkDirSize(app, f, &opt, pvcExpandSize) - if err != nil { - e2elog.Failf("failed to check directory size: %v", err) - } + // resize PVC + err = expandPVCSize(f.ClientSet, pvc, pvcExpandSize, deployTimeout) + if err != nil { + e2elog.Failf("failed to expand pvc: %v", err) + } + // wait for application pod to come up after resize + err = waitForPodInRunningState(app.Name, app.Namespace, f.ClientSet, deployTimeout, noError) + if err != nil { + e2elog.Failf("timeout waiting for pod to be in running state: %v", err) + } + // validate if resize is successful. + err = checkDirSize(app, f, &opt, pvcExpandSize) + if err != nil { + e2elog.Failf("failed to check directory size: %v", err) } }) From b64c7583a9733b471b261de1a32f3e65af2d5b5e Mon Sep 17 00:00:00 2001 From: Humble Chirammal Date: Mon, 11 Apr 2022 16:56:21 +0530 Subject: [PATCH 05/11] e2e: remove the release check in clone test validation we no longer require the kubernetes validation for clone tests in the e2e tests. This commit remove it for CephFS. Signed-off-by: Humble Chirammal --- e2e/upgrade-cephfs.go | 257 ++++++++++++++++++++---------------------- e2e/upgrade-rbd.go | 1 - 2 files changed, 120 insertions(+), 138 deletions(-) diff --git a/e2e/upgrade-cephfs.go b/e2e/upgrade-cephfs.go index 235302abd..4bd2c36bf 100644 --- a/e2e/upgrade-cephfs.go +++ b/e2e/upgrade-cephfs.go @@ -230,19 +230,15 @@ var _ = Describe("CephFS Upgrade Testing", func() { if err != nil { e2elog.Failf("failed to calculate checksum: %v", err) } - - // pvc clone is only supported from v1.16+ - if k8sVersionGreaterEquals(f.ClientSet, 1, 17) { - // Create snapshot of the pvc - snapshotPath := cephFSExamplePath + "snapshot.yaml" - snap := getSnapshot(snapshotPath) - snap.Name = "cephfs-pvc-snapshot" - snap.Namespace = f.UniqueName - snap.Spec.Source.PersistentVolumeClaimName = &pvc.Name - err = createSnapshot(&snap, deployTimeout) - if err != nil { - e2elog.Failf("failed to create snapshot %v", err) - } + // Create snapshot of the pvc + snapshotPath := cephFSExamplePath + "snapshot.yaml" + snap := getSnapshot(snapshotPath) + snap.Name = "cephfs-pvc-snapshot" + snap.Namespace = f.UniqueName + snap.Spec.Source.PersistentVolumeClaimName = &pvc.Name + err = createSnapshot(&snap, deployTimeout) + if err != nil { + e2elog.Failf("failed to create snapshot %v", err) } err = deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout) if err != nil { @@ -280,62 +276,56 @@ var _ = Describe("CephFS Upgrade Testing", func() { pvcClonePath := cephFSExamplePath + "pvc-restore.yaml" appClonePath := cephFSExamplePath + "pod-restore.yaml" label := make(map[string]string) + pvcClone, err = loadPVC(pvcClonePath) + if err != nil { + e2elog.Failf("failed to load pvc: %v", err) + } + pvcClone.Namespace = f.UniqueName + pvcClone.Spec.Resources.Requests[v1.ResourceStorage] = resource.MustParse(pvcSize) + appClone, err = loadApp(appClonePath) + if err != nil { + e2elog.Failf("failed to load application: %v", err) + } + label[appKey] = "validate-snap-cephfs" + appClone.Namespace = f.UniqueName + appClone.Name = "snap-clone-cephfs" + appClone.Labels = label + err = createPVCAndApp("", f, pvcClone, appClone, deployTimeout) + if err != nil { + e2elog.Failf("failed to create pvc and application: %v", err) + } + opt := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", appKey, label[appKey]), + } + mountPath := appClone.Spec.Containers[0].VolumeMounts[0].MountPath + testFilePath := filepath.Join(mountPath, "testClone") + newCheckSum, err = calculateSHA512sum(f, appClone, testFilePath, &opt) + if err != nil { + e2elog.Failf("failed to calculate checksum: %v", err) + } + if strings.Compare(newCheckSum, checkSum) != 0 { + e2elog.Failf( + "The checksum of files did not match, expected %s received %s ", + checkSum, + newCheckSum) + } + e2elog.Logf("The checksum of files matched") - // pvc clone is only supported from v1.16+ - if k8sVersionGreaterEquals(f.ClientSet, 1, 17) { - pvcClone, err = loadPVC(pvcClonePath) - if err != nil { - e2elog.Failf("failed to load pvc: %v", err) - } - pvcClone.Namespace = f.UniqueName - pvcClone.Spec.Resources.Requests[v1.ResourceStorage] = resource.MustParse(pvcSize) - appClone, err = loadApp(appClonePath) - if err != nil { - e2elog.Failf("failed to load application: %v", err) - } - label[appKey] = "validate-snap-cephfs" - appClone.Namespace = f.UniqueName - appClone.Name = "snap-clone-cephfs" - appClone.Labels = label - err = createPVCAndApp("", f, pvcClone, appClone, deployTimeout) - if err != nil { - e2elog.Failf("failed to create pvc and application: %v", err) - } - opt := metav1.ListOptions{ - LabelSelector: fmt.Sprintf("%s=%s", appKey, label[appKey]), - } - mountPath := appClone.Spec.Containers[0].VolumeMounts[0].MountPath - testFilePath := filepath.Join(mountPath, "testClone") - newCheckSum, err = calculateSHA512sum(f, appClone, testFilePath, &opt) - if err != nil { - e2elog.Failf("failed to calculate checksum: %v", err) - } - - if strings.Compare(newCheckSum, checkSum) != 0 { - e2elog.Failf( - "The checksum of files did not match, expected %s received %s ", - checkSum, - newCheckSum) - } - e2elog.Logf("The checksum of files matched") - - // delete cloned pvc and pod - err = deletePVCAndApp("", f, pvcClone, appClone) - if err != nil { - Fail(err.Error()) - } - - // Delete the snapshot of the parent pvc. - snapshotPath := cephFSExamplePath + "snapshot.yaml" - snap := getSnapshot(snapshotPath) - snap.Name = "cephfs-pvc-snapshot" - snap.Namespace = f.UniqueName - snap.Spec.Source.PersistentVolumeClaimName = &pvc.Name - err = deleteSnapshot(&snap, deployTimeout) - if err != nil { - e2elog.Failf("failed to delete snapshot %v", err) - } + // delete cloned pvc and pod + err = deletePVCAndApp("", f, pvcClone, appClone) + if err != nil { + Fail(err.Error()) + } + // Delete the snapshot of the parent pvc. + snapshotPath := cephFSExamplePath + "snapshot.yaml" + snap := getSnapshot(snapshotPath) + snap.Name = "cephfs-pvc-snapshot" + snap.Namespace = f.UniqueName + snap.Spec.Source.PersistentVolumeClaimName = &pvc.Name + err = deleteSnapshot(&snap, deployTimeout) + if err != nil { + e2elog.Failf("failed to delete snapshot %v", err) } }) @@ -344,51 +334,47 @@ var _ = Describe("CephFS Upgrade Testing", func() { appSmartClonePath := cephFSExamplePath + "pod-clone.yaml" label := make(map[string]string) - // pvc clone is only supported from v1.16+ - if k8sVersionGreaterEquals(f.ClientSet, 1, 16) { - pvcClone, err = loadPVC(pvcSmartClonePath) - if err != nil { - e2elog.Failf("failed to load pvc: %v", err) - } - pvcClone.Spec.DataSource.Name = pvc.Name - pvcClone.Namespace = f.UniqueName - pvcClone.Spec.Resources.Requests[v1.ResourceStorage] = resource.MustParse(pvcSize) - appClone, err = loadApp(appSmartClonePath) - if err != nil { - e2elog.Failf("failed to load application: %v", err) - } - label[appKey] = "validate-snap-cephfs" - appClone.Namespace = f.UniqueName - appClone.Name = "appclone" - appClone.Labels = label - err = createPVCAndApp("", f, pvcClone, appClone, deployTimeout) - if err != nil { - e2elog.Failf("failed to create pvc and application: %v", err) - } - opt := metav1.ListOptions{ - LabelSelector: fmt.Sprintf("%s=%s", appKey, label[appKey]), - } - mountPath := appClone.Spec.Containers[0].VolumeMounts[0].MountPath - testFilePath := filepath.Join(mountPath, "testClone") - newCheckSum, err = calculateSHA512sum(f, appClone, testFilePath, &opt) - if err != nil { - e2elog.Failf("failed to calculate checksum: %v", err) - } + pvcClone, err = loadPVC(pvcSmartClonePath) + if err != nil { + e2elog.Failf("failed to load pvc: %v", err) + } + pvcClone.Spec.DataSource.Name = pvc.Name + pvcClone.Namespace = f.UniqueName + pvcClone.Spec.Resources.Requests[v1.ResourceStorage] = resource.MustParse(pvcSize) + appClone, err = loadApp(appSmartClonePath) + if err != nil { + e2elog.Failf("failed to load application: %v", err) + } + label[appKey] = "validate-snap-cephfs" + appClone.Namespace = f.UniqueName + appClone.Name = "appclone" + appClone.Labels = label + err = createPVCAndApp("", f, pvcClone, appClone, deployTimeout) + if err != nil { + e2elog.Failf("failed to create pvc and application: %v", err) + } + opt := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", appKey, label[appKey]), + } + mountPath := appClone.Spec.Containers[0].VolumeMounts[0].MountPath + testFilePath := filepath.Join(mountPath, "testClone") + newCheckSum, err = calculateSHA512sum(f, appClone, testFilePath, &opt) + if err != nil { + e2elog.Failf("failed to calculate checksum: %v", err) + } - if strings.Compare(newCheckSum, checkSum) != 0 { - e2elog.Failf( - "The checksum of files did not match, expected %s received %s", - checkSum, - newCheckSum) - } - e2elog.Logf("The checksum of files matched") - - // delete cloned pvc and pod - err = deletePVCAndApp("", f, pvcClone, appClone) - if err != nil { - e2elog.Failf("failed to delete pvc and application: %v", err) - } + if strings.Compare(newCheckSum, checkSum) != 0 { + e2elog.Failf( + "The checksum of files did not match, expected %s received %s", + checkSum, + newCheckSum) + } + e2elog.Logf("The checksum of files matched") + // delete cloned pvc and pod + err = deletePVCAndApp("", f, pvcClone, appClone) + if err != nil { + e2elog.Failf("failed to delete pvc and application: %v", err) } }) @@ -396,34 +382,31 @@ var _ = Describe("CephFS Upgrade Testing", func() { pvcExpandSize := "5Gi" label := make(map[string]string) - // Resize 0.3.0 is only supported from v1.15+ - if k8sVersionGreaterEquals(f.ClientSet, 1, 15) { - label[appKey] = appLabel - opt := metav1.ListOptions{ - LabelSelector: fmt.Sprintf("%s=%s", appKey, label[appKey]), - } - pvc, err = f.ClientSet.CoreV1(). - PersistentVolumeClaims(pvc.Namespace). - Get(context.TODO(), pvc.Name, metav1.GetOptions{}) - if err != nil { - e2elog.Failf("failed to get pvc: %v", err) - } + label[appKey] = appLabel + opt := metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s", appKey, label[appKey]), + } + pvc, err = f.ClientSet.CoreV1(). + PersistentVolumeClaims(pvc.Namespace). + Get(context.TODO(), pvc.Name, metav1.GetOptions{}) + if err != nil { + e2elog.Failf("failed to get pvc: %v", err) + } - // resize PVC - err = expandPVCSize(f.ClientSet, pvc, pvcExpandSize, deployTimeout) - if err != nil { - e2elog.Failf("failed to expand pvc: %v", err) - } - // wait for application pod to come up after resize - err = waitForPodInRunningState(app.Name, app.Namespace, f.ClientSet, deployTimeout, noError) - if err != nil { - e2elog.Failf("timeout waiting for pod to be in running state: %v", err) - } - // validate if resize is successful. - err = checkDirSize(app, f, &opt, pvcExpandSize) - if err != nil { - e2elog.Failf("failed to check directory size: %v", err) - } + // resize PVC + err = expandPVCSize(f.ClientSet, pvc, pvcExpandSize, deployTimeout) + if err != nil { + e2elog.Failf("failed to expand pvc: %v", err) + } + // wait for application pod to come up after resize + err = waitForPodInRunningState(app.Name, app.Namespace, f.ClientSet, deployTimeout, noError) + if err != nil { + e2elog.Failf("timeout waiting for pod to be in running state: %v", err) + } + // validate if resize is successful. + err = checkDirSize(app, f, &opt, pvcExpandSize) + if err != nil { + e2elog.Failf("failed to check directory size: %v", err) } }) diff --git a/e2e/upgrade-rbd.go b/e2e/upgrade-rbd.go index 1b5e57ea6..486fce7ed 100644 --- a/e2e/upgrade-rbd.go +++ b/e2e/upgrade-rbd.go @@ -338,7 +338,6 @@ var _ = Describe("RBD Upgrade Testing", func() { if err != nil { e2elog.Failf("failed to delete pvc and application: %v", err) } - }) By("Create clone from existing PVC", func() { From 64a9b1fa5906d65478cdb5fb244b133bc1b1cfbe Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Mon, 11 Apr 2022 10:05:07 +0530 Subject: [PATCH 06/11] rbd: consider remote image health for primary To consider the image is healthy during the Promote operation currently we are checking only the image state on the primary site. If the network is flaky or the remote site is down the image health is not as expected. To make sure the image is healthy across the clusters check the state on both local and the remote clusters. some details: https://bugzilla.redhat.com/show_bug.cgi?id=2014495 Signed-off-by: Madhu Rajanna --- internal/rbd/replicationcontrollerserver.go | 25 +++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/internal/rbd/replicationcontrollerserver.go b/internal/rbd/replicationcontrollerserver.go index 7ce3b4c98..edeb0c0d3 100644 --- a/internal/rbd/replicationcontrollerserver.go +++ b/internal/rbd/replicationcontrollerserver.go @@ -615,8 +615,9 @@ func (rs *ReplicationServer) PromoteVolume(ctx context.Context, } // checkHealthyPrimary checks if the image is a healhty primary or not. -// healthy primary image will be in up+stopped state, for states other -// than this it returns an error message. +// healthy primary image will be in up+stopped state in local cluster and +// up+replaying in the remote clusters, for states other than this it returns +// an error message. func checkHealthyPrimary(ctx context.Context, rbdVol *rbdVolume) error { mirrorStatus, err := rbdVol.getImageMirroringStatus() if err != nil { @@ -640,6 +641,26 @@ func checkHealthyPrimary(ctx context.Context, rbdVol *rbdVolume) error { localStatus.State) } + // Remote image should be in up+replaying state. + for _, s := range mirrorStatus.SiteStatuses { + log.UsefulLog( + ctx, + "peer site mirrorUUID=%q, daemon up=%t, mirroring state=%q, description=%q and lastUpdate=%d", + s.MirrorUUID, + s.Up, + s.State, + s.Description, + s.LastUpdate) + if s.MirrorUUID != "" { + if !s.Up && s.State != librbd.MirrorImageStatusStateReplaying { + return fmt.Errorf("remote image %s is not healthy. State is up=%t, state=%q", + rbdVol, + s.Up, + s.State) + } + } + } + return nil } From 784b086ea5ba1f57d14c67d57aecd8b5c2de6ad3 Mon Sep 17 00:00:00 2001 From: Rakshith R Date: Wed, 13 Apr 2022 11:43:05 +0530 Subject: [PATCH 07/11] nfs: add provisioner & plugin sa to scc.yaml This commit adds nfs provisioner & plugin sa to scc.yaml to be used with openshift. Signed-off-by: Rakshith R --- api/deploy/ocp/scc.yaml | 2 ++ deploy/scc.yaml | 2 ++ vendor/github.com/ceph/ceph-csi/api/deploy/ocp/scc.yaml | 2 ++ 3 files changed, 6 insertions(+) diff --git a/api/deploy/ocp/scc.yaml b/api/deploy/ocp/scc.yaml index 139bbdfdc..c5903f330 100644 --- a/api/deploy/ocp/scc.yaml +++ b/api/deploy/ocp/scc.yaml @@ -41,3 +41,5 @@ users: - "system:serviceaccount:{{ .Namespace }}:{{ .Prefix }}csi-cephfs-plugin-sa" # yamllint disable-line rule:line-length - "system:serviceaccount:{{ .Namespace }}:{{ .Prefix }}csi-cephfs-provisioner-sa" + - "system:serviceaccount:{{ .Namespace }}:{{ .Prefix }}csi-nfs-plugin-sa" + - "system:serviceaccount:{{ .Namespace }}:{{ .Prefix }}csi-nfs-provisioner-sa" diff --git a/deploy/scc.yaml b/deploy/scc.yaml index 36c1973a7..e4fb9595a 100644 --- a/deploy/scc.yaml +++ b/deploy/scc.yaml @@ -48,3 +48,5 @@ users: - "system:serviceaccount:ceph-csi:csi-cephfs-plugin-sa" # yamllint disable-line rule:line-length - "system:serviceaccount:ceph-csi:csi-cephfs-provisioner-sa" + - "system:serviceaccount:ceph-csi:csi-nfs-plugin-sa" + - "system:serviceaccount:ceph-csi:csi-nfs-provisioner-sa" diff --git a/vendor/github.com/ceph/ceph-csi/api/deploy/ocp/scc.yaml b/vendor/github.com/ceph/ceph-csi/api/deploy/ocp/scc.yaml index 139bbdfdc..c5903f330 100644 --- a/vendor/github.com/ceph/ceph-csi/api/deploy/ocp/scc.yaml +++ b/vendor/github.com/ceph/ceph-csi/api/deploy/ocp/scc.yaml @@ -41,3 +41,5 @@ users: - "system:serviceaccount:{{ .Namespace }}:{{ .Prefix }}csi-cephfs-plugin-sa" # yamllint disable-line rule:line-length - "system:serviceaccount:{{ .Namespace }}:{{ .Prefix }}csi-cephfs-provisioner-sa" + - "system:serviceaccount:{{ .Namespace }}:{{ .Prefix }}csi-nfs-plugin-sa" + - "system:serviceaccount:{{ .Namespace }}:{{ .Prefix }}csi-nfs-provisioner-sa" From 2205145654cdcfd87c8f9695745c02e391b0b540 Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Mon, 11 Apr 2022 19:54:19 +0530 Subject: [PATCH 08/11] e2e: remove claimRef after deleting the PVC Instead of patching the PV to update the persistentVolumeReclaimPolicy and the claimRef before deleting the PVC. Patch PV persistentVolumeReclaimPolicy to Retain to retain the PV after deleting the PVC. Remove the claimRef on the PV after deleting the PVC so that claim can be attached to a new PVC. Signed-off-by: Madhu Rajanna --- e2e/rbd.go | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/e2e/rbd.go b/e2e/rbd.go index a8a7d8c55..f708ca5d7 100644 --- a/e2e/rbd.go +++ b/e2e/rbd.go @@ -525,7 +525,8 @@ var _ = Describe("RBD", func() { e2elog.Logf("pv name is empty %q in namespace %q: %v", pvc.Name, pvc.Namespace, err) } - patchBytes := []byte(`{"spec":{"persistentVolumeReclaimPolicy": "Retain", "claimRef": null}}`) + // patch PV to Retain it after deleting the PVC. + patchBytes := []byte(`{"spec":{"persistentVolumeReclaimPolicy": "Retain"}}`) _, err = c.CoreV1().PersistentVolumes().Patch( context.TODO(), pvcObj.Spec.VolumeName, @@ -533,7 +534,7 @@ var _ = Describe("RBD", func() { patchBytes, metav1.PatchOptions{}) if err != nil { - e2elog.Logf("error Patching PV %q for persistentVolumeReclaimPolicy and claimRef: %v", + e2elog.Logf("error Patching PV %q for persistentVolumeReclaimPolicy: %v", pvcObj.Spec.VolumeName, err) } @@ -545,6 +546,19 @@ var _ = Describe("RBD", func() { e2elog.Logf("failed to delete pvc: %w", err) } + // Remove the claimRef to bind this PV to a new PVC. + patchBytes = []byte(`{"spec":{"claimRef": null}}`) + _, err = c.CoreV1().PersistentVolumes().Patch( + context.TODO(), + pvcObj.Spec.VolumeName, + types.StrategicMergePatchType, + patchBytes, + metav1.PatchOptions{}) + if err != nil { + e2elog.Logf("error Patching PV %q for claimRef: %v", + pvcObj.Spec.VolumeName, err) + } + // validate created backend rbd images validateRBDImageCount(f, 1, defaultRBDPool) From d886ab0d6634bca5a5b055bb7783138a2e3e5ded Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Mon, 11 Apr 2022 13:30:08 +0530 Subject: [PATCH 09/11] rbd: use leases for leader election use leases for leader election instead of the deprecated configmap based leader election. This PR is making leases as default leader election refer https://github.com/kubernetes-sigs/ controller-runtime/pull/1773, default from configmap to configmap leases was done with https://github.com/kubernetes-sigs/ controller-runtime/pull/1144. Release notes https://github.com/kubernetes-sigs/ controller-runtime/releases/tag/v0.7.0 Signed-off-by: Madhu Rajanna --- internal/controller/controller.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/internal/controller/controller.go b/internal/controller/controller.go index 518ff0271..5f1cc2fe2 100644 --- a/internal/controller/controller.go +++ b/internal/controller/controller.go @@ -20,6 +20,7 @@ import ( "github.com/ceph/ceph-csi/internal/util/log" + "k8s.io/client-go/tools/leaderelection/resourcelock" clientConfig "sigs.k8s.io/controller-runtime/pkg/client/config" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/manager/signals" @@ -59,9 +60,10 @@ func Start(config Config) error { opts := manager.Options{ LeaderElection: true, // disable metrics - MetricsBindAddress: "0", - LeaderElectionNamespace: config.Namespace, - LeaderElectionID: electionID, + MetricsBindAddress: "0", + LeaderElectionNamespace: config.Namespace, + LeaderElectionResourceLock: resourcelock.LeasesResourceLock, + LeaderElectionID: electionID, } mgr, err := manager.New(clientConfig.GetConfigOrDie(), opts) if err != nil { From 28369702d275381b559f77c7e38adf599128e457 Mon Sep 17 00:00:00 2001 From: Niels de Vos Date: Wed, 30 Mar 2022 10:45:07 +0200 Subject: [PATCH 10/11] nfs: use go-ceph API for creating/deleting exports Recent versions of Ceph allow calling the NFS-export management functions over the go-ceph API. This seems incompatible with older versions that have been tested with the `ceph nfs` commands that this commit replaces. Signed-off-by: Niels de Vos --- internal/nfs/controller/volume.go | 96 +++++++++++++++---------------- internal/util/connection.go | 11 ++++ 2 files changed, 57 insertions(+), 50 deletions(-) diff --git a/internal/nfs/controller/volume.go b/internal/nfs/controller/volume.go index c3c2f5bf7..2177ce904 100644 --- a/internal/nfs/controller/volume.go +++ b/internal/nfs/controller/volume.go @@ -26,6 +26,7 @@ import ( fsutil "github.com/ceph/ceph-csi/internal/cephfs/util" "github.com/ceph/ceph-csi/internal/util" + "github.com/ceph/go-ceph/common/admin/nfs" "github.com/container-storage-interface/spec/lib/go/csi" ) @@ -135,24 +136,33 @@ func (nv *NFSVolume) CreateExport(backend *csi.Volume) error { return fmt.Errorf("failed to set NFS-cluster: %w", err) } - // TODO: use new go-ceph API, see ceph/ceph-csi#2977 - // new versions of Ceph use a different command, and the go-ceph API - // also seems to be different :-/ - // - // run the new command, but fall back to the previous one in case of an - // error - cmds := [][]string{ - // ceph nfs export create cephfs --cluster-id - // --pseudo-path --fsname - // [--readonly] [--path=/path/in/cephfs] - nv.createExportCommand("--cluster-id="+nfsCluster, - "--fsname="+fs, "--pseudo-path="+nv.GetExportPath(), - "--path="+path), - // ceph nfs export create cephfs ${FS} ${NFS} /${EXPORT} ${SUBVOL_PATH} - nv.createExportCommand(nfsCluster, fs, nv.GetExportPath(), path), + nfsa, err := nv.conn.GetNFSAdmin() + if err != nil { + return fmt.Errorf("failed to get NFSAdmin: %w", err) } - stderr, err := nv.retryIfInvalid(cmds) + _, err = nfsa.CreateCephFSExport(nfs.CephFSExportSpec{ + FileSystemName: fs, + ClusterID: nfsCluster, + PseudoPath: nv.GetExportPath(), + Path: path, + }) + switch { + case err == nil: + return nil + case strings.Contains(err.Error(), "rados: ret=-2"): // try with the old command + break + default: // any other error + return fmt.Errorf("exporting %q on NFS-cluster %q failed: %w", + nv, nfsCluster, err) + } + + // if we get here, the API call failed, fallback to the old command + + // ceph nfs export create cephfs ${FS} ${NFS} /${EXPORT} ${SUBVOL_PATH} + cmd := nv.createExportCommand(nfsCluster, fs, nv.GetExportPath(), path) + + _, stderr, err := util.ExecCommand(nv.ctx, "ceph", cmd...) if err != nil { return fmt.Errorf("failed to create export %q in NFS-cluster %q"+ "(%v): %s", nv, nfsCluster, err, stderr) @@ -161,28 +171,6 @@ func (nv *NFSVolume) CreateExport(backend *csi.Volume) error { return nil } -// retryIfInvalid executes the "ceph" command, and falls back to the next cmd -// in case the error is EINVAL. -func (nv *NFSVolume) retryIfInvalid(cmds [][]string) (string, error) { - var ( - stderr string - err error - ) - for _, cmd := range cmds { - _, stderr, err = util.ExecCommand(nv.ctx, "ceph", cmd...) - // in case of an invalid command, fallback to the next one - if strings.Contains(stderr, "Error EINVAL: invalid command") { - continue - } - - // If we get here, either no error, or an unexpected error - // happened. There is no need to retry an other command. - break - } - - return stderr, err -} - // createExportCommand returns the "ceph nfs export create ..." command // arguments (without "ceph"). The order of the parameters matches old Ceph // releases, new Ceph releases added --option formats, which can be added when @@ -214,20 +202,28 @@ func (nv *NFSVolume) DeleteExport() error { return fmt.Errorf("failed to identify NFS cluster: %w", err) } - // TODO: use new go-ceph API, see ceph/ceph-csi#2977 - // new versions of Ceph use a different command, and the go-ceph API - // also seems to be different :-/ - // - // run the new command, but fall back to the previous one in case of an - // error - cmds := [][]string{ - // ceph nfs export rm - nv.deleteExportCommand("rm", nfsCluster), - // ceph nfs export delete - nv.deleteExportCommand("delete", nfsCluster), + nfsa, err := nv.conn.GetNFSAdmin() + if err != nil { + return fmt.Errorf("failed to get NFSAdmin: %w", err) } - stderr, err := nv.retryIfInvalid(cmds) + err = nfsa.RemoveExport(nfsCluster, nv.GetExportPath()) + switch { + case err == nil: + return nil + case strings.Contains(err.Error(), "API call not implemented"): // try with the old command + break + default: // any other error + return fmt.Errorf("failed to remove %q from NFS-cluster %q: "+ + "%w", nv, nfsCluster, err) + } + + // if we get here, the API call failed, fallback to the old command + + // ceph nfs export delete + cmd := nv.deleteExportCommand("delete", nfsCluster) + + _, stderr, err := util.ExecCommand(nv.ctx, "ceph", cmd...) if err != nil { return fmt.Errorf("failed to delete export %q from NFS-cluster"+ "%q (%v): %s", nv, nfsCluster, err, stderr) diff --git a/internal/util/connection.go b/internal/util/connection.go index 492d27e12..48fba7ef2 100644 --- a/internal/util/connection.go +++ b/internal/util/connection.go @@ -22,6 +22,7 @@ import ( "time" ca "github.com/ceph/go-ceph/cephfs/admin" + "github.com/ceph/go-ceph/common/admin/nfs" "github.com/ceph/go-ceph/rados" ra "github.com/ceph/go-ceph/rbd/admin" ) @@ -140,3 +141,13 @@ func (cc *ClusterConnection) GetTaskAdmin() (*ra.TaskAdmin, error) { return rbdAdmin.Task(), nil } + +// GetNFSAdmin returns an Admin type that can be used to interact with the +// NFS-cluster that is managed by Ceph. +func (cc *ClusterConnection) GetNFSAdmin() (*nfs.Admin, error) { + if cc.conn == nil { + return nil, errors.New("cluster is not connected yet") + } + + return nfs.NewFromConn(cc.conn), nil +} From 282c33cb58fce8c1870e12bbda2268138a4fa70e Mon Sep 17 00:00:00 2001 From: Niels de Vos Date: Wed, 30 Mar 2022 10:46:01 +0200 Subject: [PATCH 11/11] rebase: use go-ceph version with NFS-Admin API The NFS-Admin API has been added to go-ceph v0.15.0. As the API can not be tested in the go-ceph CI, it requires build-tag `ceph_ci_untested`. This additional build-tag has been added to the `Makefile` and should be removed when the API does not require the build-tag anymore. See-also: ceph/go-ceph#655 Signed-off-by: Niels de Vos --- Makefile | 3 +- go.mod | 3 +- go.sum | 4 +- .../ceph/go-ceph/common/admin/nfs/admin.go | 21 ++ .../ceph/go-ceph/common/admin/nfs/doc.go | 5 + .../ceph/go-ceph/common/admin/nfs/export.go | 198 ++++++++++++++++++ .../go-ceph/internal/commands/response.go | 13 ++ .../go-ceph/internal/cutil/buffergroup.go | 89 ++++++++ .../ceph/go-ceph/internal/log/log.go | 14 ++ vendor/github.com/ceph/go-ceph/rados/omap.go | 132 +----------- .../ceph/go-ceph/rados/rados_set_locator.go | 31 +++ .../github.com/ceph/go-ceph/rados/read_op.go | 14 +- .../rados/read_op_omap_get_vals_by_keys.go | 35 ++-- .../github.com/ceph/go-ceph/rados/watcher.go | 20 +- .../github.com/ceph/go-ceph/rados/write_op.go | 40 ++-- vendor/modules.txt | 4 +- 16 files changed, 455 insertions(+), 171 deletions(-) create mode 100644 vendor/github.com/ceph/go-ceph/common/admin/nfs/admin.go create mode 100644 vendor/github.com/ceph/go-ceph/common/admin/nfs/doc.go create mode 100644 vendor/github.com/ceph/go-ceph/common/admin/nfs/export.go create mode 100644 vendor/github.com/ceph/go-ceph/internal/cutil/buffergroup.go create mode 100644 vendor/github.com/ceph/go-ceph/internal/log/log.go create mode 100644 vendor/github.com/ceph/go-ceph/rados/rados_set_locator.go diff --git a/Makefile b/Makefile index 3fa146818..125fc711d 100644 --- a/Makefile +++ b/Makefile @@ -48,7 +48,8 @@ GO_PROJECT=github.com/ceph/ceph-csi CEPH_VERSION ?= $(shell . $(CURDIR)/build.env ; echo $${CEPH_VERSION}) # TODO: ceph_preview tag may be removed with go-ceph 0.16.0 -GO_TAGS_LIST ?= $(CEPH_VERSION) ceph_preview +# TODO: ceph_ci_untested is added for NFS-export management (go-ceph#655) +GO_TAGS_LIST ?= $(CEPH_VERSION) ceph_preview ceph_ci_untested # go build flags LDFLAGS ?= diff --git a/go.mod b/go.mod index a9e7dcc50..acfcae112 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,8 @@ require ( github.com/aws/aws-sdk-go v1.43.32 github.com/aws/aws-sdk-go-v2/service/sts v1.16.3 github.com/ceph/ceph-csi/api v0.0.0-00010101000000-000000000000 - github.com/ceph/go-ceph v0.14.0 + // TODO: API for managing NFS-exports requires `ceph_ci_untested` build-tag + github.com/ceph/go-ceph v0.15.0 github.com/container-storage-interface/spec v1.5.0 github.com/csi-addons/replication-lib-utils v0.2.0 github.com/csi-addons/spec v0.1.2-0.20211220115741-32fa508dadbe diff --git a/go.sum b/go.sum index 09ee4de5e..aa41791d7 100644 --- a/go.sum +++ b/go.sum @@ -180,8 +180,8 @@ github.com/cenkalti/backoff/v3 v3.0.0 h1:ske+9nBpD9qZsTBoF41nW5L+AIuFBKMeze18XQ3 github.com/cenkalti/backoff/v3 v3.0.0/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/centrify/cloud-golang-sdk v0.0.0-20190214225812-119110094d0f/go.mod h1:C0rtzmGXgN78pYR0tGJFhtHgkbAs0lIbHwkB81VxDQE= -github.com/ceph/go-ceph v0.14.0 h1:sJoT0au7NT3TPmDWf5W9w6tZy0U/5xZrIXVVauZR+Xo= -github.com/ceph/go-ceph v0.14.0/go.mod h1:mafFpf5Vg8Ai8Bd+FAMvKBHLmtdpTXdRP/TNq8XWegY= +github.com/ceph/go-ceph v0.15.0 h1:ILB3NaLWOtt4u/2d8I8HZTC4Ycm1PsOYVar3IFU1xlo= +github.com/ceph/go-ceph v0.15.0/go.mod h1:mafFpf5Vg8Ai8Bd+FAMvKBHLmtdpTXdRP/TNq8XWegY= github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= diff --git a/vendor/github.com/ceph/go-ceph/common/admin/nfs/admin.go b/vendor/github.com/ceph/go-ceph/common/admin/nfs/admin.go new file mode 100644 index 000000000..2f644d0e4 --- /dev/null +++ b/vendor/github.com/ceph/go-ceph/common/admin/nfs/admin.go @@ -0,0 +1,21 @@ +//go:build !(nautilus || octopus) && ceph_preview && ceph_ci_untested +// +build !nautilus,!octopus,ceph_preview,ceph_ci_untested + +package nfs + +import ( + ccom "github.com/ceph/go-ceph/common/commands" +) + +// Admin is used to administer ceph nfs features. +type Admin struct { + conn ccom.RadosCommander +} + +// NewFromConn creates an new management object from a preexisting +// rados connection. The existing connection can be rados.Conn or any +// type implementing the RadosCommander interface. +// PREVIEW +func NewFromConn(conn ccom.RadosCommander) *Admin { + return &Admin{conn} +} diff --git a/vendor/github.com/ceph/go-ceph/common/admin/nfs/doc.go b/vendor/github.com/ceph/go-ceph/common/admin/nfs/doc.go new file mode 100644 index 000000000..05fb3a48d --- /dev/null +++ b/vendor/github.com/ceph/go-ceph/common/admin/nfs/doc.go @@ -0,0 +1,5 @@ +/* +Package nfs from common/admin contains a set of APIs used to interact +with and administer NFS support for ceph clusters. +*/ +package nfs diff --git a/vendor/github.com/ceph/go-ceph/common/admin/nfs/export.go b/vendor/github.com/ceph/go-ceph/common/admin/nfs/export.go new file mode 100644 index 000000000..230946410 --- /dev/null +++ b/vendor/github.com/ceph/go-ceph/common/admin/nfs/export.go @@ -0,0 +1,198 @@ +//go:build !(nautilus || octopus) && ceph_preview && ceph_ci_untested +// +build !nautilus,!octopus,ceph_preview,ceph_ci_untested + +package nfs + +import ( + "github.com/ceph/go-ceph/internal/commands" +) + +// SquashMode indicates the kind of user-id squashing performed on an export. +type SquashMode string + +// src: https://github.com/nfs-ganesha/nfs-ganesha/blob/next/src/config_samples/export.txt +const ( + // NoneSquash performs no id squashing. + NoneSquash SquashMode = "None" + // RootSquash performs squashing of root user (with any gid). + RootSquash SquashMode = "Root" + // AllSquash performs squashing of all users. + AllSquash SquashMode = "All" + // RootIDSquash performs squashing of root uid/gid. + RootIDSquash SquashMode = "RootId" + // NoRootSquash is equivalent to NoneSquash + NoRootSquash = NoneSquash + // Unspecifiedquash + Unspecifiedquash SquashMode = "" +) + +// CephFSExportSpec is used to specify the parameters used to create a new +// CephFS based export. +type CephFSExportSpec struct { + FileSystemName string `json:"fsname"` + ClusterID string `json:"cluster_id"` + PseudoPath string `json:"pseudo_path"` + Path string `json:"path,omitempty"` + ReadOnly bool `json:"readonly"` + ClientAddr []string `json:"client_addr,omitempty"` + Squash SquashMode `json:"squash,omitempty"` +} + +// ExportResult is returned along with newly created exports. +type ExportResult struct { + Bind string `json:"bind"` + FileSystemName string `json:"fs"` + Path string `json:"path"` + ClusterID string `json:"cluster"` + Mode string `json:"mode"` +} + +type cephFSExportFields struct { + Prefix string `json:"prefix"` + Format string `json:"format"` + + CephFSExportSpec +} + +// FSALInfo describes NFS-Ganesha specific FSAL properties of an export. +type FSALInfo struct { + Name string `json:"name"` + UserID string `json:"user_id"` + FileSystemName string `json:"fs_name"` +} + +// ClientInfo describes per-client parameters of an export. +type ClientInfo struct { + Addresses []string `json:"addresses"` + AccessType string `json:"access_type"` + Squash SquashMode `json:"squash"` +} + +// ExportInfo describes an NFS export. +type ExportInfo struct { + ExportID int64 `json:"export_id"` + Path string `json:"path"` + ClusterID string `json:"cluster_id"` + PseudoPath string `json:"pseudo"` + AccessType string `json:"access_type"` + Squash SquashMode `json:"squash"` + SecurityLabel bool `json:"security_label"` + Protocols []int `json:"protocols"` + Transports []string `json:"transports"` + FSAL FSALInfo `json:"fsal"` + Clients []ClientInfo `json:"clients"` +} + +func parseExportResult(res commands.Response) (*ExportResult, error) { + r := &ExportResult{} + if err := res.NoStatus().Unmarshal(r).End(); err != nil { + return nil, err + } + return r, nil +} + +func parseExportsList(res commands.Response) ([]ExportInfo, error) { + l := []ExportInfo{} + if err := res.NoStatus().Unmarshal(&l).End(); err != nil { + return nil, err + } + return l, nil +} + +func parseExportInfo(res commands.Response) (ExportInfo, error) { + i := ExportInfo{} + if err := res.NoStatus().Unmarshal(&i).End(); err != nil { + return i, err + } + return i, nil +} + +// CreateCephFSExport will create a new NFS export for a CephFS file system. +// PREVIEW +// +// Similar To: +// ceph nfs export create cephfs +func (nfsa *Admin) CreateCephFSExport(spec CephFSExportSpec) ( + *ExportResult, error) { + // --- + f := &cephFSExportFields{ + Prefix: "nfs export create cephfs", + Format: "json", + CephFSExportSpec: spec, + } + return parseExportResult(commands.MarshalMgrCommand(nfsa.conn, f)) +} + +const delSucc = "Successfully deleted export" + +// RemoveExport will remove an NFS export based on the pseudo-path of the export. +// PREVIEW +// +// Similar To: +// ceph nfs export rm +func (nfsa *Admin) RemoveExport(clusterID, pseudoPath string) error { + m := map[string]string{ + "prefix": "nfs export rm", + "format": "json", + "cluster_id": clusterID, + "pseudo_path": pseudoPath, + } + return (commands.MarshalMgrCommand(nfsa.conn, m). + FilterBodyPrefix(delSucc).NoData().End()) +} + +// ListDetailedExports will return a list of exports with details. +// PREVIEW +// +// Similar To: +// ceph nfs export ls --detailed +func (nfsa *Admin) ListDetailedExports(clusterID string) ([]ExportInfo, error) { + /* + NOTE: there is no simple list because based on a quick reading of the code + in ceph, the details fetching should not be significantly slower with + details than without, and since this is an API call not a CLI its easy + enough to ignore the details you don't care about. If I'm wrong, and + we discover a major perf. difference in the future we can always add a new + simpler list-without-details function. + */ + m := map[string]string{ + "prefix": "nfs export ls", + "detailed": "true", + "format": "json", + "cluster_id": clusterID, + } + return parseExportsList(commands.MarshalMgrCommand(nfsa.conn, m)) +} + +// ExportInfo will return a structure describing the export specified by it's +// pseudo-path. +// PREVIEW +// +// Similar To: +// ceph nfs export info +func (nfsa *Admin) ExportInfo(clusterID, pseudoPath string) (ExportInfo, error) { + m := map[string]string{ + "prefix": "nfs export info", + "format": "json", + "cluster_id": clusterID, + "pseudo_path": pseudoPath, + } + return parseExportInfo(commands.MarshalMgrCommand(nfsa.conn, m)) +} + +/* +TODO? + +'nfs export apply': cluster_id: str, inbuf: str +"""Create or update an export by `-i `""" + + +'nfs export create rgw': + bucket: str, + cluster_id: str, + pseudo_path: str, + readonly: Optional[bool] = False, + client_addr: Optional[List[str]] = None, + squash: str = 'none', +"""Create an RGW export""" +*/ diff --git a/vendor/github.com/ceph/go-ceph/internal/commands/response.go b/vendor/github.com/ceph/go-ceph/internal/commands/response.go index cafd4ba73..f5d3d3cf8 100644 --- a/vendor/github.com/ceph/go-ceph/internal/commands/response.go +++ b/vendor/github.com/ceph/go-ceph/internal/commands/response.go @@ -1,6 +1,7 @@ package commands import ( + "bytes" "encoding/json" "errors" "fmt" @@ -156,6 +157,18 @@ func (r Response) FilterSuffix(s string) Response { return r } +// FilterBodyPrefix sets the body value equivalent to an empty string if the +// body value contains the given prefix string. +func (r Response) FilterBodyPrefix(p string) Response { + if !r.Ok() { + return r + } + if bytes.HasPrefix(r.body, []byte(p)) { + return Response{[]byte(""), r.status, r.err} + } + return r +} + // FilterDeprecated removes deprecation warnings from the response status. // Use it when checking the response from calls that may be deprecated in ceph // if you want those calls to continue working if the warning is present. diff --git a/vendor/github.com/ceph/go-ceph/internal/cutil/buffergroup.go b/vendor/github.com/ceph/go-ceph/internal/cutil/buffergroup.go new file mode 100644 index 000000000..447ec11cc --- /dev/null +++ b/vendor/github.com/ceph/go-ceph/internal/cutil/buffergroup.go @@ -0,0 +1,89 @@ +package cutil + +// #include +import "C" + +import ( + "unsafe" +) + +// BufferGroup is a helper structure that holds Go-allocated slices of +// C-allocated strings and their respective lengths. Useful for C functions +// that consume byte buffers with explicit length instead of null-terminated +// strings. When used as input arguments in C functions, caller must make sure +// the C code will not hold any pointers to either of the struct's attributes +// after that C function returns. +type BufferGroup struct { + // C-allocated buffers. + Buffers []CharPtr + // Lengths of C buffers, where Lengths[i] = length(Buffers[i]). + Lengths []SizeT +} + +// TODO: should BufferGroup implementation change and the slices would contain +// nested Go pointers, they must be pinned with PtrGuard. + +// NewBufferGroupStrings returns new BufferGroup constructed from strings. +func NewBufferGroupStrings(strs []string) *BufferGroup { + s := &BufferGroup{ + Buffers: make([]CharPtr, len(strs)), + Lengths: make([]SizeT, len(strs)), + } + + for i, str := range strs { + bs := []byte(str) + s.Buffers[i] = CharPtr(C.CBytes(bs)) + s.Lengths[i] = SizeT(len(bs)) + } + + return s +} + +// NewBufferGroupBytes returns new BufferGroup constructed +// from slice of byte slices. +func NewBufferGroupBytes(bss [][]byte) *BufferGroup { + s := &BufferGroup{ + Buffers: make([]CharPtr, len(bss)), + Lengths: make([]SizeT, len(bss)), + } + + for i, bs := range bss { + s.Buffers[i] = CharPtr(C.CBytes(bs)) + s.Lengths[i] = SizeT(len(bs)) + } + + return s +} + +// Free free()s the C-allocated memory. +func (s *BufferGroup) Free() { + for _, ptr := range s.Buffers { + C.free(unsafe.Pointer(ptr)) + } + + s.Buffers = nil + s.Lengths = nil +} + +// BuffersPtr returns a pointer to the beginning of the Buffers slice. +func (s *BufferGroup) BuffersPtr() CharPtrPtr { + if len(s.Buffers) == 0 { + return nil + } + + return CharPtrPtr(&s.Buffers[0]) +} + +// LengthsPtr returns a pointer to the beginning of the Lengths slice. +func (s *BufferGroup) LengthsPtr() SizeTPtr { + if len(s.Lengths) == 0 { + return nil + } + + return SizeTPtr(&s.Lengths[0]) +} + +func testBufferGroupGet(s *BufferGroup, index int) (str string, length int) { + bs := C.GoBytes(unsafe.Pointer(s.Buffers[index]), C.int(s.Lengths[index])) + return string(bs), int(s.Lengths[index]) +} diff --git a/vendor/github.com/ceph/go-ceph/internal/log/log.go b/vendor/github.com/ceph/go-ceph/internal/log/log.go new file mode 100644 index 000000000..90fc30627 --- /dev/null +++ b/vendor/github.com/ceph/go-ceph/internal/log/log.go @@ -0,0 +1,14 @@ +// Package log is the internal package for go-ceph logging. This package is only +// used from go-ceph code, not from consumers of go-ceph. go-ceph code uses the +// functions in this package to log information that can't be returned as +// errors. The functions default to no-ops and can be set with the external log +// package common/log by the go-ceph consumers. +package log + +func noop(string, ...interface{}) {} + +// These variables are set by the common log package. +var ( + Warnf = noop + Debugf = noop +) diff --git a/vendor/github.com/ceph/go-ceph/rados/omap.go b/vendor/github.com/ceph/go-ceph/rados/omap.go index c2756ce32..525826ba1 100644 --- a/vendor/github.com/ceph/go-ceph/rados/omap.go +++ b/vendor/github.com/ceph/go-ceph/rados/omap.go @@ -10,69 +10,8 @@ import "C" import ( "runtime" "unsafe" - - "github.com/ceph/go-ceph/internal/cutil" ) -// setOmapStep is a write op step. It holds C memory used in the operation. -type setOmapStep struct { - withRefs - withoutUpdate - - // C arguments - cKeys cutil.CPtrCSlice - cValues cutil.CPtrCSlice - cLengths cutil.SizeTCSlice - cNum C.size_t -} - -func newSetOmapStep(pairs map[string][]byte) *setOmapStep { - - maplen := len(pairs) - cKeys := cutil.NewCPtrCSlice(maplen) - cValues := cutil.NewCPtrCSlice(maplen) - cLengths := cutil.NewSizeTCSlice(maplen) - - sos := &setOmapStep{ - cKeys: cKeys, - cValues: cValues, - cLengths: cLengths, - cNum: C.size_t(maplen), - } - - var i uintptr - for key, value := range pairs { - // key - ck := C.CString(key) - sos.add(unsafe.Pointer(ck)) - cKeys[i] = cutil.CPtr(ck) - - // value and its length - vlen := cutil.SizeT(len(value)) - if vlen > 0 { - cv := C.CBytes(value) - sos.add(cv) - cValues[i] = cutil.CPtr(cv) - } else { - cValues[i] = nil - } - - cLengths[i] = vlen - - i++ - } - - runtime.SetFinalizer(sos, opStepFinalizer) - return sos -} - -func (sos *setOmapStep) free() { - sos.cKeys.Free() - sos.cValues.Free() - sos.cLengths.Free() - sos.withRefs.free() -} - // OmapKeyValue items are returned by the GetOmapStep's Next call. type OmapKeyValue struct { Key string @@ -88,15 +27,6 @@ type OmapKeyValue struct { // Release method is called the public methods of the step must no longer be // used and may return errors. type GetOmapStep struct { - // inputs: - startAfter string - filterPrefix string - maxReturn uint64 - - // arguments: - cStartAfter *C.char - cFilterPrefix *C.char - // C returned data: iter C.rados_omap_iter_t more *C.uchar @@ -109,15 +39,10 @@ type GetOmapStep struct { canIterate bool } -func newGetOmapStep(startAfter, filterPrefix string, maxReturn uint64) *GetOmapStep { +func newGetOmapStep() *GetOmapStep { gos := &GetOmapStep{ - startAfter: startAfter, - filterPrefix: filterPrefix, - maxReturn: maxReturn, - cStartAfter: C.CString(startAfter), - cFilterPrefix: C.CString(filterPrefix), - more: (*C.uchar)(C.malloc(C.sizeof_uchar)), - rval: (*C.int)(C.malloc(C.sizeof_int)), + more: (*C.uchar)(C.malloc(C.sizeof_uchar)), + rval: (*C.int)(C.malloc(C.sizeof_int)), } runtime.SetFinalizer(gos, opStepFinalizer) return gos @@ -133,10 +58,6 @@ func (gos *GetOmapStep) free() { gos.more = nil C.free(unsafe.Pointer(gos.rval)) gos.rval = nil - C.free(unsafe.Pointer(gos.cStartAfter)) - gos.cStartAfter = nil - C.free(unsafe.Pointer(gos.cFilterPrefix)) - gos.cFilterPrefix = nil } func (gos *GetOmapStep) update() error { @@ -151,11 +72,12 @@ func (gos *GetOmapStep) Next() (*OmapKeyValue, error) { return nil, ErrOperationIncomplete } var ( - cKey *C.char - cVal *C.char - cLen C.size_t + cKey *C.char + cVal *C.char + cKeyLen C.size_t + cValLen C.size_t ) - ret := C.rados_omap_get_next(gos.iter, &cKey, &cVal, &cLen) + ret := C.rados_omap_get_next2(gos.iter, &cKey, &cVal, &cKeyLen, &cValLen) if ret != 0 { return nil, getError(ret) } @@ -163,8 +85,8 @@ func (gos *GetOmapStep) Next() (*OmapKeyValue, error) { return nil, nil } return &OmapKeyValue{ - Key: C.GoString(cKey), - Value: C.GoBytes(unsafe.Pointer(cVal), C.int(cLen)), + Key: string(C.GoBytes(unsafe.Pointer(cKey), C.int(cKeyLen))), + Value: C.GoBytes(unsafe.Pointer(cVal), C.int(cValLen)), }, nil } @@ -175,40 +97,6 @@ func (gos *GetOmapStep) More() bool { return *gos.more != 0 } -// removeOmapKeysStep is a write operation step used to track state, especially -// C memory, across the setup and use of a WriteOp. -type removeOmapKeysStep struct { - withRefs - withoutUpdate - - // arguments: - cKeys cutil.CPtrCSlice - cNum C.size_t -} - -func newRemoveOmapKeysStep(keys []string) *removeOmapKeysStep { - cKeys := cutil.NewCPtrCSlice(len(keys)) - roks := &removeOmapKeysStep{ - cKeys: cKeys, - cNum: C.size_t(len(keys)), - } - - i := 0 - for _, key := range keys { - cKeys[i] = cutil.CPtr(C.CString(key)) - roks.add(unsafe.Pointer(cKeys[i])) - i++ - } - - runtime.SetFinalizer(roks, opStepFinalizer) - return roks -} - -func (roks *removeOmapKeysStep) free() { - roks.cKeys.Free() - roks.withRefs.free() -} - // SetOmap appends the map `pairs` to the omap `oid` func (ioctx *IOContext) SetOmap(oid string, pairs map[string][]byte) error { op := CreateWriteOp() diff --git a/vendor/github.com/ceph/go-ceph/rados/rados_set_locator.go b/vendor/github.com/ceph/go-ceph/rados/rados_set_locator.go new file mode 100644 index 000000000..e80286794 --- /dev/null +++ b/vendor/github.com/ceph/go-ceph/rados/rados_set_locator.go @@ -0,0 +1,31 @@ +//go:build ceph_preview +// +build ceph_preview + +package rados + +// #cgo LDFLAGS: -lrados +// #include +// #include +// +import "C" + +import ( + "unsafe" +) + +// SetLocator sets the key for mapping objects to pgs within an io context. +// Until a different locator key is set, all objects in this io context will be placed in the same pg. +// To reset the locator, an empty string must be set. +// PREVIEW +// +// Implements: +// void rados_ioctx_locator_set_key(rados_ioctx_t io, const char *key); +func (ioctx *IOContext) SetLocator(locator string) { + if locator == "" { + C.rados_ioctx_locator_set_key(ioctx.ioctx, nil) + } else { + var cLoc *C.char = C.CString(locator) + defer C.free(unsafe.Pointer(cLoc)) + C.rados_ioctx_locator_set_key(ioctx.ioctx, cLoc) + } +} diff --git a/vendor/github.com/ceph/go-ceph/rados/read_op.go b/vendor/github.com/ceph/go-ceph/rados/read_op.go index 74f2eb2bd..8487664ae 100644 --- a/vendor/github.com/ceph/go-ceph/rados/read_op.go +++ b/vendor/github.com/ceph/go-ceph/rados/read_op.go @@ -69,13 +69,19 @@ func (r *ReadOp) AssertExists() { // function. The GetOmapStep may be used to iterate over the key-value // pairs after the Operate call has been performed. func (r *ReadOp) GetOmapValues(startAfter, filterPrefix string, maxReturn uint64) *GetOmapStep { - gos := newGetOmapStep(startAfter, filterPrefix, maxReturn) + gos := newGetOmapStep() r.steps = append(r.steps, gos) + + cStartAfter := C.CString(startAfter) + cFilterPrefix := C.CString(filterPrefix) + defer C.free(unsafe.Pointer(cStartAfter)) + defer C.free(unsafe.Pointer(cFilterPrefix)) + C.rados_read_op_omap_get_vals2( r.op, - gos.cStartAfter, - gos.cFilterPrefix, - C.uint64_t(gos.maxReturn), + cStartAfter, + cFilterPrefix, + C.uint64_t(maxReturn), &gos.iter, gos.more, gos.rval, diff --git a/vendor/github.com/ceph/go-ceph/rados/read_op_omap_get_vals_by_keys.go b/vendor/github.com/ceph/go-ceph/rados/read_op_omap_get_vals_by_keys.go index c3289073b..feed5d295 100644 --- a/vendor/github.com/ceph/go-ceph/rados/read_op_omap_get_vals_by_keys.go +++ b/vendor/github.com/ceph/go-ceph/rados/read_op_omap_get_vals_by_keys.go @@ -11,6 +11,8 @@ import "C" import ( "unsafe" + + "github.com/ceph/go-ceph/internal/cutil" ) // ReadOpOmapGetValsByKeysStep holds the result of the @@ -65,10 +67,11 @@ func (s *ReadOpOmapGetValsByKeysStep) Next() (*OmapKeyValue, error) { var ( cKey *C.char cVal *C.char + cKeyLen C.size_t cValLen C.size_t ) - ret := C.rados_omap_get_next(s.iter, &cKey, &cVal, &cValLen) + ret := C.rados_omap_get_next2(s.iter, &cKey, &cVal, &cKeyLen, &cValLen) if ret != 0 { return nil, getError(ret) } @@ -79,7 +82,7 @@ func (s *ReadOpOmapGetValsByKeysStep) Next() (*OmapKeyValue, error) { } return &OmapKeyValue{ - Key: C.GoString(cKey), + Key: string(C.GoBytes(unsafe.Pointer(cKey), C.int(cKeyLen))), Value: C.GoBytes(unsafe.Pointer(cVal), C.int(cValLen)), }, nil } @@ -88,30 +91,24 @@ func (s *ReadOpOmapGetValsByKeysStep) Next() (*OmapKeyValue, error) { // PREVIEW // // Implements: -// void rados_read_op_omap_get_vals_by_keys(rados_read_op_t read_op, -// char const * const * keys, -// size_t keys_len, -// rados_omap_iter_t * iter, -// int * prval) +// void rados_read_op_omap_get_vals_by_keys2(rados_read_op_t read_op, +// char const * const * keys, +// size_t num_keys, +// const size_t * key_lens, +// rados_omap_iter_t * iter, +// int * prval) func (r *ReadOp) GetOmapValuesByKeys(keys []string) *ReadOpOmapGetValsByKeysStep { s := newReadOpOmapGetValsByKeysStep() r.steps = append(r.steps, s) - cKeys := make([]*C.char, len(keys)) - defer func() { - for _, cKeyPtr := range cKeys { - C.free(unsafe.Pointer(cKeyPtr)) - } - }() + cKeys := cutil.NewBufferGroupStrings(keys) + defer cKeys.Free() - for i, key := range keys { - cKeys[i] = C.CString(key) - } - - C.rados_read_op_omap_get_vals_by_keys( + C.rados_read_op_omap_get_vals_by_keys2( r.op, - &cKeys[0], + (**C.char)(cKeys.BuffersPtr()), C.size_t(len(keys)), + (*C.size_t)(cKeys.LengthsPtr()), &s.iter, s.prval, ) diff --git a/vendor/github.com/ceph/go-ceph/rados/watcher.go b/vendor/github.com/ceph/go-ceph/rados/watcher.go index 4569c6849..3df1cd044 100644 --- a/vendor/github.com/ceph/go-ceph/rados/watcher.go +++ b/vendor/github.com/ceph/go-ceph/rados/watcher.go @@ -19,6 +19,8 @@ import ( "sync" "time" "unsafe" + + "github.com/ceph/go-ceph/internal/log" ) type ( @@ -340,14 +342,6 @@ func decodeNotifyResponse(response *C.char, len C.size_t) ([]NotifyAck, []Notify //export watchNotifyCb func watchNotifyCb(_ unsafe.Pointer, notifyID C.uint64_t, id C.uint64_t, notifierID C.uint64_t, cData unsafe.Pointer, dataLen C.size_t) { - watchersMtx.RLock() - w, ok := watchers[WatcherID(id)] - watchersMtx.RUnlock() - if !ok { - // usually this should not happen, but who knows - // TODO: some log message (once we have logging) - return - } ev := NotifyEvent{ ID: NotifyID(notifyID), WatcherID: WatcherID(id), @@ -356,6 +350,14 @@ func watchNotifyCb(_ unsafe.Pointer, notifyID C.uint64_t, id C.uint64_t, if dataLen > 0 { ev.Data = C.GoBytes(cData, C.int(dataLen)) } + watchersMtx.RLock() + w, ok := watchers[WatcherID(id)] + watchersMtx.RUnlock() + if !ok { + // usually this should not happen, but who knows + log.Warnf("received notification for unknown watcher ID: %#v", ev) + return + } select { case <-w.done: // unblock when deleted case w.events <- ev: @@ -369,7 +371,7 @@ func watchErrorCb(_ unsafe.Pointer, id C.uint64_t, err C.int) { watchersMtx.RUnlock() if !ok { // usually this should not happen, but who knows - // TODO: some log message (once we have logging) + log.Warnf("received error for unknown watcher ID: id=%d err=%#v", id, err) return } select { diff --git a/vendor/github.com/ceph/go-ceph/rados/write_op.go b/vendor/github.com/ceph/go-ceph/rados/write_op.go index cdb838d0c..fa25f3da8 100644 --- a/vendor/github.com/ceph/go-ceph/rados/write_op.go +++ b/vendor/github.com/ceph/go-ceph/rados/write_op.go @@ -10,6 +10,7 @@ import "C" import ( "unsafe" + "github.com/ceph/go-ceph/internal/cutil" ts "github.com/ceph/go-ceph/internal/timespec" ) @@ -92,24 +93,39 @@ func (w *WriteOp) Create(exclusive CreateOption) { // SetOmap appends the map `pairs` to the omap `oid`. func (w *WriteOp) SetOmap(pairs map[string][]byte) { - sos := newSetOmapStep(pairs) - w.steps = append(w.steps, sos) - C.rados_write_op_omap_set( + keys := make([]string, len(pairs)) + values := make([][]byte, len(pairs)) + idx := 0 + for k, v := range pairs { + keys[idx] = k + values[idx] = v + idx++ + } + + cKeys := cutil.NewBufferGroupStrings(keys) + cValues := cutil.NewBufferGroupBytes(values) + defer cKeys.Free() + defer cValues.Free() + + C.rados_write_op_omap_set2( w.op, - (**C.char)(sos.cKeys.Ptr()), - (**C.char)(sos.cValues.Ptr()), - (*C.size_t)(sos.cLengths.Ptr()), - sos.cNum) + (**C.char)(cKeys.BuffersPtr()), + (**C.char)(cValues.BuffersPtr()), + (*C.size_t)(cKeys.LengthsPtr()), + (*C.size_t)(cValues.LengthsPtr()), + (C.size_t)(len(pairs))) } // RmOmapKeys removes the specified `keys` from the omap `oid`. func (w *WriteOp) RmOmapKeys(keys []string) { - roks := newRemoveOmapKeysStep(keys) - w.steps = append(w.steps, roks) - C.rados_write_op_omap_rm_keys( + cKeys := cutil.NewBufferGroupStrings(keys) + defer cKeys.Free() + + C.rados_write_op_omap_rm_keys2( w.op, - (**C.char)(roks.cKeys.Ptr()), - roks.cNum) + (**C.char)(cKeys.BuffersPtr()), + (*C.size_t)(cKeys.LengthsPtr()), + (C.size_t)(len(keys))) } // CleanOmap clears the omap `oid`. diff --git a/vendor/modules.txt b/vendor/modules.txt index f8028e68f..d2d7e8949 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -116,15 +116,17 @@ github.com/cenkalti/backoff/v3 github.com/ceph/ceph-csi/api/deploy/kubernetes/nfs github.com/ceph/ceph-csi/api/deploy/kubernetes/rbd github.com/ceph/ceph-csi/api/deploy/ocp -# github.com/ceph/go-ceph v0.14.0 +# github.com/ceph/go-ceph v0.15.0 ## explicit; go 1.12 github.com/ceph/go-ceph/cephfs/admin github.com/ceph/go-ceph/common/admin/manager +github.com/ceph/go-ceph/common/admin/nfs github.com/ceph/go-ceph/common/commands github.com/ceph/go-ceph/internal/callbacks github.com/ceph/go-ceph/internal/commands github.com/ceph/go-ceph/internal/cutil github.com/ceph/go-ceph/internal/errutil +github.com/ceph/go-ceph/internal/log github.com/ceph/go-ceph/internal/retry github.com/ceph/go-ceph/internal/timespec github.com/ceph/go-ceph/rados