nfs: use go-ceph API for creating/deleting exports

Recent versions of Ceph allow calling the NFS-export management
functions over the go-ceph API.

This seems incompatible with older versions that have been tested with
the `ceph nfs` commands that this commit replaces.

Signed-off-by: Niels de Vos <ndevos@redhat.com>
(cherry picked from commit 28369702d2)

internal/nfs/controller/volume.go

@@ -26,6 +26,7 @@ import (
 	fsutil "github.com/ceph/ceph-csi/internal/cephfs/util"
 	"github.com/ceph/ceph-csi/internal/util"
 
+	"github.com/ceph/go-ceph/common/admin/nfs"
 	"github.com/container-storage-interface/spec/lib/go/csi"
 )
 
@@ -135,24 +136,33 @@ func (nv *NFSVolume) CreateExport(backend *csi.Volume) error {
 		return fmt.Errorf("failed to set NFS-cluster: %w", err)
 	}
 
-	// TODO: use new go-ceph API, see ceph/ceph-csi#2977
+	nfsa, err := nv.conn.GetNFSAdmin()
-	// new versions of Ceph use a different command, and the go-ceph API
+	if err != nil {
-	// also seems to be different :-/
+		return fmt.Errorf("failed to get NFSAdmin: %w", err)
-	//
-	// run the new command, but fall back to the previous one in case of an
-	// error
-	cmds := [][]string{
-		// ceph nfs export create cephfs --cluster-id <cluster_id>
-		//     --pseudo-path <pseudo_path> --fsname <fsname>
-		//     [--readonly] [--path=/path/in/cephfs]
-		nv.createExportCommand("--cluster-id="+nfsCluster,
-			"--fsname="+fs, "--pseudo-path="+nv.GetExportPath(),
-			"--path="+path),
-		// ceph nfs export create cephfs ${FS} ${NFS} /${EXPORT} ${SUBVOL_PATH}
-		nv.createExportCommand(nfsCluster, fs, nv.GetExportPath(), path),
 	}
 
-	stderr, err := nv.retryIfInvalid(cmds)
+	_, err = nfsa.CreateCephFSExport(nfs.CephFSExportSpec{
+		FileSystemName: fs,
+		ClusterID:      nfsCluster,
+		PseudoPath:     nv.GetExportPath(),
+		Path:           path,
+	})
+	switch {
+	case err == nil:
+		return nil
+	case strings.Contains(err.Error(), "rados: ret=-2"): // try with the old command
+		break
+	default: // any other error
+		return fmt.Errorf("exporting %q on NFS-cluster %q failed: %w",
+			nv, nfsCluster, err)
+	}
+
+	// if we get here, the API call failed, fallback to the old command
+
+	// ceph nfs export create cephfs ${FS} ${NFS} /${EXPORT} ${SUBVOL_PATH}
+	cmd := nv.createExportCommand(nfsCluster, fs, nv.GetExportPath(), path)
+
+	_, stderr, err := util.ExecCommand(nv.ctx, "ceph", cmd...)
 	if err != nil {
 		return fmt.Errorf("failed to create export %q in NFS-cluster %q"+
 			"(%v): %s", nv, nfsCluster, err, stderr)
@@ -161,28 +171,6 @@ func (nv *NFSVolume) CreateExport(backend *csi.Volume) error {
 	return nil
 }
 
-// retryIfInvalid executes the "ceph" command, and falls back to the next cmd
-// in case the error is EINVAL.
-func (nv *NFSVolume) retryIfInvalid(cmds [][]string) (string, error) {
-	var (
-		stderr string
-		err    error
-	)
-	for _, cmd := range cmds {
-		_, stderr, err = util.ExecCommand(nv.ctx, "ceph", cmd...)
-		// in case of an invalid command, fallback to the next one
-		if strings.Contains(stderr, "Error EINVAL: invalid command") {
-			continue
-		}
-
-		// If we get here, either no error, or an unexpected error
-		// happened. There is no need to retry an other command.
-		break
-	}
-
-	return stderr, err
-}
-
 // createExportCommand returns the "ceph nfs export create ..." command
 // arguments (without "ceph"). The order of the parameters matches old Ceph
 // releases, new Ceph releases added --option formats, which can be added  when
@@ -214,20 +202,28 @@ func (nv *NFSVolume) DeleteExport() error {
 		return fmt.Errorf("failed to identify NFS cluster: %w", err)
 	}
 
-	// TODO: use new go-ceph API, see ceph/ceph-csi#2977
+	nfsa, err := nv.conn.GetNFSAdmin()
-	// new versions of Ceph use a different command, and the go-ceph API
+	if err != nil {
-	// also seems to be different :-/
+		return fmt.Errorf("failed to get NFSAdmin: %w", err)
-	//
-	// run the new command, but fall back to the previous one in case of an
-	// error
-	cmds := [][]string{
-		// ceph nfs export rm <cluster_id> <pseudo_path>
-		nv.deleteExportCommand("rm", nfsCluster),
-		// ceph nfs export delete <cluster_id> <pseudo_path>
-		nv.deleteExportCommand("delete", nfsCluster),
 	}
 
-	stderr, err := nv.retryIfInvalid(cmds)
+	err = nfsa.RemoveExport(nfsCluster, nv.GetExportPath())
+	switch {
+	case err == nil:
+		return nil
+	case strings.Contains(err.Error(), "API call not implemented"): // try with the old command
+		break
+	default: // any other error
+		return fmt.Errorf("failed to remove %q from NFS-cluster %q: "+
+			"%w", nv, nfsCluster, err)
+	}
+
+	// if we get here, the API call failed, fallback to the old command
+
+	// ceph nfs export delete <cluster_id> <pseudo_path>
+	cmd := nv.deleteExportCommand("delete", nfsCluster)
+
+	_, stderr, err := util.ExecCommand(nv.ctx, "ceph", cmd...)
 	if err != nil {
 		return fmt.Errorf("failed to delete export %q from NFS-cluster"+
 			"%q (%v): %s", nv, nfsCluster, err, stderr)

internal/util/connection.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	ca "github.com/ceph/go-ceph/cephfs/admin"
+	"github.com/ceph/go-ceph/common/admin/nfs"
 	"github.com/ceph/go-ceph/rados"
 	ra "github.com/ceph/go-ceph/rbd/admin"
 )
@@ -140,3 +141,13 @@ func (cc *ClusterConnection) GetTaskAdmin() (*ra.TaskAdmin, error) {
 
 	return rbdAdmin.Task(), nil
 }
+
+// GetNFSAdmin returns an Admin type that can be used to interact with the
+// NFS-cluster that is managed by Ceph.
+func (cc *ClusterConnection) GetNFSAdmin() (*nfs.Admin, error) {
+	if cc.conn == nil {
+		return nil, errors.New("cluster is not connected yet")
+	}
+
+	return nfs.NewFromConn(cc.conn), nil
+}