rbd/go-ceph: add GetMetadata() and GetMetadata() functions

Signed-off-by: Niels de Vos <ndevos@redhat.com>

internal/rbd/controllerserver.go

@@ -172,7 +172,7 @@ func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
 	}
 	if found {
 		if rbdVol.Encrypted {
-			err = ensureEncryptionMetadataSet(ctx, cr, rbdVol)
+			err = rbdVol.ensureEncryptionMetadataSet(rbdImageRequiresEncryption)
 			if err != nil {
 				klog.Errorf(util.Log(ctx, err.Error()))
 				return nil, err
@@ -224,7 +224,7 @@ func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
 	}
 
 	if rbdVol.Encrypted {
-		err = ensureEncryptionMetadataSet(ctx, cr, rbdVol)
+		err = rbdVol.ensureEncryptionMetadataSet(rbdImageRequiresEncryption)
 		if err != nil {
 			klog.Errorf(util.Log(ctx, "failed to save encryption status, deleting image %s"),
 				rbdVol.RbdImageName)

internal/rbd/nodeserver.go

@@ -202,6 +202,13 @@ func (ns *NodeServer) stageTransaction(ctx context.Context, req *csi.NodeStageVo
 	}
 	defer cr.DeleteCredentials()
 
+	err = volOptions.Connect(cr)
+	if err != nil {
+		klog.Errorf(util.Log(ctx, "failed to connect to volume %v: %v"), volOptions.RbdImageName, err)
+		return transaction, err
+	}
+	defer volOptions.Destroy()
+
 	// Mapping RBD image
 	var devicePath string
 	devicePath, err = attachRBDImage(ctx, volOptions, cr)
@@ -213,7 +220,7 @@ func (ns *NodeServer) stageTransaction(ctx context.Context, req *csi.NodeStageVo
 		req.GetVolumeId(), volOptions.Pool, devicePath)
 
 	if volOptions.Encrypted {
-		devicePath, err = ns.processEncryptedDevice(ctx, volOptions, devicePath, cr)
+		devicePath, err = ns.processEncryptedDevice(ctx, volOptions, devicePath)
 		if err != nil {
 			return transaction, err
 		}
@@ -704,9 +711,9 @@ func (ns *NodeServer) NodeGetCapabilities(ctx context.Context, req *csi.NodeGetC
 	}, nil
 }
 
-func (ns *NodeServer) processEncryptedDevice(ctx context.Context, volOptions *rbdVolume, devicePath string, cr *util.Credentials) (string, error) {
+func (ns *NodeServer) processEncryptedDevice(ctx context.Context, volOptions *rbdVolume, devicePath string) (string, error) {
 	imageSpec := volOptions.Pool + "/" + volOptions.RbdImageName
-	encrypted, err := util.CheckRbdImageEncrypted(ctx, cr, volOptions.Monitors, imageSpec)
+	encrypted, err := volOptions.checkRbdImageEncrypted(ctx)
 	if err != nil {
 		klog.Errorf(util.Log(ctx, "failed to get encryption status for rbd image %s: %v"),
 			imageSpec, err)
@@ -724,15 +731,14 @@ func (ns *NodeServer) processEncryptedDevice(ctx context.Context, volOptions *rb
 
 		switch existingFormat {
 		case "":
-			err = encryptDevice(ctx, volOptions, cr, devicePath)
+			err = encryptDevice(ctx, volOptions, devicePath)
 			if err != nil {
 				return "", fmt.Errorf("failed to encrypt rbd image %s: %v", imageSpec, err)
 			}
 		case "crypt":
 			klog.Warningf(util.Log(ctx, "rbd image %s is encrypted, but encryption state was not updated"),
 				imageSpec)
-			err = util.SaveRbdImageEncryptionStatus(
+			err = volOptions.ensureEncryptionMetadataSet(rbdImageEncrypted)
-				ctx, cr, volOptions.Monitors, imageSpec, rbdImageEncrypted)
 			if err != nil {
 				return "", fmt.Errorf("failed to update encryption state for rbd image %s", imageSpec)
 			}
@@ -753,7 +759,7 @@ func (ns *NodeServer) processEncryptedDevice(ctx context.Context, volOptions *rb
 	return devicePath, nil
 }
 
-func encryptDevice(ctx context.Context, rbdVol *rbdVolume, cr *util.Credentials, devicePath string) error {
+func encryptDevice(ctx context.Context, rbdVol *rbdVolume, devicePath string) error {
 	passphrase, err := util.GetCryptoPassphrase(ctx, rbdVol.VolID, rbdVol.KMS)
 	if err != nil {
 		klog.Errorf(util.Log(ctx, "failed to get crypto passphrase for %s/%s: %v"),
@@ -767,12 +773,15 @@ func encryptDevice(ctx context.Context, rbdVol *rbdVolume, cr *util.Credentials,
 		return err
 	}
 
-	imageSpec := rbdVol.Pool + "/" + rbdVol.RbdImageName
+	err = rbdVol.ensureEncryptionMetadataSet(rbdImageEncrypted)
-	err = util.SaveRbdImageEncryptionStatus(ctx, cr, rbdVol.Monitors, imageSpec, rbdImageEncrypted)
+	if err != nil {
-
+		klog.Error(util.Log(ctx, err.Error()))
 		return err
 	}
 
+	return nil
+}
+
 func openEncryptedDevice(ctx context.Context, volOptions *rbdVolume, devicePath string) (string, error) {
 	passphrase, err := util.GetCryptoPassphrase(ctx, volOptions.VolID, volOptions.KMS)
 	if err != nil {

internal/rbd/rbd_util.go

@@ -60,6 +60,8 @@ const (
 	// Encryption statuses for RbdImage
 	rbdImageEncrypted          = "encrypted"
 	rbdImageRequiresEncryption = "requiresEncryption"
+	// image metadata key for encryption
+	encryptionMetaKey = ".rbd.csi.ceph.com/encrypted"
 )
 
 // rbdVolume represents a CSI volume and its RBD image specifics
@@ -979,21 +981,53 @@ func resizeRBDImage(rbdVol *rbdVolume, cr *util.Credentials) error {
 	return nil
 }
 
-func ensureEncryptionMetadataSet(ctx context.Context, cr *util.Credentials, rbdVol *rbdVolume) error {
+func (rv *rbdVolume) GetMetadata(key string) (string, error) {
-	var vi util.CSIIdentifier
+	ioctx, err := rv.conn.GetIoctx(rv.Pool)
-
-	err := vi.DecomposeCSIID(rbdVol.VolID)
 	if err != nil {
-		err = fmt.Errorf("error decoding volume ID (%s) (%s)", rbdVol.VolID, err)
+		return "", errors.Wrapf(err, "failed to get ioctx for %q", rv.RbdImageName)
-		return ErrInvalidVolID{err}
 	}
 
-	rbdImageName := volJournal.GetNameForUUID(rbdVol.NamePrefix, vi.ObjectUUID, false)
+	image, err := librbd.OpenImage(ioctx, rv.RbdImageName, librbd.NoSnapshot)
-	imageSpec := rbdVol.Pool + "/" + rbdImageName
-
-	err = util.SaveRbdImageEncryptionStatus(ctx, cr, rbdVol.Monitors, imageSpec, rbdImageRequiresEncryption)
 	if err != nil {
-		return fmt.Errorf("failed to save encryption status for %s: %v", imageSpec, err)
+		return "", errors.Wrapf(err, "could not open image %q", rv.RbdImageName)
+	}
+	defer image.Close()
+
+	return image.GetMetadata(key)
+}
+
+func (rv *rbdVolume) SetMetadata(key, value string) error {
+	ioctx, err := rv.conn.GetIoctx(rv.Pool)
+	if err != nil {
+		return errors.Wrapf(err, "failed to get ioctx for %q", rv.RbdImageName)
+	}
+
+	image, err := librbd.OpenImage(ioctx, rv.RbdImageName, librbd.NoSnapshot)
+	if err != nil {
+		return errors.Wrapf(err, "could not open image %q", rv.RbdImageName)
+	}
+	defer image.Close()
+
+	return image.SetMetadata(key, value)
+}
+
+// checkRbdImageEncrypted verifies if rbd image was encrypted when created
+func (rv *rbdVolume) checkRbdImageEncrypted(ctx context.Context) (string, error) {
+	value, err := rv.GetMetadata(encryptionMetaKey)
+	if err != nil {
+		klog.Errorf(util.Log(ctx, "checking image %s encrypted state metadata failed: %s"), rv.RbdImageName, err)
+		return "", err
+	}
+
+	encrypted := strings.TrimSpace(value)
+	klog.V(4).Infof(util.Log(ctx, "image %s encrypted state metadata reports %q"), rv.RbdImageName, encrypted)
+	return encrypted, nil
+}
+
+func (rv *rbdVolume) ensureEncryptionMetadataSet(status string) error {
+	err := rv.SetMetadata(encryptionMetaKey, status)
+	if err != nil {
+		return fmt.Errorf("failed to save encryption status for %s: %v", rv.RbdImageName, err)
 	}
 
 	return nil

internal/util/crypto.go

@@ -36,9 +36,6 @@ const (
 	mapperFilePrefix     = "luks-rbd-"
 	mapperFilePathPrefix = "/dev/mapper"
 
-	// image metadata key for encryption
-	encryptionMetaKey = ".rbd.csi.ceph.com/encrypted"
-
 	// Encryption passphrase location in K8s secrets
 	encryptionPassphraseKey = "encryptionPassphrase"
 	kmsTypeKey              = "encryptionKMSType"
@@ -248,27 +245,3 @@ func DeviceEncryptionStatus(ctx context.Context, devicePath string) (mappedDevic
 	// Identified as LUKS, but failed to identify a mapped device
 	return "", "", fmt.Errorf("mapped device not found in path %s", devicePath)
 }
-
-// CheckRbdImageEncrypted verifies if rbd image was encrypted when created
-func CheckRbdImageEncrypted(ctx context.Context, cr *Credentials, monitors, imageSpec string) (string, error) {
-	value, err := GetImageMeta(ctx, cr, monitors, imageSpec, encryptionMetaKey)
-	if err != nil {
-		klog.Errorf(Log(ctx, "checking image %s encrypted state metadata failed: %s"), imageSpec, err)
-		return "", err
-	}
-
-	encrypted := strings.TrimSpace(value)
-	klog.V(4).Infof(Log(ctx, "image %s encrypted state metadata reports %q"), imageSpec, encrypted)
-	return encrypted, nil
-}
-
-// SaveRbdImageEncryptionStatus sets image metadata for encryption status
-func SaveRbdImageEncryptionStatus(ctx context.Context, cr *Credentials, monitors, imageSpec, status string) error {
-	err := SetImageMeta(ctx, cr, monitors, imageSpec, encryptionMetaKey, status)
-	if err != nil {
-		err = fmt.Errorf("failed to save image metadata encryption status for %s: %v", imageSpec, err.Error())
-		klog.Errorf(Log(ctx, err.Error()))
-		return err
-	}
-	return nil
-}