From 630798d95e5fb738027ba96744cf1403015b7116 Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Thu, 26 Aug 2021 10:07:58 +0530 Subject: [PATCH 01/17] ci: remove stale bot configuration This commit removes the stale bot configration as stale bot repo is not actively maintained anymore. Signed-off-by: Madhu Rajanna --- .github/stale.yml | 45 --------------------------------------------- 1 file changed, 45 deletions(-) delete mode 100644 .github/stale.yml diff --git a/.github/stale.yml b/.github/stale.yml deleted file mode 100644 index d39de7d0f..000000000 --- a/.github/stale.yml +++ /dev/null @@ -1,45 +0,0 @@ -# Configuration for probot-stale - https://github.com/probot/stale - -# Issues or Pull Requests with these labels will never be -# considered stale. Set to `[]` to disable ---- -exemptLabels: - - keepalive - - security - - reliability - - release requirement - -# Set to true to ignore issues in a project (defaults to false) -exemptProjects: false - -# Set to true to ignore issues in a milestone (defaults to false) -exemptMilestones: true - -# Label to use when marking as stale -staleLabel: wontfix - -# Limit the number of actions per hour, from 1-30. Default is 30 -limitPerRun: 5 - -# Specify configuration settings that are specific to issues -issues: - daysUntilStale: 90 - daysUntilClose: 7 - markComment: > - This issue has been automatically marked as stale because it has not had - recent activity. It will be closed in a week if no further activity occurs. - Thank you for your contributions. - closeComment: > - This issue has been automatically closed due to inactivity. Please re-open - if this still requires investigation. -# Specify configuration settings that are specific to PRs -pulls: - daysUntilStale: 60 - daysUntilClose: 30 - markComment: > - This pull request has been automatically marked as stale because it has not - had recent activity. It will be closed in a month if no further activity - occurs. Thank you for your contributions. - closeComment: > - This pull request has been automatically closed due to inactivity. Please - re-open if these changes are still required. From 2036b587d72df7d8d207aaf60bcf3f2edba13068 Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Thu, 26 Aug 2021 10:10:26 +0530 Subject: [PATCH 02/17] ci: add github workflow for stale added github action to check for the stale issues and PRs. the action will get scheduled everydata at 21:00 UTC. Signed-off-by: Madhu Rajanna --- .github/workflows/stale.yaml | 40 ++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 .github/workflows/stale.yaml diff --git a/.github/workflows/stale.yaml b/.github/workflows/stale.yaml new file mode 100644 index 000000000..bcad108d3 --- /dev/null +++ b/.github/workflows/stale.yaml @@ -0,0 +1,40 @@ +--- +# Reference https://github.com/actions/stale +name: "Mark or close stale issues and PRs" +# yamllint disable-line rule:truthy +on: + schedule: + # Run the stalebot every day at 9pm UTC + - cron: "00 21 * * *" +# yamllint disable rule:line-length +jobs: + stale: + runs-on: ubuntu-18.04 + if: github.repository == 'ceph/ceph-csi' + steps: + - uses: actions/stale@v3 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + days-before-issue-stale: 30 + days-before-issue-close: 7 + stale-issue-message: > + This issue has been automatically marked as stale because it has not had recent activity. + It will be closed in a week if no further activity occurs. + Thank you for your contributions. + close-issue-message: > + This issue has been automatically closed due to inactivity. + Please re-open if this still requires investigation. + stale-issue-label: "wontfix" + exempt-issue-labels: "keepalive,security,reliability,release requirement" + + days-before-pr-stale: 30 + days-before-pr-close: 14 + stale-pr-message: > + This pull request has been automatically marked as stale because it has not had + recent activity. It will be closed in two weeks if no further activity occurs. + Thank you for your contributions. + close-pr-message: > + This pull request has been automatically closed due to inactivity. + Please re-open if these changes are still required. + stale-pr-label: "stale" + exempt-pr-labels: "keepalive,security,reliability,release requirement" From 6d00b39886fd979f2a5814462bc367dfc8f77444 Mon Sep 17 00:00:00 2001 From: Niels de Vos Date: Tue, 24 Aug 2021 17:03:25 +0200 Subject: [PATCH 03/17] cleanup: move log functions to new internal/util/log package Moving the log functions into its own internal/util/log package makes it possible to split out the humongous internal/util packages in further smaller pieces. This reduces the inter-dependencies between utility functions and components, preventing circular dependencies which are not allowed in Go. Updates: #852 Signed-off-by: Niels de Vos --- cmd/cephcsi.go | 11 +- internal/cephfs/cephfs_util.go | 15 +- internal/cephfs/clone.go | 40 +++--- internal/cephfs/controllerserver.go | 101 ++++++------- internal/cephfs/driver.go | 15 +- internal/cephfs/fsjournal.go | 15 +- internal/cephfs/nodeserver.go | 41 +++--- internal/cephfs/snapshot.go | 26 ++-- internal/cephfs/util.go | 5 +- internal/cephfs/volume.go | 25 ++-- internal/cephfs/volumemounter.go | 17 +-- internal/controller/controller.go | 8 +- .../persistentvolume/persistentvolume.go | 11 +- .../csi-common/controllerserver-default.go | 4 +- internal/csi-common/driver.go | 8 +- internal/csi-common/identityserver-default.go | 6 +- internal/csi-common/nodeserver-default.go | 6 +- internal/csi-common/server.go | 6 +- internal/csi-common/utils.go | 27 ++-- internal/journal/omap.go | 15 +- internal/journal/voljournal.go | 13 +- internal/liveness/liveness.go | 15 +- internal/rbd/clone.go | 14 +- internal/rbd/controllerserver.go | 133 +++++++++--------- internal/rbd/driver.go | 19 +-- internal/rbd/encryption.go | 27 ++-- internal/rbd/nodeserver.go | 85 +++++------ internal/rbd/rbd_attach.go | 27 ++-- internal/rbd/rbd_healer.go | 17 +-- internal/rbd/rbd_journal.go | 33 ++--- internal/rbd/rbd_util.go | 67 ++++----- internal/rbd/replicationcontrollerserver.go | 43 +++--- internal/rbd/snapshot.go | 19 +-- internal/util/cephcmds.go | 10 +- internal/util/crypto.go | 16 ++- internal/util/httpserver.go | 6 +- internal/util/idlocker.go | 4 +- internal/util/k8s.go | 8 +- internal/util/{ => log}/log.go | 2 +- internal/util/topology.go | 6 +- internal/util/util.go | 6 +- 41 files changed, 505 insertions(+), 467 deletions(-) rename internal/util/{ => log}/log.go (99%) diff --git a/cmd/cephcsi.go b/cmd/cephcsi.go index b97869824..1cf351598 100644 --- a/cmd/cephcsi.go +++ b/cmd/cephcsi.go @@ -29,6 +29,7 @@ import ( "github.com/ceph/ceph-csi/internal/liveness" "github.com/ceph/ceph-csi/internal/rbd" "github.com/ceph/ceph-csi/internal/util" + "github.com/ceph/ceph-csi/internal/util/log" "k8s.io/klog/v2" ) @@ -164,7 +165,7 @@ func main() { } os.Exit(0) } - util.DefaultLog("Driver version: %s and Git version: %s", util.DriverVersion, util.GitCommit) + log.DefaultLog("Driver version: %s and Git version: %s", util.DriverVersion, util.GitCommit) if conf.Vtype == "" { logAndExit("driver type not specified") @@ -182,15 +183,15 @@ func main() { if pidErr != nil { klog.Errorf("Failed to get the PID limit, can not reconfigure: %v", pidErr) } else { - util.DefaultLog("Initial PID limit is set to %d", currentLimit) + log.DefaultLog("Initial PID limit is set to %d", currentLimit) err = util.SetPIDLimit(conf.PidLimit) switch { case err != nil: klog.Errorf("Failed to set new PID limit to %d: %v", conf.PidLimit, err) case conf.PidLimit == -1: - util.DefaultLog("Reconfigured PID limit to %d (max)", conf.PidLimit) + log.DefaultLog("Reconfigured PID limit to %d (max)", conf.PidLimit) default: - util.DefaultLog("Reconfigured PID limit to %d", conf.PidLimit) + log.DefaultLog("Reconfigured PID limit to %d", conf.PidLimit) } } } @@ -209,7 +210,7 @@ func main() { } } - util.DefaultLog("Starting driver type: %v with name: %v", conf.Vtype, dname) + log.DefaultLog("Starting driver type: %v with name: %v", conf.Vtype, dname) switch conf.Vtype { case rbdType: validateCloneDepthFlag(&conf) diff --git a/internal/cephfs/cephfs_util.go b/internal/cephfs/cephfs_util.go index 4613fc2dc..23aba68c7 100644 --- a/internal/cephfs/cephfs_util.go +++ b/internal/cephfs/cephfs_util.go @@ -21,19 +21,20 @@ import ( "fmt" "github.com/ceph/ceph-csi/internal/util" + "github.com/ceph/ceph-csi/internal/util/log" ) func (vo *volumeOptions) getFscID(ctx context.Context) (int64, error) { fsa, err := vo.conn.GetFSAdmin() if err != nil { - util.ErrorLog(ctx, "could not get FSAdmin, can not fetch filesystem ID for %s:", vo.FsName, err) + log.ErrorLog(ctx, "could not get FSAdmin, can not fetch filesystem ID for %s:", vo.FsName, err) return 0, err } volumes, err := fsa.EnumerateVolumes() if err != nil { - util.ErrorLog(ctx, "could not list volumes, can not fetch filesystem ID for %s:", vo.FsName, err) + log.ErrorLog(ctx, "could not list volumes, can not fetch filesystem ID for %s:", vo.FsName, err) return 0, err } @@ -44,7 +45,7 @@ func (vo *volumeOptions) getFscID(ctx context.Context) (int64, error) { } } - util.ErrorLog(ctx, "failed to list volume %s", vo.FsName) + log.ErrorLog(ctx, "failed to list volume %s", vo.FsName) return 0, ErrVolumeNotFound } @@ -52,14 +53,14 @@ func (vo *volumeOptions) getFscID(ctx context.Context) (int64, error) { func (vo *volumeOptions) getMetadataPool(ctx context.Context) (string, error) { fsa, err := vo.conn.GetFSAdmin() if err != nil { - util.ErrorLog(ctx, "could not get FSAdmin, can not fetch metadata pool for %s:", vo.FsName, err) + log.ErrorLog(ctx, "could not get FSAdmin, can not fetch metadata pool for %s:", vo.FsName, err) return "", err } fsPoolInfos, err := fsa.ListFileSystems() if err != nil { - util.ErrorLog(ctx, "could not list filesystems, can not fetch metadata pool for %s:", vo.FsName, err) + log.ErrorLog(ctx, "could not list filesystems, can not fetch metadata pool for %s:", vo.FsName, err) return "", err } @@ -76,14 +77,14 @@ func (vo *volumeOptions) getMetadataPool(ctx context.Context) (string, error) { func (vo *volumeOptions) getFsName(ctx context.Context) (string, error) { fsa, err := vo.conn.GetFSAdmin() if err != nil { - util.ErrorLog(ctx, "could not get FSAdmin, can not fetch filesystem name for ID %d:", vo.FscID, err) + log.ErrorLog(ctx, "could not get FSAdmin, can not fetch filesystem name for ID %d:", vo.FscID, err) return "", err } volumes, err := fsa.EnumerateVolumes() if err != nil { - util.ErrorLog(ctx, "could not list volumes, can not fetch filesystem name for ID %d:", vo.FscID, err) + log.ErrorLog(ctx, "could not list volumes, can not fetch filesystem name for ID %d:", vo.FscID, err) return "", err } diff --git a/internal/cephfs/clone.go b/internal/cephfs/clone.go index aa03c1f99..304c5cdf7 100644 --- a/internal/cephfs/clone.go +++ b/internal/cephfs/clone.go @@ -20,7 +20,7 @@ import ( "context" "errors" - "github.com/ceph/ceph-csi/internal/util" + "github.com/ceph/ceph-csi/internal/util/log" ) // cephFSCloneState describes the status of the clone. @@ -64,7 +64,7 @@ func createCloneFromSubvolume(ctx context.Context, volID, cloneID volumeID, volO snapshotID := cloneID err := parentvolOpt.createSnapshot(ctx, snapshotID, volID) if err != nil { - util.ErrorLog(ctx, "failed to create snapshot %s %v", snapshotID, err) + log.ErrorLog(ctx, "failed to create snapshot %s %v", snapshotID, err) return err } @@ -78,57 +78,57 @@ func createCloneFromSubvolume(ctx context.Context, volID, cloneID volumeID, volO if protectErr != nil { err = parentvolOpt.deleteSnapshot(ctx, snapshotID, volID) if err != nil { - util.ErrorLog(ctx, "failed to delete snapshot %s %v", snapshotID, err) + log.ErrorLog(ctx, "failed to delete snapshot %s %v", snapshotID, err) } } if cloneErr != nil { if err = volOpt.purgeVolume(ctx, cloneID, true); err != nil { - util.ErrorLog(ctx, "failed to delete volume %s: %v", cloneID, err) + log.ErrorLog(ctx, "failed to delete volume %s: %v", cloneID, err) } if err = parentvolOpt.unprotectSnapshot(ctx, snapshotID, volID); err != nil { // In case the snap is already unprotected we get ErrSnapProtectionExist error code // in that case we are safe and we could discard this error and we are good to go // ahead with deletion if !errors.Is(err, ErrSnapProtectionExist) { - util.ErrorLog(ctx, "failed to unprotect snapshot %s %v", snapshotID, err) + log.ErrorLog(ctx, "failed to unprotect snapshot %s %v", snapshotID, err) } } if err = parentvolOpt.deleteSnapshot(ctx, snapshotID, volID); err != nil { - util.ErrorLog(ctx, "failed to delete snapshot %s %v", snapshotID, err) + log.ErrorLog(ctx, "failed to delete snapshot %s %v", snapshotID, err) } } }() protectErr = parentvolOpt.protectSnapshot(ctx, snapshotID, volID) if protectErr != nil { - util.ErrorLog(ctx, "failed to protect snapshot %s %v", snapshotID, protectErr) + log.ErrorLog(ctx, "failed to protect snapshot %s %v", snapshotID, protectErr) return protectErr } cloneErr = parentvolOpt.cloneSnapshot(ctx, volID, snapshotID, cloneID, volOpt) if cloneErr != nil { - util.ErrorLog(ctx, "failed to clone snapshot %s %s to %s %v", volID, snapshotID, cloneID, cloneErr) + log.ErrorLog(ctx, "failed to clone snapshot %s %s to %s %v", volID, snapshotID, cloneID, cloneErr) return cloneErr } cloneState, cloneErr := volOpt.getCloneState(ctx, cloneID) if cloneErr != nil { - util.ErrorLog(ctx, "failed to get clone state: %v", cloneErr) + log.ErrorLog(ctx, "failed to get clone state: %v", cloneErr) return cloneErr } if cloneState != cephFSCloneComplete { - util.ErrorLog(ctx, "clone %s did not complete: %v", cloneID, cloneState.toError()) + log.ErrorLog(ctx, "clone %s did not complete: %v", cloneID, cloneState.toError()) return cloneState.toError() } // This is a work around to fix sizing issue for cloned images err = volOpt.resizeVolume(ctx, cloneID, volOpt.Size) if err != nil { - util.ErrorLog(ctx, "failed to expand volume %s: %v", cloneID, err) + log.ErrorLog(ctx, "failed to expand volume %s: %v", cloneID, err) return err } @@ -138,13 +138,13 @@ func createCloneFromSubvolume(ctx context.Context, volID, cloneID volumeID, volO // in that case we are safe and we could discard this error and we are good to go // ahead with deletion if !errors.Is(err, ErrSnapProtectionExist) { - util.ErrorLog(ctx, "failed to unprotect snapshot %s %v", snapshotID, err) + log.ErrorLog(ctx, "failed to unprotect snapshot %s %v", snapshotID, err) return err } } if err = parentvolOpt.deleteSnapshot(ctx, snapshotID, volID); err != nil { - util.ErrorLog(ctx, "failed to delete snapshot %s %v", snapshotID, err) + log.ErrorLog(ctx, "failed to delete snapshot %s %v", snapshotID, err) return err } @@ -171,14 +171,14 @@ func cleanupCloneFromSubvolumeSnapshot( if snapInfo.Protected == snapshotIsProtected { err = parentVolOpt.unprotectSnapshot(ctx, snapShotID, volID) if err != nil { - util.ErrorLog(ctx, "failed to unprotect snapshot %s %v", snapShotID, err) + log.ErrorLog(ctx, "failed to unprotect snapshot %s %v", snapShotID, err) return err } } err = parentVolOpt.deleteSnapshot(ctx, snapShotID, volID) if err != nil { - util.ErrorLog(ctx, "failed to delete snapshot %s %v", snapShotID, err) + log.ErrorLog(ctx, "failed to delete snapshot %s %v", snapShotID, err) return err } @@ -206,7 +206,7 @@ func createCloneFromSnapshot( if err != nil { if !isCloneRetryError(err) { if dErr := volOptions.purgeVolume(ctx, volumeID(vID.FsSubvolName), true); dErr != nil { - util.ErrorLog(ctx, "failed to delete volume %s: %v", vID.FsSubvolName, dErr) + log.ErrorLog(ctx, "failed to delete volume %s: %v", vID.FsSubvolName, dErr) } } } @@ -214,7 +214,7 @@ func createCloneFromSnapshot( cloneState, err := volOptions.getCloneState(ctx, volumeID(vID.FsSubvolName)) if err != nil { - util.ErrorLog(ctx, "failed to get clone state: %v", err) + log.ErrorLog(ctx, "failed to get clone state: %v", err) return err } @@ -227,7 +227,7 @@ func createCloneFromSnapshot( // in the new cloned volume too. Till then we are explicitly making the size set err = volOptions.resizeVolume(ctx, volumeID(vID.FsSubvolName), volOptions.Size) if err != nil { - util.ErrorLog(ctx, "failed to expand volume %s with error: %v", vID.FsSubvolName, err) + log.ErrorLog(ctx, "failed to expand volume %s with error: %v", vID.FsSubvolName, err) return err } @@ -238,7 +238,7 @@ func createCloneFromSnapshot( func (vo *volumeOptions) getCloneState(ctx context.Context, volID volumeID) (cephFSCloneState, error) { fsa, err := vo.conn.GetFSAdmin() if err != nil { - util.ErrorLog( + log.ErrorLog( ctx, "could not get FSAdmin, can get clone status for volume %s with ID %s: %v", vo.FsName, @@ -250,7 +250,7 @@ func (vo *volumeOptions) getCloneState(ctx context.Context, volID volumeID) (cep cs, err := fsa.CloneStatus(vo.FsName, vo.SubvolumeGroup, string(volID)) if err != nil { - util.ErrorLog(ctx, "could not get clone state for volume %s with ID %s: %v", vo.FsName, string(volID), err) + log.ErrorLog(ctx, "could not get clone state for volume %s with ID %s: %v", vo.FsName, string(volID), err) return cephFSCloneError, err } diff --git a/internal/cephfs/controllerserver.go b/internal/cephfs/controllerserver.go index 25c298bcf..2860b8c93 100644 --- a/internal/cephfs/controllerserver.go +++ b/internal/cephfs/controllerserver.go @@ -23,6 +23,7 @@ import ( csicommon "github.com/ceph/ceph-csi/internal/csi-common" "github.com/ceph/ceph-csi/internal/util" + "github.com/ceph/ceph-csi/internal/util/log" "github.com/container-storage-interface/spec/lib/go/csi" "github.com/golang/protobuf/ptypes/timestamp" @@ -59,7 +60,7 @@ func (cs *ControllerServer) createBackingVolume( var err error if sID != nil { if err = cs.OperationLocks.GetRestoreLock(sID.SnapshotID); err != nil { - util.ErrorLog(ctx, err.Error()) + log.ErrorLog(ctx, err.Error()) return status.Error(codes.Aborted, err.Error()) } @@ -67,7 +68,7 @@ func (cs *ControllerServer) createBackingVolume( err = createCloneFromSnapshot(ctx, parentVolOpt, volOptions, vID, sID) if err != nil { - util.ErrorLog(ctx, "failed to create clone from snapshot %s: %v", sID.FsSnapshotName, err) + log.ErrorLog(ctx, "failed to create clone from snapshot %s: %v", sID.FsSnapshotName, err) return err } @@ -76,7 +77,7 @@ func (cs *ControllerServer) createBackingVolume( } if parentVolOpt != nil { if err = cs.OperationLocks.GetCloneLock(pvID.VolumeID); err != nil { - util.ErrorLog(ctx, err.Error()) + log.ErrorLog(ctx, err.Error()) return status.Error(codes.Aborted, err.Error()) } @@ -88,7 +89,7 @@ func (cs *ControllerServer) createBackingVolume( volOptions, parentVolOpt) if err != nil { - util.ErrorLog(ctx, "failed to create clone from subvolume %s: %v", volumeID(pvID.FsSubvolName), err) + log.ErrorLog(ctx, "failed to create clone from subvolume %s: %v", volumeID(pvID.FsSubvolName), err) return err } @@ -97,7 +98,7 @@ func (cs *ControllerServer) createBackingVolume( } if err = createVolume(ctx, volOptions, volumeID(vID.FsSubvolName), volOptions.Size); err != nil { - util.ErrorLog(ctx, "failed to create volume %s: %v", volOptions.RequestName, err) + log.ErrorLog(ctx, "failed to create volume %s: %v", volOptions.RequestName, err) return status.Error(codes.Internal, err.Error()) } @@ -150,7 +151,7 @@ func (cs *ControllerServer) CreateVolume( ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) { if err := cs.validateCreateVolumeRequest(req); err != nil { - util.ErrorLog(ctx, "CreateVolumeRequest validation failed: %v", err) + log.ErrorLog(ctx, "CreateVolumeRequest validation failed: %v", err) return nil, err } @@ -161,7 +162,7 @@ func (cs *ControllerServer) CreateVolume( cr, err := util.NewAdminCredentials(secret) if err != nil { - util.ErrorLog(ctx, "failed to retrieve admin credentials: %v", err) + log.ErrorLog(ctx, "failed to retrieve admin credentials: %v", err) return nil, status.Error(codes.InvalidArgument, err.Error()) } @@ -169,7 +170,7 @@ func (cs *ControllerServer) CreateVolume( // Existence and conflict checks if acquired := cs.VolumeLocks.TryAcquire(requestName); !acquired { - util.ErrorLog(ctx, util.VolumeOperationAlreadyExistsFmt, requestName) + log.ErrorLog(ctx, util.VolumeOperationAlreadyExistsFmt, requestName) return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, requestName) } @@ -177,7 +178,7 @@ func (cs *ControllerServer) CreateVolume( volOptions, err := newVolumeOptions(ctx, requestName, req, cr) if err != nil { - util.ErrorLog(ctx, "validation and extraction of volume options failed: %v", err) + log.ErrorLog(ctx, "validation and extraction of volume options failed: %v", err) return nil, status.Error(codes.InvalidArgument, err.Error()) } @@ -216,7 +217,7 @@ func (cs *ControllerServer) CreateVolume( if err != nil { purgeErr := volOptions.purgeVolume(ctx, volumeID(vID.FsSubvolName), false) if purgeErr != nil { - util.ErrorLog(ctx, "failed to delete volume %s: %v", requestName, purgeErr) + log.ErrorLog(ctx, "failed to delete volume %s: %v", requestName, purgeErr) // All errors other than ErrVolumeNotFound should return an error back to the caller if !errors.Is(purgeErr, ErrVolumeNotFound) { return nil, status.Error(codes.Internal, purgeErr.Error()) @@ -224,10 +225,10 @@ func (cs *ControllerServer) CreateVolume( } errUndo := undoVolReservation(ctx, volOptions, *vID, secret) if errUndo != nil { - util.WarningLog(ctx, "failed undoing reservation of volume: %s (%s)", + log.WarningLog(ctx, "failed undoing reservation of volume: %s (%s)", requestName, errUndo) } - util.ErrorLog(ctx, "failed to expand volume %s: %v", volumeID(vID.FsSubvolName), err) + log.ErrorLog(ctx, "failed to expand volume %s: %v", volumeID(vID.FsSubvolName), err) return nil, status.Error(codes.Internal, err.Error()) } @@ -264,7 +265,7 @@ func (cs *ControllerServer) CreateVolume( if !isCloneRetryError(err) { errDefer := undoVolReservation(ctx, volOptions, *vID, secret) if errDefer != nil { - util.WarningLog(ctx, "failed undoing reservation of volume: %s (%s)", + log.WarningLog(ctx, "failed undoing reservation of volume: %s (%s)", requestName, errDefer) } } @@ -285,7 +286,7 @@ func (cs *ControllerServer) CreateVolume( if err != nil { purgeErr := volOptions.purgeVolume(ctx, volumeID(vID.FsSubvolName), true) if purgeErr != nil { - util.ErrorLog(ctx, "failed to delete volume %s: %v", vID.FsSubvolName, purgeErr) + log.ErrorLog(ctx, "failed to delete volume %s: %v", vID.FsSubvolName, purgeErr) // All errors other than ErrVolumeNotFound should return an error back to the caller if !errors.Is(purgeErr, ErrVolumeNotFound) { // If the subvolume deletion is failed, we should not cleanup @@ -297,12 +298,12 @@ func (cs *ControllerServer) CreateVolume( return nil, status.Error(codes.Internal, purgeErr.Error()) } } - util.ErrorLog(ctx, "failed to get subvolume path %s: %v", vID.FsSubvolName, err) + log.ErrorLog(ctx, "failed to get subvolume path %s: %v", vID.FsSubvolName, err) return nil, status.Error(codes.Internal, err.Error()) } - util.DebugLog(ctx, "cephfs: successfully created backing volume named %s for request name %s", + log.DebugLog(ctx, "cephfs: successfully created backing volume named %s for request name %s", vID.FsSubvolName, requestName) volumeContext := req.GetParameters() volumeContext["subvolumeName"] = vID.FsSubvolName @@ -330,7 +331,7 @@ func (cs *ControllerServer) DeleteVolume( ctx context.Context, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) { if err := cs.validateDeleteVolumeRequest(); err != nil { - util.ErrorLog(ctx, "DeleteVolumeRequest validation failed: %v", err) + log.ErrorLog(ctx, "DeleteVolumeRequest validation failed: %v", err) return nil, err } @@ -340,7 +341,7 @@ func (cs *ControllerServer) DeleteVolume( // lock out parallel delete operations if acquired := cs.VolumeLocks.TryAcquire(string(volID)); !acquired { - util.ErrorLog(ctx, util.VolumeOperationAlreadyExistsFmt, volID) + log.ErrorLog(ctx, util.VolumeOperationAlreadyExistsFmt, volID) return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, string(volID)) } @@ -348,7 +349,7 @@ func (cs *ControllerServer) DeleteVolume( // lock out volumeID for clone and expand operation if err := cs.OperationLocks.GetDeleteLock(req.GetVolumeId()); err != nil { - util.ErrorLog(ctx, err.Error()) + log.ErrorLog(ctx, err.Error()) return nil, status.Error(codes.Aborted, err.Error()) } @@ -360,7 +361,7 @@ func (cs *ControllerServer) DeleteVolume( // if error is ErrPoolNotFound, the pool is already deleted we dont // need to worry about deleting subvolume or omap data, return success if errors.Is(err, util.ErrPoolNotFound) { - util.WarningLog(ctx, "failed to get backend volume for %s: %v", string(volID), err) + log.WarningLog(ctx, "failed to get backend volume for %s: %v", string(volID), err) return &csi.DeleteVolumeResponse{}, nil } @@ -371,7 +372,7 @@ func (cs *ControllerServer) DeleteVolume( return &csi.DeleteVolumeResponse{}, nil } - util.ErrorLog(ctx, "Error returned from newVolumeOptionsFromVolID: %v", err) + log.ErrorLog(ctx, "Error returned from newVolumeOptionsFromVolID: %v", err) // All errors other than ErrVolumeNotFound should return an error back to the caller if !errors.Is(err, ErrVolumeNotFound) { @@ -404,14 +405,14 @@ func (cs *ControllerServer) DeleteVolume( // Deleting a volume requires admin credentials cr, err := util.NewAdminCredentials(secrets) if err != nil { - util.ErrorLog(ctx, "failed to retrieve admin credentials: %v", err) + log.ErrorLog(ctx, "failed to retrieve admin credentials: %v", err) return nil, status.Error(codes.InvalidArgument, err.Error()) } defer cr.DeleteCredentials() if err = volOptions.purgeVolume(ctx, volumeID(vID.FsSubvolName), false); err != nil { - util.ErrorLog(ctx, "failed to delete volume %s: %v", volID, err) + log.ErrorLog(ctx, "failed to delete volume %s: %v", volID, err) if errors.Is(err, ErrVolumeHasSnapshots) { return nil, status.Error(codes.FailedPrecondition, err.Error()) } @@ -425,7 +426,7 @@ func (cs *ControllerServer) DeleteVolume( return nil, status.Error(codes.Internal, err.Error()) } - util.DebugLog(ctx, "cephfs: successfully deleted volume %s", volID) + log.DebugLog(ctx, "cephfs: successfully deleted volume %s", volID) return &csi.DeleteVolumeResponse{}, nil } @@ -454,7 +455,7 @@ func (cs *ControllerServer) ControllerExpandVolume( ctx context.Context, req *csi.ControllerExpandVolumeRequest) (*csi.ControllerExpandVolumeResponse, error) { if err := cs.validateExpandVolumeRequest(req); err != nil { - util.ErrorLog(ctx, "ControllerExpandVolumeRequest validation failed: %v", err) + log.ErrorLog(ctx, "ControllerExpandVolumeRequest validation failed: %v", err) return nil, err } @@ -464,7 +465,7 @@ func (cs *ControllerServer) ControllerExpandVolume( // lock out parallel delete operations if acquired := cs.VolumeLocks.TryAcquire(volID); !acquired { - util.ErrorLog(ctx, util.VolumeOperationAlreadyExistsFmt, volID) + log.ErrorLog(ctx, util.VolumeOperationAlreadyExistsFmt, volID) return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, volID) } @@ -472,7 +473,7 @@ func (cs *ControllerServer) ControllerExpandVolume( // lock out volumeID for clone and delete operation if err := cs.OperationLocks.GetExpandLock(volID); err != nil { - util.ErrorLog(ctx, err.Error()) + log.ErrorLog(ctx, err.Error()) return nil, status.Error(codes.Aborted, err.Error()) } @@ -486,7 +487,7 @@ func (cs *ControllerServer) ControllerExpandVolume( volOptions, volIdentifier, err := newVolumeOptionsFromVolID(ctx, volID, nil, secret) if err != nil { - util.ErrorLog(ctx, "validation and extraction of volume options failed: %v", err) + log.ErrorLog(ctx, "validation and extraction of volume options failed: %v", err) return nil, status.Error(codes.InvalidArgument, err.Error()) } @@ -495,7 +496,7 @@ func (cs *ControllerServer) ControllerExpandVolume( RoundOffSize := util.RoundOffBytes(req.GetCapacityRange().GetRequiredBytes()) if err = volOptions.resizeVolume(ctx, volumeID(volIdentifier.FsSubvolName), RoundOffSize); err != nil { - util.ErrorLog(ctx, "failed to expand volume %s: %v", volumeID(volIdentifier.FsSubvolName), err) + log.ErrorLog(ctx, "failed to expand volume %s: %v", volumeID(volIdentifier.FsSubvolName), err) return nil, status.Error(codes.Internal, err.Error()) } @@ -530,14 +531,14 @@ func (cs *ControllerServer) CreateSnapshot( sourceVolID := req.GetSourceVolumeId() // Existence and conflict checks if acquired := cs.SnapshotLocks.TryAcquire(requestName); !acquired { - util.ErrorLog(ctx, util.SnapshotOperationAlreadyExistsFmt, requestName) + log.ErrorLog(ctx, util.SnapshotOperationAlreadyExistsFmt, requestName) return nil, status.Errorf(codes.Aborted, util.SnapshotOperationAlreadyExistsFmt, requestName) } defer cs.SnapshotLocks.Release(requestName) if err = cs.OperationLocks.GetSnapshotCreateLock(sourceVolID); err != nil { - util.ErrorLog(ctx, err.Error()) + log.ErrorLog(ctx, err.Error()) return nil, status.Error(codes.Aborted, err.Error()) } @@ -548,7 +549,7 @@ func (cs *ControllerServer) CreateSnapshot( parentVolOptions, vid, err := newVolumeOptionsFromVolID(ctx, sourceVolID, nil, req.GetSecrets()) if err != nil { if errors.Is(err, util.ErrPoolNotFound) { - util.WarningLog(ctx, "failed to get backend volume for %s: %v", sourceVolID, err) + log.WarningLog(ctx, "failed to get backend volume for %s: %v", sourceVolID, err) return nil, status.Error(codes.NotFound, err.Error()) } @@ -576,7 +577,7 @@ func (cs *ControllerServer) CreateSnapshot( // lock out parallel snapshot create operations if acquired := cs.VolumeLocks.TryAcquire(sourceVolID); !acquired { - util.ErrorLog(ctx, util.VolumeOperationAlreadyExistsFmt, sourceVolID) + log.ErrorLog(ctx, util.VolumeOperationAlreadyExistsFmt, sourceVolID) return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, sourceVolID) } @@ -605,7 +606,7 @@ func (cs *ControllerServer) CreateSnapshot( if sid != nil { errDefer := undoSnapReservation(ctx, parentVolOptions, *sid, snapName, cr) if errDefer != nil { - util.WarningLog(ctx, "failed undoing reservation of snapshot: %s (%s)", + log.WarningLog(ctx, "failed undoing reservation of snapshot: %s (%s)", requestName, errDefer) } } @@ -620,7 +621,7 @@ func (cs *ControllerServer) CreateSnapshot( err = parentVolOptions.protectSnapshot(ctx, volumeID(sid.FsSnapshotName), volumeID(vid.FsSubvolName)) if err != nil { protected = false - util.WarningLog(ctx, "failed to protect snapshot of snapshot: %s (%s)", + log.WarningLog(ctx, "failed to protect snapshot of snapshot: %s (%s)", sid.FsSnapshotName, err) } } @@ -645,7 +646,7 @@ func (cs *ControllerServer) CreateSnapshot( if err != nil { errDefer := undoSnapReservation(ctx, parentVolOptions, *sID, snapName, cr) if errDefer != nil { - util.WarningLog(ctx, "failed undoing reservation of snapshot: %s (%s)", + log.WarningLog(ctx, "failed undoing reservation of snapshot: %s (%s)", requestName, errDefer) } } @@ -672,7 +673,7 @@ func doSnapshot(ctx context.Context, volOpt *volumeOptions, subvolumeName, snaps snap := snapshotInfo{} err := volOpt.createSnapshot(ctx, snapID, volID) if err != nil { - util.ErrorLog(ctx, "failed to create snapshot %s %v", snapID, err) + log.ErrorLog(ctx, "failed to create snapshot %s %v", snapID, err) return snap, err } @@ -680,13 +681,13 @@ func doSnapshot(ctx context.Context, volOpt *volumeOptions, subvolumeName, snaps if err != nil { dErr := volOpt.deleteSnapshot(ctx, snapID, volID) if dErr != nil { - util.ErrorLog(ctx, "failed to delete snapshot %s %v", snapID, err) + log.ErrorLog(ctx, "failed to delete snapshot %s %v", snapID, err) } } }() snap, err = volOpt.getSnapshotInfo(ctx, snapID, volID) if err != nil { - util.ErrorLog(ctx, "failed to get snapshot info %s %v", snapID, err) + log.ErrorLog(ctx, "failed to get snapshot info %s %v", snapID, err) return snap, fmt.Errorf("failed to get snapshot info for snapshot:%s", snapID) } @@ -698,7 +699,7 @@ func doSnapshot(ctx context.Context, volOpt *volumeOptions, subvolumeName, snaps snap.CreationTime = t err = volOpt.protectSnapshot(ctx, snapID, volID) if err != nil { - util.ErrorLog(ctx, "failed to protect snapshot %s %v", snapID, err) + log.ErrorLog(ctx, "failed to protect snapshot %s %v", snapID, err) } return snap, err @@ -707,7 +708,7 @@ func doSnapshot(ctx context.Context, volOpt *volumeOptions, subvolumeName, snaps func (cs *ControllerServer) validateSnapshotReq(ctx context.Context, req *csi.CreateSnapshotRequest) error { if err := cs.Driver.ValidateControllerServiceRequest( csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT); err != nil { - util.ErrorLog(ctx, "invalid create snapshot req: %v", protosanitizer.StripSecrets(req)) + log.ErrorLog(ctx, "invalid create snapshot req: %v", protosanitizer.StripSecrets(req)) return err } @@ -730,7 +731,7 @@ func (cs *ControllerServer) DeleteSnapshot( req *csi.DeleteSnapshotRequest) (*csi.DeleteSnapshotResponse, error) { if err := cs.Driver.ValidateControllerServiceRequest( csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT); err != nil { - util.ErrorLog(ctx, "invalid delete snapshot req: %v", protosanitizer.StripSecrets(req)) + log.ErrorLog(ctx, "invalid delete snapshot req: %v", protosanitizer.StripSecrets(req)) return nil, err } @@ -746,7 +747,7 @@ func (cs *ControllerServer) DeleteSnapshot( } if acquired := cs.SnapshotLocks.TryAcquire(snapshotID); !acquired { - util.ErrorLog(ctx, util.SnapshotOperationAlreadyExistsFmt, snapshotID) + log.ErrorLog(ctx, util.SnapshotOperationAlreadyExistsFmt, snapshotID) return nil, status.Errorf(codes.Aborted, util.SnapshotOperationAlreadyExistsFmt, snapshotID) } @@ -754,7 +755,7 @@ func (cs *ControllerServer) DeleteSnapshot( // lock out snapshotID for restore operation if err = cs.OperationLocks.GetDeleteLock(snapshotID); err != nil { - util.ErrorLog(ctx, err.Error()) + log.ErrorLog(ctx, err.Error()) return nil, status.Error(codes.Aborted, err.Error()) } @@ -766,7 +767,7 @@ func (cs *ControllerServer) DeleteSnapshot( case errors.Is(err, util.ErrPoolNotFound): // if error is ErrPoolNotFound, the pool is already deleted we dont // need to worry about deleting snapshot or omap data, return success - util.WarningLog(ctx, "failed to get backend snapshot for %s: %v", snapshotID, err) + log.WarningLog(ctx, "failed to get backend snapshot for %s: %v", snapshotID, err) return &csi.DeleteSnapshotResponse{}, nil case errors.Is(err, util.ErrKeyNotFound): @@ -777,7 +778,7 @@ func (cs *ControllerServer) DeleteSnapshot( case errors.Is(err, ErrSnapNotFound): err = undoSnapReservation(ctx, volOpt, *sid, sid.FsSnapshotName, cr) if err != nil { - util.ErrorLog(ctx, "failed to remove reservation for snapname (%s) with backing snap (%s) (%s)", + log.ErrorLog(ctx, "failed to remove reservation for snapname (%s) with backing snap (%s) (%s)", sid.FsSubvolName, sid.FsSnapshotName, err) return nil, status.Error(codes.Internal, err.Error()) @@ -787,10 +788,10 @@ func (cs *ControllerServer) DeleteSnapshot( case errors.Is(err, ErrVolumeNotFound): // if the error is ErrVolumeNotFound, the subvolume is already deleted // from backend, Hence undo the omap entries and return success - util.ErrorLog(ctx, "Volume not present") + log.ErrorLog(ctx, "Volume not present") err = undoSnapReservation(ctx, volOpt, *sid, sid.FsSnapshotName, cr) if err != nil { - util.ErrorLog(ctx, "failed to remove reservation for snapname (%s) with backing snap (%s) (%s)", + log.ErrorLog(ctx, "failed to remove reservation for snapname (%s) with backing snap (%s) (%s)", sid.FsSubvolName, sid.FsSnapshotName, err) return nil, status.Error(codes.Internal, err.Error()) @@ -806,7 +807,7 @@ func (cs *ControllerServer) DeleteSnapshot( // safeguard against parallel create or delete requests against the same // name if acquired := cs.SnapshotLocks.TryAcquire(sid.RequestName); !acquired { - util.ErrorLog(ctx, util.SnapshotOperationAlreadyExistsFmt, sid.RequestName) + log.ErrorLog(ctx, util.SnapshotOperationAlreadyExistsFmt, sid.RequestName) return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, sid.RequestName) } @@ -827,7 +828,7 @@ func (cs *ControllerServer) DeleteSnapshot( } err = undoSnapReservation(ctx, volOpt, *sid, sid.FsSnapshotName, cr) if err != nil { - util.ErrorLog(ctx, "failed to remove reservation for snapname (%s) with backing snap (%s) (%s)", + log.ErrorLog(ctx, "failed to remove reservation for snapname (%s) with backing snap (%s) (%s)", sid.RequestName, sid.FsSnapshotName, err) return nil, status.Error(codes.Internal, err.Error()) diff --git a/internal/cephfs/driver.go b/internal/cephfs/driver.go index c8699bbc6..9c0da4041 100644 --- a/internal/cephfs/driver.go +++ b/internal/cephfs/driver.go @@ -20,6 +20,7 @@ import ( csicommon "github.com/ceph/ceph-csi/internal/csi-common" "github.com/ceph/ceph-csi/internal/journal" "github.com/ceph/ceph-csi/internal/util" + "github.com/ceph/ceph-csi/internal/util/log" "github.com/container-storage-interface/spec/lib/go/csi" ) @@ -93,11 +94,11 @@ func (fs *Driver) Run(conf *util.Config) { // Configuration if err = loadAvailableMounters(conf); err != nil { - util.FatalLogMsg("cephfs: failed to load ceph mounters: %v", err) + log.FatalLogMsg("cephfs: failed to load ceph mounters: %v", err) } if err = util.WriteCephConfig(); err != nil { - util.FatalLogMsg("failed to write ceph configuration file: %v", err) + log.FatalLogMsg("failed to write ceph configuration file: %v", err) } // Use passed in instance ID, if provided for omap suffix naming @@ -112,7 +113,7 @@ func (fs *Driver) Run(conf *util.Config) { fs.cd = csicommon.NewCSIDriver(conf.DriverName, util.DriverVersion, conf.NodeID) if fs.cd == nil { - util.FatalLogMsg("failed to initialize CSI driver") + log.FatalLogMsg("failed to initialize CSI driver") } if conf.IsControllerServer || !conf.IsNodeServer { @@ -134,7 +135,7 @@ func (fs *Driver) Run(conf *util.Config) { if conf.IsNodeServer { topology, err = util.GetTopologyFromDomainLabels(conf.DomainLabels, conf.NodeID, conf.DriverName) if err != nil { - util.FatalLogMsg(err.Error()) + log.FatalLogMsg(err.Error()) } fs.ns = NewNodeServer(fs.cd, conf.Vtype, topology) } @@ -145,7 +146,7 @@ func (fs *Driver) Run(conf *util.Config) { if !conf.IsControllerServer && !conf.IsNodeServer { topology, err = util.GetTopologyFromDomainLabels(conf.DomainLabels, conf.NodeID, conf.DriverName) if err != nil { - util.FatalLogMsg(err.Error()) + log.FatalLogMsg(err.Error()) } fs.ns = NewNodeServer(fs.cd, conf.Vtype, topology) fs.cs = NewControllerServer(fs.cd) @@ -161,14 +162,14 @@ func (fs *Driver) Run(conf *util.Config) { } server.Start(conf.Endpoint, conf.HistogramOption, srv, conf.EnableGRPCMetrics) if conf.EnableGRPCMetrics { - util.WarningLogMsg("EnableGRPCMetrics is deprecated") + log.WarningLogMsg("EnableGRPCMetrics is deprecated") go util.StartMetricsServer(conf) } if conf.EnableProfiling { if !conf.EnableGRPCMetrics { go util.StartMetricsServer(conf) } - util.DebugLogMsg("Registering profiling handler") + log.DebugLogMsg("Registering profiling handler") go util.EnableProfiling() } server.Wait() diff --git a/internal/cephfs/fsjournal.go b/internal/cephfs/fsjournal.go index 31dbad94e..a30969571 100644 --- a/internal/cephfs/fsjournal.go +++ b/internal/cephfs/fsjournal.go @@ -22,6 +22,7 @@ import ( "fmt" "github.com/ceph/ceph-csi/internal/util" + "github.com/ceph/ceph-csi/internal/util/log" "github.com/golang/protobuf/ptypes/timestamp" ) @@ -112,7 +113,7 @@ func checkVolExists(ctx context.Context, if cloneState == cephFSCloneFailed { err = volOptions.purgeVolume(ctx, volumeID(vid.FsSubvolName), true) if err != nil { - util.ErrorLog(ctx, "failed to delete volume %s: %v", vid.FsSubvolName, err) + log.ErrorLog(ctx, "failed to delete volume %s: %v", vid.FsSubvolName, err) return nil, err } @@ -171,7 +172,7 @@ func checkVolExists(ctx context.Context, return nil, err } - util.DebugLog(ctx, "Found existing volume (%s) with subvolume name (%s) for request (%s)", + log.DebugLog(ctx, "Found existing volume (%s) with subvolume name (%s) for request (%s)", vid.VolumeID, vid.FsSubvolName, volOptions.RequestName) if parentVolOpt != nil && pvID != nil { @@ -269,7 +270,7 @@ func reserveVol(ctx context.Context, volOptions *volumeOptions, secret map[strin return nil, err } - util.DebugLog(ctx, "Generated Volume ID (%s) and subvolume name (%s) for request name (%s)", + log.DebugLog(ctx, "Generated Volume ID (%s) and subvolume name (%s) for request name (%s)", vid.VolumeID, vid.FsSubvolName, volOptions.RequestName) return &vid, nil @@ -311,7 +312,7 @@ func reserveSnap( return nil, err } - util.DebugLog(ctx, "Generated Snapshot ID (%s) for request name (%s)", + log.DebugLog(ctx, "Generated Snapshot ID (%s) for request name (%s)", vid.SnapshotID, snap.RequestName) return &vid, nil @@ -392,14 +393,14 @@ func checkSnapExists( if err != nil { err = volOptions.deleteSnapshot(ctx, volumeID(snapID), volumeID(parentSubVolName)) if err != nil { - util.ErrorLog(ctx, "failed to delete snapshot %s: %v", snapID, err) + log.ErrorLog(ctx, "failed to delete snapshot %s: %v", snapID, err) return } err = j.UndoReservation(ctx, volOptions.MetadataPool, volOptions.MetadataPool, snapID, snap.RequestName) if err != nil { - util.ErrorLog(ctx, "removing reservation failed for snapshot %s: %v", snapID, err) + log.ErrorLog(ctx, "removing reservation failed for snapshot %s: %v", snapID, err) } } }() @@ -415,7 +416,7 @@ func checkSnapExists( if err != nil { return nil, nil, err } - util.DebugLog(ctx, "Found existing snapshot (%s) with subvolume name (%s) for request (%s)", + log.DebugLog(ctx, "Found existing snapshot (%s) with subvolume name (%s) for request (%s)", snapData.ImageAttributes.RequestName, parentSubVolName, sid.FsSnapshotName) return sid, &snapInfo, nil diff --git a/internal/cephfs/nodeserver.go b/internal/cephfs/nodeserver.go index d321ed681..4d2498532 100644 --- a/internal/cephfs/nodeserver.go +++ b/internal/cephfs/nodeserver.go @@ -25,6 +25,7 @@ import ( csicommon "github.com/ceph/ceph-csi/internal/csi-common" "github.com/ceph/ceph-csi/internal/util" + "github.com/ceph/ceph-csi/internal/util/log" "github.com/container-storage-interface/spec/lib/go/csi" "google.golang.org/grpc/codes" @@ -81,7 +82,7 @@ func (ns *NodeServer) NodeStageVolume( volID := volumeID(req.GetVolumeId()) if acquired := ns.VolumeLocks.TryAcquire(req.GetVolumeId()); !acquired { - util.ErrorLog(ctx, util.VolumeOperationAlreadyExistsFmt, volID) + log.ErrorLog(ctx, util.VolumeOperationAlreadyExistsFmt, volID) return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, req.GetVolumeId()) } @@ -114,13 +115,13 @@ func (ns *NodeServer) NodeStageVolume( isMnt, err := util.IsMountPoint(stagingTargetPath) if err != nil { - util.ErrorLog(ctx, "stat failed: %v", err) + log.ErrorLog(ctx, "stat failed: %v", err) return nil, status.Error(codes.Internal, err.Error()) } if isMnt { - util.DebugLog(ctx, "cephfs: volume %s is already mounted to %s, skipping", volID, stagingTargetPath) + log.DebugLog(ctx, "cephfs: volume %s is already mounted to %s, skipping", volID, stagingTargetPath) return &csi.NodeStageVolumeResponse{}, nil } @@ -130,7 +131,7 @@ func (ns *NodeServer) NodeStageVolume( return nil, err } - util.DebugLog(ctx, "cephfs: successfully mounted volume %s to %s", volID, stagingTargetPath) + log.DebugLog(ctx, "cephfs: successfully mounted volume %s to %s", volID, stagingTargetPath) return &csi.NodeStageVolumeResponse{}, nil } @@ -141,7 +142,7 @@ func (*NodeServer) mount(ctx context.Context, volOptions *volumeOptions, req *cs cr, err := getCredentialsForVolume(volOptions, req) if err != nil { - util.ErrorLog(ctx, "failed to get ceph credentials for volume %s: %v", volID, err) + log.ErrorLog(ctx, "failed to get ceph credentials for volume %s: %v", volID, err) return status.Error(codes.Internal, err.Error()) } @@ -149,12 +150,12 @@ func (*NodeServer) mount(ctx context.Context, volOptions *volumeOptions, req *cs m, err := newMounter(volOptions) if err != nil { - util.ErrorLog(ctx, "failed to create mounter for volume %s: %v", volID, err) + log.ErrorLog(ctx, "failed to create mounter for volume %s: %v", volID, err) return status.Error(codes.Internal, err.Error()) } - util.DebugLog(ctx, "cephfs: mounting volume %s with %s", volID, m.name()) + log.DebugLog(ctx, "cephfs: mounting volume %s with %s", volID, m.name()) readOnly := "ro" fuseMountOptions := strings.Split(volOptions.FuseMountOptions, ",") @@ -177,7 +178,7 @@ func (*NodeServer) mount(ctx context.Context, volOptions *volumeOptions, req *cs } if err = m.mount(ctx, stagingTargetPath, cr, volOptions); err != nil { - util.ErrorLog(ctx, + log.ErrorLog(ctx, "failed to mount volume %s: %v Check dmesg logs if required.", volID, err) @@ -189,7 +190,7 @@ func (*NodeServer) mount(ctx context.Context, volOptions *volumeOptions, req *cs // #nosec - allow anyone to write inside the stagingtarget path err = os.Chmod(stagingTargetPath, 0o777) if err != nil { - util.ErrorLog( + log.ErrorLog( ctx, "failed to change stagingtarget path %s permission for volume %s: %v", stagingTargetPath, @@ -197,7 +198,7 @@ func (*NodeServer) mount(ctx context.Context, volOptions *volumeOptions, req *cs err) uErr := unmountVolume(ctx, stagingTargetPath) if uErr != nil { - util.ErrorLog( + log.ErrorLog( ctx, "failed to umount stagingtarget path %s for volume %s: %v", stagingTargetPath, @@ -229,7 +230,7 @@ func (ns *NodeServer) NodePublishVolume( // are serialized, we dont need any extra locking in nodePublish if err := util.CreateMountPoint(targetPath); err != nil { - util.ErrorLog(ctx, "failed to create mount point at %s: %v", targetPath, err) + log.ErrorLog(ctx, "failed to create mount point at %s: %v", targetPath, err) return nil, status.Error(codes.Internal, err.Error()) } @@ -244,13 +245,13 @@ func (ns *NodeServer) NodePublishVolume( isMnt, err := util.IsMountPoint(targetPath) if err != nil { - util.ErrorLog(ctx, "stat failed: %v", err) + log.ErrorLog(ctx, "stat failed: %v", err) return nil, status.Error(codes.Internal, err.Error()) } if isMnt { - util.DebugLog(ctx, "cephfs: volume %s is already bind-mounted to %s", volID, targetPath) + log.DebugLog(ctx, "cephfs: volume %s is already bind-mounted to %s", volID, targetPath) return &csi.NodePublishVolumeResponse{}, nil } @@ -258,12 +259,12 @@ func (ns *NodeServer) NodePublishVolume( // It's not, mount now if err = bindMount(ctx, req.GetStagingTargetPath(), req.GetTargetPath(), req.GetReadonly(), mountOptions); err != nil { - util.ErrorLog(ctx, "failed to bind-mount volume %s: %v", volID, err) + log.ErrorLog(ctx, "failed to bind-mount volume %s: %v", volID, err) return nil, status.Error(codes.Internal, err.Error()) } - util.DebugLog(ctx, "cephfs: successfully bind-mounted volume %s to %s", volID, targetPath) + log.DebugLog(ctx, "cephfs: successfully bind-mounted volume %s to %s", volID, targetPath) return &csi.NodePublishVolumeResponse{}, nil } @@ -283,7 +284,7 @@ func (ns *NodeServer) NodeUnpublishVolume( if err != nil { if os.IsNotExist(err) { // targetPath has already been deleted - util.DebugLog(ctx, "targetPath: %s has already been deleted", targetPath) + log.DebugLog(ctx, "targetPath: %s has already been deleted", targetPath) return &csi.NodeUnpublishVolumeResponse{}, nil } @@ -308,7 +309,7 @@ func (ns *NodeServer) NodeUnpublishVolume( return nil, status.Error(codes.Internal, err.Error()) } - util.DebugLog(ctx, "cephfs: successfully unbounded volume %s from %s", req.GetVolumeId(), targetPath) + log.DebugLog(ctx, "cephfs: successfully unbounded volume %s from %s", req.GetVolumeId(), targetPath) return &csi.NodeUnpublishVolumeResponse{}, nil } @@ -324,7 +325,7 @@ func (ns *NodeServer) NodeUnstageVolume( volID := req.GetVolumeId() if acquired := ns.VolumeLocks.TryAcquire(volID); !acquired { - util.ErrorLog(ctx, util.VolumeOperationAlreadyExistsFmt, volID) + log.ErrorLog(ctx, util.VolumeOperationAlreadyExistsFmt, volID) return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, volID) } @@ -336,7 +337,7 @@ func (ns *NodeServer) NodeUnstageVolume( if err != nil { if os.IsNotExist(err) { // targetPath has already been deleted - util.DebugLog(ctx, "targetPath: %s has already been deleted", stagingTargetPath) + log.DebugLog(ctx, "targetPath: %s has already been deleted", stagingTargetPath) return &csi.NodeUnstageVolumeResponse{}, nil } @@ -351,7 +352,7 @@ func (ns *NodeServer) NodeUnstageVolume( return nil, status.Error(codes.Internal, err.Error()) } - util.DebugLog(ctx, "cephfs: successfully unmounted volume %s from %s", req.GetVolumeId(), stagingTargetPath) + log.DebugLog(ctx, "cephfs: successfully unmounted volume %s from %s", req.GetVolumeId(), stagingTargetPath) return &csi.NodeUnstageVolumeResponse{}, nil } diff --git a/internal/cephfs/snapshot.go b/internal/cephfs/snapshot.go index 4e448d68a..ed392b673 100644 --- a/internal/cephfs/snapshot.go +++ b/internal/cephfs/snapshot.go @@ -21,7 +21,7 @@ import ( "errors" "time" - "github.com/ceph/ceph-csi/internal/util" + "github.com/ceph/ceph-csi/internal/util/log" "github.com/ceph/go-ceph/cephfs/admin" "github.com/ceph/go-ceph/rados" @@ -51,14 +51,14 @@ type cephfsSnapshot struct { func (vo *volumeOptions) createSnapshot(ctx context.Context, snapID, volID volumeID) error { fsa, err := vo.conn.GetFSAdmin() if err != nil { - util.ErrorLog(ctx, "could not get FSAdmin: %s", err) + log.ErrorLog(ctx, "could not get FSAdmin: %s", err) return err } err = fsa.CreateSubVolumeSnapshot(vo.FsName, vo.SubvolumeGroup, string(volID), string(snapID)) if err != nil { - util.ErrorLog(ctx, "failed to create subvolume snapshot %s %s in fs %s: %s", + log.ErrorLog(ctx, "failed to create subvolume snapshot %s %s in fs %s: %s", string(snapID), string(volID), vo.FsName, err) return err @@ -70,14 +70,14 @@ func (vo *volumeOptions) createSnapshot(ctx context.Context, snapID, volID volum func (vo *volumeOptions) deleteSnapshot(ctx context.Context, snapID, volID volumeID) error { fsa, err := vo.conn.GetFSAdmin() if err != nil { - util.ErrorLog(ctx, "could not get FSAdmin: %s", err) + log.ErrorLog(ctx, "could not get FSAdmin: %s", err) return err } err = fsa.ForceRemoveSubVolumeSnapshot(vo.FsName, vo.SubvolumeGroup, string(volID), string(snapID)) if err != nil { - util.ErrorLog(ctx, "failed to delete subvolume snapshot %s %s in fs %s: %s", + log.ErrorLog(ctx, "failed to delete subvolume snapshot %s %s in fs %s: %s", string(snapID), string(volID), vo.FsName, err) return err @@ -97,7 +97,7 @@ func (vo *volumeOptions) getSnapshotInfo(ctx context.Context, snapID, volID volu snap := snapshotInfo{} fsa, err := vo.conn.GetFSAdmin() if err != nil { - util.ErrorLog(ctx, "could not get FSAdmin: %s", err) + log.ErrorLog(ctx, "could not get FSAdmin: %s", err) return snap, err } @@ -107,7 +107,7 @@ func (vo *volumeOptions) getSnapshotInfo(ctx context.Context, snapID, volID volu if errors.Is(err, rados.ErrNotFound) { return snap, ErrSnapNotFound } - util.ErrorLog( + log.ErrorLog( ctx, "failed to get subvolume snapshot info %s %s in fs %s with error %s", string(volID), @@ -132,7 +132,7 @@ func (vo *volumeOptions) protectSnapshot(ctx context.Context, snapID, volID volu } fsa, err := vo.conn.GetFSAdmin() if err != nil { - util.ErrorLog(ctx, "could not get FSAdmin: %s", err) + log.ErrorLog(ctx, "could not get FSAdmin: %s", err) return err } @@ -143,7 +143,7 @@ func (vo *volumeOptions) protectSnapshot(ctx context.Context, snapID, volID volu if errors.Is(err, rados.ErrObjectExists) { return nil } - util.ErrorLog( + log.ErrorLog( ctx, "failed to protect subvolume snapshot %s %s in fs %s with error: %s", string(volID), @@ -165,7 +165,7 @@ func (vo *volumeOptions) unprotectSnapshot(ctx context.Context, snapID, volID vo } fsa, err := vo.conn.GetFSAdmin() if err != nil { - util.ErrorLog(ctx, "could not get FSAdmin: %s", err) + log.ErrorLog(ctx, "could not get FSAdmin: %s", err) return err } @@ -178,7 +178,7 @@ func (vo *volumeOptions) unprotectSnapshot(ctx context.Context, snapID, volID vo if errors.Is(err, rados.ErrObjectExists) { return nil } - util.ErrorLog( + log.ErrorLog( ctx, "failed to unprotect subvolume snapshot %s %s in fs %s with error: %s", string(volID), @@ -199,7 +199,7 @@ func (vo *volumeOptions) cloneSnapshot( ) error { fsa, err := vo.conn.GetFSAdmin() if err != nil { - util.ErrorLog(ctx, "could not get FSAdmin: %s", err) + log.ErrorLog(ctx, "could not get FSAdmin: %s", err) return err } @@ -212,7 +212,7 @@ func (vo *volumeOptions) cloneSnapshot( err = fsa.CloneSubVolumeSnapshot(vo.FsName, vo.SubvolumeGroup, string(volID), string(snapID), string(cloneID), co) if err != nil { - util.ErrorLog( + log.ErrorLog( ctx, "failed to clone subvolume snapshot %s %s in fs %s with error: %s", string(volID), diff --git a/internal/cephfs/util.go b/internal/cephfs/util.go index 31c3cbcda..ee1920bd1 100644 --- a/internal/cephfs/util.go +++ b/internal/cephfs/util.go @@ -22,6 +22,7 @@ import ( "time" "github.com/ceph/ceph-csi/internal/util" + "github.com/ceph/ceph-csi/internal/util/log" "github.com/container-storage-interface/spec/lib/go/csi" "github.com/golang/protobuf/ptypes" @@ -130,7 +131,7 @@ func genSnapFromOptions(ctx context.Context, req *csi.CreateSnapshotRequest) (sn cephfsSnap.Monitors, cephfsSnap.ClusterID, err = util.GetMonsAndClusterID(snapOptions) if err != nil { - util.ErrorLog(ctx, "failed getting mons (%s)", err) + log.ErrorLog(ctx, "failed getting mons (%s)", err) return nil, err } @@ -144,7 +145,7 @@ func genSnapFromOptions(ctx context.Context, req *csi.CreateSnapshotRequest) (sn func parseTime(ctx context.Context, createTime time.Time) (*timestamp.Timestamp, error) { tm, err := ptypes.TimestampProto(createTime) if err != nil { - util.ErrorLog(ctx, "failed to convert time %s %v", createTime, err) + log.ErrorLog(ctx, "failed to convert time %s %v", createTime, err) return tm, err } diff --git a/internal/cephfs/volume.go b/internal/cephfs/volume.go index 9f3c84a67..cfa8b84e3 100644 --- a/internal/cephfs/volume.go +++ b/internal/cephfs/volume.go @@ -24,6 +24,7 @@ import ( "strings" "github.com/ceph/ceph-csi/internal/util" + "github.com/ceph/ceph-csi/internal/util/log" fsAdmin "github.com/ceph/go-ceph/cephfs/admin" "github.com/ceph/go-ceph/rados" @@ -59,13 +60,13 @@ func getVolumeRootPathCephDeprecated(volID volumeID) string { func (vo *volumeOptions) getVolumeRootPathCeph(ctx context.Context, volID volumeID) (string, error) { fsa, err := vo.conn.GetFSAdmin() if err != nil { - util.ErrorLog(ctx, "could not get FSAdmin err %s", err) + log.ErrorLog(ctx, "could not get FSAdmin err %s", err) return "", err } svPath, err := fsa.SubVolumePath(vo.FsName, vo.SubvolumeGroup, string(volID)) if err != nil { - util.ErrorLog(ctx, "failed to get the rootpath for the vol %s: %s", string(volID), err) + log.ErrorLog(ctx, "failed to get the rootpath for the vol %s: %s", string(volID), err) if errors.Is(err, rados.ErrNotFound) { return "", util.JoinErrors(ErrVolumeNotFound, err) } @@ -79,14 +80,14 @@ func (vo *volumeOptions) getVolumeRootPathCeph(ctx context.Context, volID volume func (vo *volumeOptions) getSubVolumeInfo(ctx context.Context, volID volumeID) (*Subvolume, error) { fsa, err := vo.conn.GetFSAdmin() if err != nil { - util.ErrorLog(ctx, "could not get FSAdmin, can not fetch metadata pool for %s:", vo.FsName, err) + log.ErrorLog(ctx, "could not get FSAdmin, can not fetch metadata pool for %s:", vo.FsName, err) return nil, err } info, err := fsa.SubVolumeInfo(vo.FsName, vo.SubvolumeGroup, string(volID)) if err != nil { - util.ErrorLog(ctx, "failed to get subvolume info for the vol %s: %s", string(volID), err) + log.ErrorLog(ctx, "failed to get subvolume info for the vol %s: %s", string(volID), err) if errors.Is(err, rados.ErrNotFound) { return nil, ErrVolumeNotFound } @@ -148,7 +149,7 @@ func createVolume(ctx context.Context, volOptions *volumeOptions, volID volumeID ca, err := volOptions.conn.GetFSAdmin() if err != nil { - util.ErrorLog(ctx, "could not get FSAdmin, can not create subvolume %s: %s", string(volID), err) + log.ErrorLog(ctx, "could not get FSAdmin, can not create subvolume %s: %s", string(volID), err) return err } @@ -158,7 +159,7 @@ func createVolume(ctx context.Context, volOptions *volumeOptions, volID volumeID opts := fsAdmin.SubVolumeGroupOptions{} err = ca.CreateSubVolumeGroup(volOptions.FsName, volOptions.SubvolumeGroup, &opts) if err != nil { - util.ErrorLog( + log.ErrorLog( ctx, "failed to create subvolume group %s, for the vol %s: %s", volOptions.SubvolumeGroup, @@ -167,7 +168,7 @@ func createVolume(ctx context.Context, volOptions *volumeOptions, volID volumeID return err } - util.DebugLog(ctx, "cephfs: created subvolume group %s", volOptions.SubvolumeGroup) + log.DebugLog(ctx, "cephfs: created subvolume group %s", volOptions.SubvolumeGroup) clusterAdditionalInfo[volOptions.ClusterID].subVolumeGroupCreated = true } @@ -182,7 +183,7 @@ func createVolume(ctx context.Context, volOptions *volumeOptions, volID volumeID // FIXME: check if the right credentials are used ("-n", cephEntityClientPrefix + cr.ID) err = ca.CreateSubVolume(volOptions.FsName, volOptions.SubvolumeGroup, string(volID), &opts) if err != nil { - util.ErrorLog(ctx, "failed to create subvolume %s in fs %s: %s", string(volID), volOptions.FsName, err) + log.ErrorLog(ctx, "failed to create subvolume %s in fs %s: %s", string(volID), volOptions.FsName, err) return err } @@ -207,7 +208,7 @@ func (vo *volumeOptions) resizeVolume(ctx context.Context, volID volumeID, bytes clusterAdditionalInfo[vo.ClusterID].resizeState == supported { fsa, err := vo.conn.GetFSAdmin() if err != nil { - util.ErrorLog(ctx, "could not get FSAdmin, can not resize volume %s:", vo.FsName, err) + log.ErrorLog(ctx, "could not get FSAdmin, can not resize volume %s:", vo.FsName, err) return err } @@ -220,7 +221,7 @@ func (vo *volumeOptions) resizeVolume(ctx context.Context, volID volumeID, bytes var invalid fsAdmin.NotImplementedError // In case the error is other than invalid command return error to the caller. if !errors.As(err, &invalid) { - util.ErrorLog(ctx, "failed to resize subvolume %s in fs %s: %s", string(volID), vo.FsName, err) + log.ErrorLog(ctx, "failed to resize subvolume %s in fs %s: %s", string(volID), vo.FsName, err) return err } @@ -233,7 +234,7 @@ func (vo *volumeOptions) resizeVolume(ctx context.Context, volID volumeID, bytes func (vo *volumeOptions) purgeVolume(ctx context.Context, volID volumeID, force bool) error { fsa, err := vo.conn.GetFSAdmin() if err != nil { - util.ErrorLog(ctx, "could not get FSAdmin %s:", err) + log.ErrorLog(ctx, "could not get FSAdmin %s:", err) return err } @@ -247,7 +248,7 @@ func (vo *volumeOptions) purgeVolume(ctx context.Context, volID volumeID, force err = fsa.RemoveSubVolumeWithFlags(vo.FsName, vo.SubvolumeGroup, string(volID), opt) if err != nil { - util.ErrorLog(ctx, "failed to purge subvolume %s in fs %s: %s", string(volID), vo.FsName, err) + log.ErrorLog(ctx, "failed to purge subvolume %s in fs %s: %s", string(volID), vo.FsName, err) if strings.Contains(err.Error(), volumeNotEmpty) { return util.JoinErrors(ErrVolumeHasSnapshots, err) } diff --git a/internal/cephfs/volumemounter.go b/internal/cephfs/volumemounter.go index 98fb78ddb..04ad0f5b0 100644 --- a/internal/cephfs/volumemounter.go +++ b/internal/cephfs/volumemounter.go @@ -28,6 +28,7 @@ import ( "sync" "github.com/ceph/ceph-csi/internal/util" + "github.com/ceph/ceph-csi/internal/util/log" ) const ( @@ -75,7 +76,7 @@ func loadAvailableMounters(conf *util.Config) error { err := kernelMounterProbe.Run() if err != nil { - util.ErrorLogMsg("failed to run mount.ceph %v", err) + log.ErrorLogMsg("failed to run mount.ceph %v", err) } else { // fetch the current running kernel info release, kvErr := util.GetKernelVersion() @@ -84,18 +85,18 @@ func loadAvailableMounters(conf *util.Config) error { } if conf.ForceKernelCephFS || util.CheckKernelSupport(release, quotaSupport) { - util.DefaultLog("loaded mounter: %s", volumeMounterKernel) + log.DefaultLog("loaded mounter: %s", volumeMounterKernel) availableMounters = append(availableMounters, volumeMounterKernel) } else { - util.DefaultLog("kernel version < 4.17 might not support quota feature, hence not loading kernel client") + log.DefaultLog("kernel version < 4.17 might not support quota feature, hence not loading kernel client") } } err = fuseMounterProbe.Run() if err != nil { - util.ErrorLogMsg("failed to run ceph-fuse %v", err) + log.ErrorLogMsg("failed to run ceph-fuse %v", err) } else { - util.DefaultLog("loaded mounter: %s", volumeMounterFuse) + log.DefaultLog("loaded mounter: %s", volumeMounterFuse) availableMounters = append(availableMounters, volumeMounterFuse) } @@ -131,7 +132,7 @@ func newMounter(volOptions *volumeOptions) (volumeMounter, error) { if chosenMounter == "" { // Otherwise pick whatever is left chosenMounter = availableMounters[0] - util.DebugLogMsg("requested mounter: %s, chosen mounter: %s", wantMounter, chosenMounter) + log.DebugLogMsg("requested mounter: %s, chosen mounter: %s", wantMounter, chosenMounter) } // Create the mounter @@ -291,10 +292,10 @@ func unmountVolume(ctx context.Context, mountPoint string) error { if ok { p, err := os.FindProcess(pid) if err != nil { - util.WarningLog(ctx, "failed to find process %d: %v", pid, err) + log.WarningLog(ctx, "failed to find process %d: %v", pid, err) } else { if _, err = p.Wait(); err != nil { - util.WarningLog(ctx, "%d is not a child process: %v", pid, err) + log.WarningLog(ctx, "%d is not a child process: %v", pid, err) } } } diff --git a/internal/controller/controller.go b/internal/controller/controller.go index d5c78b1d2..518ff0271 100644 --- a/internal/controller/controller.go +++ b/internal/controller/controller.go @@ -18,7 +18,7 @@ package controller import ( "fmt" - "github.com/ceph/ceph-csi/internal/util" + "github.com/ceph/ceph-csi/internal/util/log" clientConfig "sigs.k8s.io/controller-runtime/pkg/client/config" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -65,19 +65,19 @@ func Start(config Config) error { } mgr, err := manager.New(clientConfig.GetConfigOrDie(), opts) if err != nil { - util.ErrorLogMsg("failed to create manager %s", err) + log.ErrorLogMsg("failed to create manager %s", err) return err } err = addToManager(mgr, config) if err != nil { - util.ErrorLogMsg("failed to add manager %s", err) + log.ErrorLogMsg("failed to add manager %s", err) return err } err = mgr.Start(signals.SetupSignalHandler()) if err != nil { - util.ErrorLogMsg("failed to start manager %s", err) + log.ErrorLogMsg("failed to start manager %s", err) } return err diff --git a/internal/controller/persistentvolume/persistentvolume.go b/internal/controller/persistentvolume/persistentvolume.go index 8fd039ffa..0e81b58d5 100644 --- a/internal/controller/persistentvolume/persistentvolume.go +++ b/internal/controller/persistentvolume/persistentvolume.go @@ -24,6 +24,7 @@ import ( ctrl "github.com/ceph/ceph-csi/internal/controller" "github.com/ceph/ceph-csi/internal/rbd" "github.com/ceph/ceph-csi/internal/util" + "github.com/ceph/ceph-csi/internal/util/log" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -98,7 +99,7 @@ func (r *ReconcilePersistentVolume) getCredentials( if name == "" || namespace == "" { errStr := "secret name or secret namespace is empty" - util.ErrorLogMsg(errStr) + log.ErrorLogMsg(errStr) return nil, errors.New(errStr) } @@ -117,7 +118,7 @@ func (r *ReconcilePersistentVolume) getCredentials( cr, err = util.NewUserCredentials(credentials) if err != nil { - util.ErrorLogMsg("failed to get user credentials %s", err) + log.ErrorLogMsg("failed to get user credentials %s", err) return nil, err } @@ -201,7 +202,7 @@ func (r ReconcilePersistentVolume) reconcilePV(ctx context.Context, obj runtime. cr, err := r.getCredentials(ctx, secretName, secretNamespace) if err != nil { - util.ErrorLogMsg("failed to get credentials from secret %s", err) + log.ErrorLogMsg("failed to get credentials from secret %s", err) return err } @@ -209,14 +210,14 @@ func (r ReconcilePersistentVolume) reconcilePV(ctx context.Context, obj runtime. rbdVolID, err := rbd.RegenerateJournal(pv.Spec.CSI.VolumeAttributes, volumeHandler, requestName, cr) if err != nil { - util.ErrorLogMsg("failed to regenerate journal %s", err) + log.ErrorLogMsg("failed to regenerate journal %s", err) return err } if rbdVolID != volumeHandler { err = r.storeVolumeIDInPV(ctx, pv, rbdVolID) if err != nil { - util.ErrorLogMsg("failed to store volumeID in PV %s", err) + log.ErrorLogMsg("failed to store volumeID in PV %s", err) return err } diff --git a/internal/csi-common/controllerserver-default.go b/internal/csi-common/controllerserver-default.go index 05a0d674d..96b005989 100644 --- a/internal/csi-common/controllerserver-default.go +++ b/internal/csi-common/controllerserver-default.go @@ -19,7 +19,7 @@ package csicommon import ( "context" - "github.com/ceph/ceph-csi/internal/util" + "github.com/ceph/ceph-csi/internal/util/log" "github.com/container-storage-interface/spec/lib/go/csi" "google.golang.org/grpc/codes" @@ -71,7 +71,7 @@ func (cs *DefaultControllerServer) GetCapacity( func (cs *DefaultControllerServer) ControllerGetCapabilities( ctx context.Context, req *csi.ControllerGetCapabilitiesRequest) (*csi.ControllerGetCapabilitiesResponse, error) { - util.TraceLog(ctx, "Using default ControllerGetCapabilities") + log.TraceLog(ctx, "Using default ControllerGetCapabilities") if cs.Driver == nil { return nil, status.Error(codes.Unimplemented, "Controller server is not enabled") } diff --git a/internal/csi-common/driver.go b/internal/csi-common/driver.go index 670e4ca3f..b7bb3bd0c 100644 --- a/internal/csi-common/driver.go +++ b/internal/csi-common/driver.go @@ -19,12 +19,12 @@ package csicommon import ( "fmt" + "github.com/ceph/ceph-csi/internal/util/log" + "github.com/container-storage-interface/spec/lib/go/csi" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "k8s.io/klog/v2" - - "github.com/ceph/ceph-csi/internal/util" ) // CSIDriver stores driver information. @@ -91,7 +91,7 @@ func (d *CSIDriver) AddControllerServiceCapabilities(cl []csi.ControllerServiceC csc := make([]*csi.ControllerServiceCapability, 0, len(cl)) for _, c := range cl { - util.DefaultLog("Enabling controller service capability: %v", c.String()) + log.DefaultLog("Enabling controller service capability: %v", c.String()) csc = append(csc, NewControllerServiceCapability(c)) } @@ -103,7 +103,7 @@ func (d *CSIDriver) AddVolumeCapabilityAccessModes( vc []csi.VolumeCapability_AccessMode_Mode) []*csi.VolumeCapability_AccessMode { vca := make([]*csi.VolumeCapability_AccessMode, 0, len(vc)) for _, c := range vc { - util.DefaultLog("Enabling volume access mode: %v", c.String()) + log.DefaultLog("Enabling volume access mode: %v", c.String()) vca = append(vca, NewVolumeCapabilityAccessMode(c)) } d.vc = vca diff --git a/internal/csi-common/identityserver-default.go b/internal/csi-common/identityserver-default.go index 29a7f3e56..ac7cb6fe9 100644 --- a/internal/csi-common/identityserver-default.go +++ b/internal/csi-common/identityserver-default.go @@ -19,7 +19,7 @@ package csicommon import ( "context" - "github.com/ceph/ceph-csi/internal/util" + "github.com/ceph/ceph-csi/internal/util/log" "github.com/container-storage-interface/spec/lib/go/csi" "google.golang.org/grpc/codes" @@ -35,7 +35,7 @@ type DefaultIdentityServer struct { func (ids *DefaultIdentityServer) GetPluginInfo( ctx context.Context, req *csi.GetPluginInfoRequest) (*csi.GetPluginInfoResponse, error) { - util.TraceLog(ctx, "Using default GetPluginInfo") + log.TraceLog(ctx, "Using default GetPluginInfo") if ids.Driver.name == "" { return nil, status.Error(codes.Unavailable, "Driver name not configured") @@ -60,7 +60,7 @@ func (ids *DefaultIdentityServer) Probe(ctx context.Context, req *csi.ProbeReque func (ids *DefaultIdentityServer) GetPluginCapabilities( ctx context.Context, req *csi.GetPluginCapabilitiesRequest) (*csi.GetPluginCapabilitiesResponse, error) { - util.TraceLog(ctx, "Using default capabilities") + log.TraceLog(ctx, "Using default capabilities") return &csi.GetPluginCapabilitiesResponse{ Capabilities: []*csi.PluginCapability{ diff --git a/internal/csi-common/nodeserver-default.go b/internal/csi-common/nodeserver-default.go index e0f3ea304..fe848bdab 100644 --- a/internal/csi-common/nodeserver-default.go +++ b/internal/csi-common/nodeserver-default.go @@ -19,7 +19,7 @@ package csicommon import ( "context" - "github.com/ceph/ceph-csi/internal/util" + "github.com/ceph/ceph-csi/internal/util/log" "github.com/container-storage-interface/spec/lib/go/csi" "google.golang.org/grpc/codes" @@ -57,7 +57,7 @@ func (ns *DefaultNodeServer) NodeExpandVolume( func (ns *DefaultNodeServer) NodeGetInfo( ctx context.Context, req *csi.NodeGetInfoRequest) (*csi.NodeGetInfoResponse, error) { - util.TraceLog(ctx, "Using default NodeGetInfo") + log.TraceLog(ctx, "Using default NodeGetInfo") csiTopology := &csi.Topology{ Segments: ns.Driver.topology, @@ -73,7 +73,7 @@ func (ns *DefaultNodeServer) NodeGetInfo( func (ns *DefaultNodeServer) NodeGetCapabilities( ctx context.Context, req *csi.NodeGetCapabilitiesRequest) (*csi.NodeGetCapabilitiesResponse, error) { - util.TraceLog(ctx, "Using default NodeGetCapabilities") + log.TraceLog(ctx, "Using default NodeGetCapabilities") return &csi.NodeGetCapabilitiesResponse{ Capabilities: []*csi.NodeServiceCapability{ diff --git a/internal/csi-common/server.go b/internal/csi-common/server.go index a17daa153..2ab4ebc1e 100644 --- a/internal/csi-common/server.go +++ b/internal/csi-common/server.go @@ -23,6 +23,8 @@ import ( "strings" "sync" + "github.com/ceph/ceph-csi/internal/util/log" + "github.com/container-storage-interface/spec/lib/go/csi" "github.com/csi-addons/spec/lib/go/replication" grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" @@ -30,8 +32,6 @@ import ( "github.com/prometheus/client_golang/prometheus" "google.golang.org/grpc" "k8s.io/klog/v2" - - "github.com/ceph/ceph-csi/internal/util" ) // NonBlockingGRPCServer defines Non blocking GRPC server interfaces. @@ -128,7 +128,7 @@ func (s *nonBlockingGRPCServer) serve(endpoint, hstOptions string, srv Servers, replication.RegisterControllerServer(server, srv.RS) } - util.DefaultLog("Listening for connections on address: %#v", listener.Addr()) + log.DefaultLog("Listening for connections on address: %#v", listener.Addr()) if metrics { ho := strings.Split(hstOptions, ",") const expectedHo = 3 diff --git a/internal/csi-common/utils.go b/internal/csi-common/utils.go index d4d0ff946..020c67306 100644 --- a/internal/csi-common/utils.go +++ b/internal/csi-common/utils.go @@ -25,6 +25,7 @@ import ( "sync/atomic" "github.com/ceph/ceph-csi/internal/util" + "github.com/ceph/ceph-csi/internal/util/log" "github.com/container-storage-interface/spec/lib/go/csi" rp "github.com/csi-addons/replication-lib-utils/protosanitizer" @@ -160,9 +161,9 @@ func contextIDInjector( info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp interface{}, err error) { atomic.AddUint64(&id, 1) - ctx = context.WithValue(ctx, util.CtxKey, id) + ctx = context.WithValue(ctx, log.CtxKey, id) if reqID := getReqID(req); reqID != "" { - ctx = context.WithValue(ctx, util.ReqID, reqID) + ctx = context.WithValue(ctx, log.ReqID, reqID) } return handler(ctx, req) @@ -173,18 +174,18 @@ func logGRPC( req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { - util.ExtendedLog(ctx, "GRPC call: %s", info.FullMethod) + log.ExtendedLog(ctx, "GRPC call: %s", info.FullMethod) if isReplicationRequest(req) { - util.TraceLog(ctx, "GRPC request: %s", rp.StripReplicationSecrets(req)) + log.TraceLog(ctx, "GRPC request: %s", rp.StripReplicationSecrets(req)) } else { - util.TraceLog(ctx, "GRPC request: %s", protosanitizer.StripSecrets(req)) + log.TraceLog(ctx, "GRPC request: %s", protosanitizer.StripSecrets(req)) } resp, err := handler(ctx, req) if err != nil { - klog.Errorf(util.Log(ctx, "GRPC error: %v"), err) + klog.Errorf(log.Log(ctx, "GRPC error: %v"), err) } else { - util.TraceLog(ctx, "GRPC response: %s", protosanitizer.StripSecrets(resp)) + log.TraceLog(ctx, "GRPC response: %s", protosanitizer.StripSecrets(resp)) } return resp, err @@ -231,32 +232,32 @@ func FilesystemNodeGetVolumeStats(ctx context.Context, targetPath string) (*csi. available, ok := (*(volMetrics.Available)).AsInt64() if !ok { - util.ErrorLog(ctx, "failed to fetch available bytes") + log.ErrorLog(ctx, "failed to fetch available bytes") } capacity, ok := (*(volMetrics.Capacity)).AsInt64() if !ok { - util.ErrorLog(ctx, "failed to fetch capacity bytes") + log.ErrorLog(ctx, "failed to fetch capacity bytes") return nil, status.Error(codes.Unknown, "failed to fetch capacity bytes") } used, ok := (*(volMetrics.Used)).AsInt64() if !ok { - util.ErrorLog(ctx, "failed to fetch used bytes") + log.ErrorLog(ctx, "failed to fetch used bytes") } inodes, ok := (*(volMetrics.Inodes)).AsInt64() if !ok { - util.ErrorLog(ctx, "failed to fetch available inodes") + log.ErrorLog(ctx, "failed to fetch available inodes") return nil, status.Error(codes.Unknown, "failed to fetch available inodes") } inodesFree, ok := (*(volMetrics.InodesFree)).AsInt64() if !ok { - util.ErrorLog(ctx, "failed to fetch free inodes") + log.ErrorLog(ctx, "failed to fetch free inodes") } inodesUsed, ok := (*(volMetrics.InodesUsed)).AsInt64() if !ok { - util.ErrorLog(ctx, "failed to fetch used inodes") + log.ErrorLog(ctx, "failed to fetch used inodes") } return &csi.NodeGetVolumeStatsResponse{ diff --git a/internal/journal/omap.go b/internal/journal/omap.go index fcd20b8c2..0f0a6fdae 100644 --- a/internal/journal/omap.go +++ b/internal/journal/omap.go @@ -21,6 +21,7 @@ import ( "errors" "github.com/ceph/ceph-csi/internal/util" + "github.com/ceph/ceph-csi/internal/util/log" "github.com/ceph/go-ceph/rados" ) @@ -74,7 +75,7 @@ func getOMapValues( if err != nil { if errors.Is(err, rados.ErrNotFound) { - util.ErrorLog(ctx, "omap not found (pool=%q, namespace=%q, name=%q): %v", + log.ErrorLog(ctx, "omap not found (pool=%q, namespace=%q, name=%q): %v", poolName, namespace, oid, err) return nil, util.JoinErrors(util.ErrKeyNotFound, err) @@ -83,7 +84,7 @@ func getOMapValues( return nil, err } - util.DebugLog(ctx, "got omap values: (pool=%q, namespace=%q, name=%q): %+v", + log.DebugLog(ctx, "got omap values: (pool=%q, namespace=%q, name=%q): %+v", poolName, namespace, oid, results) return results, nil @@ -110,16 +111,16 @@ func removeMapKeys( // the previous implementation of removing omap keys (via the cli) // treated failure to find the omap as a non-error. Do so here to // mimic the previous behavior. - util.DebugLog(ctx, "when removing omap keys, omap not found (pool=%q, namespace=%q, name=%q): %+v", + log.DebugLog(ctx, "when removing omap keys, omap not found (pool=%q, namespace=%q, name=%q): %+v", poolName, namespace, oid, keys) } else { - util.ErrorLog(ctx, "failed removing omap keys (pool=%q, namespace=%q, name=%q): %v", + log.ErrorLog(ctx, "failed removing omap keys (pool=%q, namespace=%q, name=%q): %v", poolName, namespace, oid, err) return err } } - util.DebugLog(ctx, "removed omap keys (pool=%q, namespace=%q, name=%q): %+v", + log.DebugLog(ctx, "removed omap keys (pool=%q, namespace=%q, name=%q): %+v", poolName, namespace, oid, keys) return nil @@ -146,12 +147,12 @@ func setOMapKeys( } err = ioctx.SetOmap(oid, bpairs) if err != nil { - util.ErrorLog(ctx, "failed setting omap keys (pool=%q, namespace=%q, name=%q, pairs=%+v): %v", + log.ErrorLog(ctx, "failed setting omap keys (pool=%q, namespace=%q, name=%q, pairs=%+v): %v", poolName, namespace, oid, pairs, err) return err } - util.DebugLog(ctx, "set omap keys (pool=%q, namespace=%q, name=%q): %+v)", + log.DebugLog(ctx, "set omap keys (pool=%q, namespace=%q, name=%q): %+v)", poolName, namespace, oid, pairs) return nil diff --git a/internal/journal/voljournal.go b/internal/journal/voljournal.go index 46c12f20d..480a81414 100644 --- a/internal/journal/voljournal.go +++ b/internal/journal/voljournal.go @@ -25,6 +25,7 @@ import ( "strings" "github.com/ceph/ceph-csi/internal/util" + "github.com/ceph/ceph-csi/internal/util/log" "github.com/pborman/uuid" ) @@ -437,7 +438,7 @@ func (conn *Connection) UndoReservation(ctx context.Context, cj.cephUUIDDirectoryPrefix+imageUUID) if err != nil { if !errors.Is(err, util.ErrObjectNotFound) { - util.ErrorLog(ctx, "failed removing oMap %s (%s)", cj.cephUUIDDirectoryPrefix+imageUUID, err) + log.ErrorLog(ctx, "failed removing oMap %s (%s)", cj.cephUUIDDirectoryPrefix+imageUUID, err) return err } @@ -448,7 +449,7 @@ func (conn *Connection) UndoReservation(ctx context.Context, err := removeMapKeys(ctx, conn, csiJournalPool, cj.namespace, cj.csiDirectory, []string{cj.csiNameKeyPrefix + reqName}) if err != nil { - util.ErrorLog(ctx, "failed removing oMap key %s (%s)", cj.csiNameKeyPrefix+reqName, err) + log.ErrorLog(ctx, "failed removing oMap key %s (%s)", cj.csiNameKeyPrefix+reqName, err) return err } @@ -486,7 +487,7 @@ func reserveOMapName( if volUUID == "" && errors.Is(err, util.ErrObjectExists) { attempt++ // try again with a different uuid, for maxAttempts tries - util.DebugLog(ctx, "uuid (%s) conflict detected, retrying (attempt %d of %d)", + log.DebugLog(ctx, "uuid (%s) conflict detected, retrying (attempt %d of %d)", iterUUID, attempt, maxAttempts) continue @@ -590,10 +591,10 @@ func (conn *Connection) ReserveName(ctx context.Context, } defer func() { if err != nil { - util.WarningLog(ctx, "reservation failed for volume: %s", reqName) + log.WarningLog(ctx, "reservation failed for volume: %s", reqName) errDefer := conn.UndoReservation(ctx, imagePool, journalPool, imageName, reqName) if errDefer != nil { - util.WarningLog(ctx, "failed undoing reservation of volume: %s (%v)", reqName, errDefer) + log.WarningLog(ctx, "failed undoing reservation of volume: %s (%v)", reqName, errDefer) } } }() @@ -686,7 +687,7 @@ func (conn *Connection) GetImageAttributes( if !errors.Is(err, util.ErrKeyNotFound) && !errors.Is(err, util.ErrPoolNotFound) { return nil, err } - util.WarningLog(ctx, "unable to read omap keys: pool or key missing: %v", err) + log.WarningLog(ctx, "unable to read omap keys: pool or key missing: %v", err) } var found bool diff --git a/internal/liveness/liveness.go b/internal/liveness/liveness.go index bdd8badb4..778010e34 100644 --- a/internal/liveness/liveness.go +++ b/internal/liveness/liveness.go @@ -21,6 +21,7 @@ import ( "time" "github.com/ceph/ceph-csi/internal/util" + "github.com/ceph/ceph-csi/internal/util/log" connlib "github.com/kubernetes-csi/csi-lib-utils/connection" "github.com/kubernetes-csi/csi-lib-utils/metrics" @@ -39,23 +40,23 @@ func getLiveness(timeout time.Duration, csiConn *grpc.ClientConn) { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() - util.TraceLogMsg("Sending probe request to CSI driver") + log.TraceLogMsg("Sending probe request to CSI driver") ready, err := rpc.Probe(ctx, csiConn) if err != nil { liveness.Set(0) - util.ErrorLogMsg("health check failed: %v", err) + log.ErrorLogMsg("health check failed: %v", err) return } if !ready { liveness.Set(0) - util.ErrorLogMsg("driver responded but is not ready") + log.ErrorLogMsg("driver responded but is not ready") return } liveness.Set(1) - util.ExtendedLogMsg("Health check succeeded") + log.ExtendedLogMsg("Health check succeeded") } func recordLiveness(endpoint, drivername string, pollTime, timeout time.Duration) { @@ -63,14 +64,14 @@ func recordLiveness(endpoint, drivername string, pollTime, timeout time.Duration // register prometheus metrics err := prometheus.Register(liveness) if err != nil { - util.FatalLogMsg(err.Error()) + log.FatalLogMsg(err.Error()) } csiConn, err := connlib.Connect(endpoint, liveMetricsManager) if err != nil { // connlib should retry forever so a returned error should mean // the grpc client is misconfigured rather than an error on the network - util.FatalLogMsg("failed to establish connection to CSI driver: %v", err) + log.FatalLogMsg("failed to establish connection to CSI driver: %v", err) } // get liveness periodically @@ -83,7 +84,7 @@ func recordLiveness(endpoint, drivername string, pollTime, timeout time.Duration // Run starts liveness collection and prometheus endpoint. func Run(conf *util.Config) { - util.ExtendedLogMsg("Liveness Running") + log.ExtendedLogMsg("Liveness Running") // start liveness collection go recordLiveness(conf.Endpoint, conf.DriverName, conf.PollTime, conf.PoolTimeout) diff --git a/internal/rbd/clone.go b/internal/rbd/clone.go index d93b06e43..68fc34b3f 100644 --- a/internal/rbd/clone.go +++ b/internal/rbd/clone.go @@ -21,7 +21,7 @@ import ( "errors" "fmt" - "github.com/ceph/ceph-csi/internal/util" + "github.com/ceph/ceph-csi/internal/util/log" librbd "github.com/ceph/go-ceph/rbd" "google.golang.org/grpc/codes" @@ -103,14 +103,14 @@ func (rv *rbdVolume) checkCloneImage(ctx context.Context, parentVol *rbdVolume) // and add task to flatten temporary cloned image err = rv.cloneRbdImageFromSnapshot(ctx, snap, parentVol) if err != nil { - util.ErrorLog(ctx, "failed to clone rbd image %s from snapshot %s: %v", rv.RbdImageName, snap.RbdSnapName, err) + log.ErrorLog(ctx, "failed to clone rbd image %s from snapshot %s: %v", rv.RbdImageName, snap.RbdSnapName, err) err = fmt.Errorf("failed to clone rbd image %s from snapshot %s: %w", rv.RbdImageName, snap.RbdSnapName, err) return false, err } err = tempClone.deleteSnapshot(ctx, snap) if err != nil { - util.ErrorLog(ctx, "failed to delete snapshot: %v", err) + log.ErrorLog(ctx, "failed to delete snapshot: %v", err) return false, err } @@ -155,7 +155,7 @@ func (rv *rbdVolume) createCloneFromImage(ctx context.Context, parentVol *rbdVol err = rv.getImageID() if err != nil { - util.ErrorLog(ctx, "failed to get volume id %s: %v", rv, err) + log.ErrorLog(ctx, "failed to get volume id %s: %v", rv, err) return err } @@ -176,7 +176,7 @@ func (rv *rbdVolume) createCloneFromImage(ctx context.Context, parentVol *rbdVol err = j.StoreImageID(ctx, rv.JournalPool, rv.ReservedID, rv.ImageID) if err != nil { - util.ErrorLog(ctx, "failed to store volume %s: %v", rv, err) + log.ErrorLog(ctx, "failed to store volume %s: %v", rv, err) return err } @@ -213,7 +213,7 @@ func (rv *rbdVolume) doSnapClone(ctx context.Context, parentVol *rbdVolume) erro if err != nil || errClone != nil { cErr := cleanUpSnapshot(ctx, tempClone, cloneSnap, rv, rv.conn.Creds) if cErr != nil { - util.ErrorLog(ctx, "failed to cleanup image %s or snapshot %s: %v", cloneSnap, tempClone, cErr) + log.ErrorLog(ctx, "failed to cleanup image %s or snapshot %s: %v", cloneSnap, tempClone, cErr) } } @@ -222,7 +222,7 @@ func (rv *rbdVolume) doSnapClone(ctx context.Context, parentVol *rbdVolume) erro // cleanup snapshot cErr := cleanUpSnapshot(ctx, parentVol, tempSnap, tempClone, rv.conn.Creds) if cErr != nil { - util.ErrorLog(ctx, "failed to cleanup image %s or snapshot %s: %v", tempSnap, tempClone, cErr) + log.ErrorLog(ctx, "failed to cleanup image %s or snapshot %s: %v", tempSnap, tempClone, cErr) } } } diff --git a/internal/rbd/controllerserver.go b/internal/rbd/controllerserver.go index 9a35e3e3c..2f5b8e6c4 100644 --- a/internal/rbd/controllerserver.go +++ b/internal/rbd/controllerserver.go @@ -24,6 +24,7 @@ import ( csicommon "github.com/ceph/ceph-csi/internal/csi-common" "github.com/ceph/ceph-csi/internal/util" + "github.com/ceph/ceph-csi/internal/util/log" librbd "github.com/ceph/go-ceph/rbd" "github.com/container-storage-interface/spec/lib/go/csi" @@ -55,7 +56,7 @@ type ControllerServer struct { func (cs *ControllerServer) validateVolumeReq(ctx context.Context, req *csi.CreateVolumeRequest) error { if err := cs.Driver.ValidateControllerServiceRequest( csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME); err != nil { - util.ErrorLog(ctx, "invalid create volume req: %v", protosanitizer.StripSecrets(req)) + log.ErrorLog(ctx, "invalid create volume req: %v", protosanitizer.StripSecrets(req)) return err } @@ -286,7 +287,7 @@ func (cs *ControllerServer) CreateVolume( defer rbdVol.Destroy() // Existence and conflict checks if acquired := cs.VolumeLocks.TryAcquire(req.GetName()); !acquired { - util.ErrorLog(ctx, util.VolumeOperationAlreadyExistsFmt, req.GetName()) + log.ErrorLog(ctx, util.VolumeOperationAlreadyExistsFmt, req.GetName()) return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, req.GetName()) } @@ -294,7 +295,7 @@ func (cs *ControllerServer) CreateVolume( err = rbdVol.Connect(cr) if err != nil { - util.ErrorLog(ctx, "failed to connect to volume %v: %v", rbdVol.RbdImageName, err) + log.ErrorLog(ctx, "failed to connect to volume %v: %v", rbdVol.RbdImageName, err) return nil, status.Error(codes.Internal, err.Error()) } @@ -330,7 +331,7 @@ func (cs *ControllerServer) CreateVolume( if !errors.Is(err, ErrFlattenInProgress) { errDefer := undoVolReservation(ctx, rbdVol, cr) if errDefer != nil { - util.WarningLog(ctx, "failed undoing reservation of volume: %s (%s)", req.GetName(), errDefer) + log.WarningLog(ctx, "failed undoing reservation of volume: %s (%s)", req.GetName(), errDefer) } } } @@ -487,7 +488,7 @@ func flattenTemporaryClonedImages(ctx context.Context, rbdVol *rbdVolume, cr *ut } if len(snaps) > int(maxSnapshotsOnImage) { - util.DebugLog( + log.DebugLog( ctx, "snapshots count %d on image: %s reached configured hard limit %d", len(snaps), @@ -508,7 +509,7 @@ func flattenTemporaryClonedImages(ctx context.Context, rbdVol *rbdVolume, cr *ut } if len(snaps) > int(minSnapshotsOnImageToStartFlatten) { - util.DebugLog( + log.DebugLog( ctx, "snapshots count %d on image: %s reached configured soft limit %d", len(snaps), @@ -544,13 +545,13 @@ func checkFlatten(ctx context.Context, rbdVol *rbdVolume, cr *util.Credentials) return status.Error(codes.Aborted, err.Error()) } if errDefer := deleteImage(ctx, rbdVol, cr); errDefer != nil { - util.ErrorLog(ctx, "failed to delete rbd image: %s with error: %v", rbdVol, errDefer) + log.ErrorLog(ctx, "failed to delete rbd image: %s with error: %v", rbdVol, errDefer) return status.Error(codes.Internal, err.Error()) } errDefer := undoVolReservation(ctx, rbdVol, cr) if errDefer != nil { - util.WarningLog(ctx, "failed undoing reservation of volume: %s (%s)", rbdVol.RequestName, errDefer) + log.WarningLog(ctx, "failed undoing reservation of volume: %s (%s)", rbdVol.RequestName, errDefer) } return status.Error(codes.Internal, err.Error()) @@ -567,7 +568,7 @@ func (cs *ControllerServer) createVolumeFromSnapshot( snapshotID string) error { rbdSnap := &rbdSnapshot{} if acquired := cs.SnapshotLocks.TryAcquire(snapshotID); !acquired { - util.ErrorLog(ctx, util.SnapshotOperationAlreadyExistsFmt, snapshotID) + log.ErrorLog(ctx, util.SnapshotOperationAlreadyExistsFmt, snapshotID) return status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, snapshotID) } @@ -576,7 +577,7 @@ func (cs *ControllerServer) createVolumeFromSnapshot( err := genSnapFromSnapID(ctx, rbdSnap, snapshotID, cr, secrets) if err != nil { if errors.Is(err, util.ErrPoolNotFound) { - util.ErrorLog(ctx, "failed to get backend snapshot for %s: %v", snapshotID, err) + log.ErrorLog(ctx, "failed to get backend snapshot for %s: %v", snapshotID, err) return status.Error(codes.InvalidArgument, err.Error()) } @@ -603,13 +604,13 @@ func (cs *ControllerServer) createVolumeFromSnapshot( // create clone image and delete snapshot err = rbdVol.cloneRbdImageFromSnapshot(ctx, rbdSnap, parentVol) if err != nil { - util.ErrorLog(ctx, "failed to clone rbd image %s from snapshot %s: %v", rbdVol, rbdSnap, err) + log.ErrorLog(ctx, "failed to clone rbd image %s from snapshot %s: %v", rbdVol, rbdSnap, err) return err } } - util.DebugLog(ctx, "create volume %s from snapshot %s", rbdVol.RequestName, rbdSnap.RbdSnapName) + log.DebugLog(ctx, "create volume %s from snapshot %s", rbdVol.RequestName, rbdSnap.RbdSnapName) return nil } @@ -631,7 +632,7 @@ func (cs *ControllerServer) createBackingImage( switch { case rbdSnap != nil: if err = cs.OperationLocks.GetRestoreLock(rbdSnap.VolID); err != nil { - util.ErrorLog(ctx, err.Error()) + log.ErrorLog(ctx, err.Error()) return status.Error(codes.Aborted, err.Error()) } @@ -641,10 +642,10 @@ func (cs *ControllerServer) createBackingImage( if err != nil { return err } - util.DebugLog(ctx, "created volume %s from snapshot %s", rbdVol.RequestName, rbdSnap.RbdSnapName) + log.DebugLog(ctx, "created volume %s from snapshot %s", rbdVol.RequestName, rbdSnap.RbdSnapName) case parentVol != nil: if err = cs.OperationLocks.GetCloneLock(parentVol.VolID); err != nil { - util.ErrorLog(ctx, err.Error()) + log.ErrorLog(ctx, err.Error()) return status.Error(codes.Aborted, err.Error()) } @@ -654,19 +655,19 @@ func (cs *ControllerServer) createBackingImage( default: err = createImage(ctx, rbdVol, cr) if err != nil { - util.ErrorLog(ctx, "failed to create volume: %v", err) + log.ErrorLog(ctx, "failed to create volume: %v", err) return status.Error(codes.Internal, err.Error()) } } - util.DebugLog(ctx, "created volume %s backed by image %s", rbdVol.RequestName, rbdVol.RbdImageName) + log.DebugLog(ctx, "created volume %s backed by image %s", rbdVol.RequestName, rbdVol.RbdImageName) defer func() { if err != nil { if !errors.Is(err, ErrFlattenInProgress) { if deleteErr := deleteImage(ctx, rbdVol, cr); deleteErr != nil { - util.ErrorLog(ctx, "failed to delete rbd image: %s with error: %v", rbdVol, deleteErr) + log.ErrorLog(ctx, "failed to delete rbd image: %s with error: %v", rbdVol, deleteErr) } } } @@ -679,7 +680,7 @@ func (cs *ControllerServer) createBackingImage( if rbdSnap != nil { err = rbdVol.flattenRbdImage(ctx, cr, false, rbdHardMaxCloneDepth, rbdSoftMaxCloneDepth) if err != nil { - util.ErrorLog(ctx, "failed to flatten image %s: %v", rbdVol, err) + log.ErrorLog(ctx, "failed to flatten image %s: %v", rbdVol, err) return err } @@ -708,7 +709,7 @@ func checkContentSource( } rbdSnap := &rbdSnapshot{} if err := genSnapFromSnapID(ctx, rbdSnap, snapshotID, cr, req.GetSecrets()); err != nil { - util.ErrorLog(ctx, "failed to get backend snapshot for %s: %v", snapshotID, err) + log.ErrorLog(ctx, "failed to get backend snapshot for %s: %v", snapshotID, err) if !errors.Is(err, ErrSnapNotFound) { return nil, nil, status.Error(codes.Internal, err.Error()) } @@ -728,7 +729,7 @@ func checkContentSource( } rbdvol, err := genVolFromVolID(ctx, volID, cr, req.GetSecrets()) if err != nil { - util.ErrorLog(ctx, "failed to get backend image for %s: %v", volID, err) + log.ErrorLog(ctx, "failed to get backend image for %s: %v", volID, err) if !errors.Is(err, ErrImageNotFound) { return nil, nil, status.Error(codes.Internal, err.Error()) } @@ -750,7 +751,7 @@ func (cs *ControllerServer) DeleteVolume( req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) { if err := cs.Driver.ValidateControllerServiceRequest( csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME); err != nil { - util.ErrorLog(ctx, "invalid delete volume req: %v", protosanitizer.StripSecrets(req)) + log.ErrorLog(ctx, "invalid delete volume req: %v", protosanitizer.StripSecrets(req)) return nil, err } @@ -768,7 +769,7 @@ func (cs *ControllerServer) DeleteVolume( } if acquired := cs.VolumeLocks.TryAcquire(volumeID); !acquired { - util.ErrorLog(ctx, util.VolumeOperationAlreadyExistsFmt, volumeID) + log.ErrorLog(ctx, util.VolumeOperationAlreadyExistsFmt, volumeID) return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, volumeID) } @@ -776,7 +777,7 @@ func (cs *ControllerServer) DeleteVolume( // lock out volumeID for clone and expand operation if err = cs.OperationLocks.GetDeleteLock(volumeID); err != nil { - util.ErrorLog(ctx, err.Error()) + log.ErrorLog(ctx, err.Error()) return nil, status.Error(codes.Aborted, err.Error()) } @@ -786,7 +787,7 @@ func (cs *ControllerServer) DeleteVolume( defer rbdVol.Destroy() if err != nil { if errors.Is(err, util.ErrPoolNotFound) { - util.WarningLog(ctx, "failed to get backend volume for %s: %v", volumeID, err) + log.WarningLog(ctx, "failed to get backend volume for %s: %v", volumeID, err) return &csi.DeleteVolumeResponse{}, nil } @@ -795,7 +796,7 @@ func (cs *ControllerServer) DeleteVolume( // or partially complete (image and imageOMap are garbage collected already), hence return // success as deletion is complete if errors.Is(err, util.ErrKeyNotFound) { - util.WarningLog(ctx, "Failed to volume options for %s: %v", volumeID, err) + log.WarningLog(ctx, "Failed to volume options for %s: %v", volumeID, err) return &csi.DeleteVolumeResponse{}, nil } @@ -809,7 +810,7 @@ func (cs *ControllerServer) DeleteVolume( // to lead us to the image, hence the imageOMap needs to be garbage collected, by calling // unreserve for the same if acquired := cs.VolumeLocks.TryAcquire(rbdVol.RequestName); !acquired { - util.ErrorLog(ctx, util.VolumeOperationAlreadyExistsFmt, rbdVol.RequestName) + log.ErrorLog(ctx, util.VolumeOperationAlreadyExistsFmt, rbdVol.RequestName) return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, rbdVol.RequestName) } @@ -825,7 +826,7 @@ func (cs *ControllerServer) DeleteVolume( // lock out parallel create requests against the same volume name as we // clean up the image and associated omaps for the same if acquired := cs.VolumeLocks.TryAcquire(rbdVol.RequestName); !acquired { - util.ErrorLog(ctx, util.VolumeOperationAlreadyExistsFmt, rbdVol.RequestName) + log.ErrorLog(ctx, util.VolumeOperationAlreadyExistsFmt, rbdVol.RequestName) return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, rbdVol.RequestName) } @@ -839,7 +840,7 @@ func cleanupRBDImage(ctx context.Context, rbdVol *rbdVolume, cr *util.Credentials) (*csi.DeleteVolumeResponse, error) { mirroringInfo, err := rbdVol.getImageMirroringInfo() if err != nil { - util.ErrorLog(ctx, err.Error()) + log.ErrorLog(ctx, err.Error()) return nil, status.Error(codes.Internal, err.Error()) } @@ -862,7 +863,7 @@ func cleanupRBDImage(ctx context.Context, } if localStatus.Up && localStatus.State == librbd.MirrorImageStatusStateReplaying { if err = undoVolReservation(ctx, rbdVol, cr); err != nil { - util.ErrorLog(ctx, "failed to remove reservation for volume (%s) with backing image (%s) (%s)", + log.ErrorLog(ctx, "failed to remove reservation for volume (%s) with backing image (%s) (%s)", rbdVol.RequestName, rbdVol.RbdImageName, err) return nil, status.Error(codes.Internal, err.Error()) @@ -870,7 +871,7 @@ func cleanupRBDImage(ctx context.Context, return &csi.DeleteVolumeResponse{}, nil } - util.ErrorLog(ctx, + log.ErrorLog(ctx, "secondary image status is up=%t and state=%s", localStatus.Up, localStatus.State) @@ -878,12 +879,12 @@ func cleanupRBDImage(ctx context.Context, inUse, err := rbdVol.isInUse() if err != nil { - util.ErrorLog(ctx, "failed getting information for image (%s): (%s)", rbdVol, err) + log.ErrorLog(ctx, "failed getting information for image (%s): (%s)", rbdVol, err) return nil, status.Error(codes.Internal, err.Error()) } if inUse { - util.ErrorLog(ctx, "rbd %s is still being used", rbdVol) + log.ErrorLog(ctx, "rbd %s is still being used", rbdVol) return nil, status.Errorf(codes.Internal, "rbd %s is still being used", rbdVol.RbdImageName) } @@ -895,7 +896,7 @@ func cleanupRBDImage(ctx context.Context, if err != nil { // return error if it is not ErrImageNotFound if !errors.Is(err, ErrImageNotFound) { - util.ErrorLog(ctx, "failed to delete rbd image: %s with error: %v", + log.ErrorLog(ctx, "failed to delete rbd image: %s with error: %v", tempClone, err) return nil, status.Error(codes.Internal, err.Error()) @@ -903,16 +904,16 @@ func cleanupRBDImage(ctx context.Context, } // Deleting rbd image - util.DebugLog(ctx, "deleting image %s", rbdVol.RbdImageName) + log.DebugLog(ctx, "deleting image %s", rbdVol.RbdImageName) if err = deleteImage(ctx, rbdVol, cr); err != nil { - util.ErrorLog(ctx, "failed to delete rbd image: %s with error: %v", + log.ErrorLog(ctx, "failed to delete rbd image: %s with error: %v", rbdVol, err) return nil, status.Error(codes.Internal, err.Error()) } if err = undoVolReservation(ctx, rbdVol, cr); err != nil { - util.ErrorLog(ctx, "failed to remove reservation for volume (%s) with backing image (%s) (%s)", + log.ErrorLog(ctx, "failed to remove reservation for volume (%s) with backing image (%s) (%s)", rbdVol.RequestName, rbdVol.RbdImageName, err) return nil, status.Error(codes.Internal, err.Error()) @@ -969,7 +970,7 @@ func (cs *ControllerServer) CreateSnapshot( case errors.Is(err, ErrImageNotFound): err = status.Errorf(codes.NotFound, "source Volume ID %s not found", req.GetSourceVolumeId()) case errors.Is(err, util.ErrPoolNotFound): - util.ErrorLog(ctx, "failed to get backend volume for %s: %v", req.GetSourceVolumeId(), err) + log.ErrorLog(ctx, "failed to get backend volume for %s: %v", req.GetSourceVolumeId(), err) err = status.Errorf(codes.NotFound, err.Error()) default: err = status.Errorf(codes.Internal, err.Error()) @@ -996,7 +997,7 @@ func (cs *ControllerServer) CreateSnapshot( rbdSnap.RequestName = req.GetName() if acquired := cs.SnapshotLocks.TryAcquire(req.GetName()); !acquired { - util.ErrorLog(ctx, util.SnapshotOperationAlreadyExistsFmt, req.GetName()) + log.ErrorLog(ctx, util.SnapshotOperationAlreadyExistsFmt, req.GetName()) return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, req.GetName()) } @@ -1004,7 +1005,7 @@ func (cs *ControllerServer) CreateSnapshot( // Take lock on parent rbd image if err = cs.OperationLocks.GetSnapshotCreateLock(rbdSnap.SourceVolumeID); err != nil { - util.ErrorLog(ctx, err.Error()) + log.ErrorLog(ctx, err.Error()) return nil, status.Error(codes.Aborted, err.Error()) } @@ -1037,7 +1038,7 @@ func (cs *ControllerServer) CreateSnapshot( if err != nil && !errors.Is(err, ErrFlattenInProgress) { errDefer := undoSnapReservation(ctx, rbdSnap, cr) if errDefer != nil { - util.WarningLog(ctx, "failed undoing reservation of snapshot: %s %v", req.GetName(), errDefer) + log.WarningLog(ctx, "failed undoing reservation of snapshot: %s %v", req.GetName(), errDefer) } } }() @@ -1070,7 +1071,7 @@ func cloneFromSnapshot( if err != nil { uErr := undoSnapshotCloning(ctx, rbdVol, rbdSnap, vol, cr) if uErr != nil { - util.WarningLog(ctx, "failed undoing reservation of snapshot: %s %v", rbdSnap.RequestName, uErr) + log.WarningLog(ctx, "failed undoing reservation of snapshot: %s %v", rbdSnap.RequestName, uErr) } return nil, status.Errorf(codes.Internal, err.Error()) @@ -1116,7 +1117,7 @@ func cloneFromSnapshot( } else if err != nil { uErr := undoSnapshotCloning(ctx, rbdVol, rbdSnap, vol, cr) if uErr != nil { - util.WarningLog(ctx, "failed undoing reservation of snapshot: %s %v", rbdSnap.RequestName, uErr) + log.WarningLog(ctx, "failed undoing reservation of snapshot: %s %v", rbdSnap.RequestName, uErr) } return nil, status.Errorf(codes.Internal, err.Error()) @@ -1136,7 +1137,7 @@ func cloneFromSnapshot( func (cs *ControllerServer) validateSnapshotReq(ctx context.Context, req *csi.CreateSnapshotRequest) error { if err := cs.Driver.ValidateControllerServiceRequest( csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT); err != nil { - util.ErrorLog(ctx, "invalid create snapshot req: %v", protosanitizer.StripSecrets(req)) + log.ErrorLog(ctx, "invalid create snapshot req: %v", protosanitizer.StripSecrets(req)) return err } @@ -1179,7 +1180,7 @@ func (cs *ControllerServer) doSnapshotClone( err = createRBDClone(ctx, parentVol, cloneRbd, rbdSnap, cr) if err != nil { - util.ErrorLog(ctx, "failed to create snapshot: %v", err) + log.ErrorLog(ctx, "failed to create snapshot: %v", err) return cloneRbd, err } @@ -1190,7 +1191,7 @@ func (cs *ControllerServer) doSnapshotClone( // cleanup clone and snapshot errCleanUp := cleanUpSnapshot(ctx, cloneRbd, rbdSnap, cloneRbd, cr) if errCleanUp != nil { - util.ErrorLog(ctx, "failed to cleanup snapshot and clone: %v", errCleanUp) + log.ErrorLog(ctx, "failed to cleanup snapshot and clone: %v", errCleanUp) } } } @@ -1199,7 +1200,7 @@ func (cs *ControllerServer) doSnapshotClone( if parentVol.isEncrypted() { cryptErr := parentVol.copyEncryptionConfig(&cloneRbd.rbdImage) if cryptErr != nil { - util.WarningLog(ctx, "failed copy encryption "+ + log.WarningLog(ctx, "failed copy encryption "+ "config for %q: %v", cloneRbd, cryptErr) return nil, err @@ -1226,7 +1227,7 @@ func (cs *ControllerServer) doSnapshotClone( if err != nil { // update rbd image name for logging rbdSnap.RbdImageName = cloneRbd.RbdImageName - util.ErrorLog(ctx, "failed to create snapshot %s: %v", rbdSnap, err) + log.ErrorLog(ctx, "failed to create snapshot %s: %v", rbdSnap, err) return cloneRbd, err } @@ -1234,14 +1235,14 @@ func (cs *ControllerServer) doSnapshotClone( err = cloneRbd.getImageID() if err != nil { - util.ErrorLog(ctx, "failed to get image id: %v", err) + log.ErrorLog(ctx, "failed to get image id: %v", err) return cloneRbd, err } // save image ID j, err := snapJournal.Connect(rbdSnap.Monitors, rbdSnap.RadosNamespace, cr) if err != nil { - util.ErrorLog(ctx, "failed to connect to cluster: %v", err) + log.ErrorLog(ctx, "failed to connect to cluster: %v", err) return cloneRbd, err } @@ -1249,7 +1250,7 @@ func (cs *ControllerServer) doSnapshotClone( err = j.StoreImageID(ctx, rbdSnap.JournalPool, rbdSnap.ReservedID, cloneRbd.ImageID) if err != nil { - util.ErrorLog(ctx, "failed to reserve volume id: %v", err) + log.ErrorLog(ctx, "failed to reserve volume id: %v", err) return cloneRbd, err } @@ -1269,7 +1270,7 @@ func (cs *ControllerServer) DeleteSnapshot( req *csi.DeleteSnapshotRequest) (*csi.DeleteSnapshotResponse, error) { if err := cs.Driver.ValidateControllerServiceRequest( csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT); err != nil { - util.ErrorLog(ctx, "invalid delete snapshot req: %v", protosanitizer.StripSecrets(req)) + log.ErrorLog(ctx, "invalid delete snapshot req: %v", protosanitizer.StripSecrets(req)) return nil, err } @@ -1286,7 +1287,7 @@ func (cs *ControllerServer) DeleteSnapshot( } if acquired := cs.SnapshotLocks.TryAcquire(snapshotID); !acquired { - util.ErrorLog(ctx, util.SnapshotOperationAlreadyExistsFmt, snapshotID) + log.ErrorLog(ctx, util.SnapshotOperationAlreadyExistsFmt, snapshotID) return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, snapshotID) } @@ -1294,7 +1295,7 @@ func (cs *ControllerServer) DeleteSnapshot( // lock out snapshotID for restore operation if err = cs.OperationLocks.GetDeleteLock(snapshotID); err != nil { - util.ErrorLog(ctx, err.Error()) + log.ErrorLog(ctx, err.Error()) return nil, status.Error(codes.Aborted, err.Error()) } @@ -1305,7 +1306,7 @@ func (cs *ControllerServer) DeleteSnapshot( // if error is ErrPoolNotFound, the pool is already deleted we don't // need to worry about deleting snapshot or omap data, return success if errors.Is(err, util.ErrPoolNotFound) { - util.WarningLog(ctx, "failed to get backend snapshot for %s: %v", snapshotID, err) + log.WarningLog(ctx, "failed to get backend snapshot for %s: %v", snapshotID, err) return &csi.DeleteSnapshotResponse{}, nil } @@ -1323,14 +1324,14 @@ func (cs *ControllerServer) DeleteSnapshot( // safeguard against parallel create or delete requests against the same // name if acquired := cs.SnapshotLocks.TryAcquire(rbdSnap.RequestName); !acquired { - util.ErrorLog(ctx, util.SnapshotOperationAlreadyExistsFmt, rbdSnap.RequestName) + log.ErrorLog(ctx, util.SnapshotOperationAlreadyExistsFmt, rbdSnap.RequestName) return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, rbdSnap.RequestName) } defer cs.SnapshotLocks.Release(rbdSnap.RequestName) // Deleting snapshot and cloned volume - util.DebugLog(ctx, "deleting cloned rbd volume %s", rbdSnap.RbdSnapName) + log.DebugLog(ctx, "deleting cloned rbd volume %s", rbdSnap.RbdSnapName) rbdVol := generateVolFromSnap(rbdSnap) @@ -1343,7 +1344,7 @@ func (cs *ControllerServer) DeleteSnapshot( err = rbdVol.getImageInfo() if err != nil { if !errors.Is(err, ErrImageNotFound) { - util.ErrorLog(ctx, "failed to delete rbd image: %s/%s with error: %v", rbdVol.Pool, rbdVol.VolName, err) + log.ErrorLog(ctx, "failed to delete rbd image: %s/%s with error: %v", rbdVol.Pool, rbdVol.VolName, err) return nil, status.Error(codes.Internal, err.Error()) } @@ -1353,14 +1354,14 @@ func (cs *ControllerServer) DeleteSnapshot( rbdSnap.RbdImageName = rbdVol.RbdImageName err = cleanUpSnapshot(ctx, rbdVol, rbdSnap, rbdVol, cr) if err != nil { - util.ErrorLog(ctx, "failed to delete image: %v", err) + log.ErrorLog(ctx, "failed to delete image: %v", err) return nil, status.Error(codes.Internal, err.Error()) } } err = undoSnapReservation(ctx, rbdSnap, cr) if err != nil { - util.ErrorLog(ctx, "failed to remove reservation for snapname (%s) with backing snap (%s) on image (%s) (%s)", + log.ErrorLog(ctx, "failed to remove reservation for snapname (%s) with backing snap (%s) on image (%s) (%s)", rbdSnap.RequestName, rbdSnap.RbdSnapName, rbdSnap.RbdImageName, err) return nil, status.Error(codes.Internal, err.Error()) @@ -1374,7 +1375,7 @@ func (cs *ControllerServer) ControllerExpandVolume( ctx context.Context, req *csi.ControllerExpandVolumeRequest) (*csi.ControllerExpandVolumeResponse, error) { if err := cs.Driver.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_EXPAND_VOLUME); err != nil { - util.ErrorLog(ctx, "invalid expand volume req: %v", protosanitizer.StripSecrets(req)) + log.ErrorLog(ctx, "invalid expand volume req: %v", protosanitizer.StripSecrets(req)) return nil, err } @@ -1391,7 +1392,7 @@ func (cs *ControllerServer) ControllerExpandVolume( // lock out parallel requests against the same volume ID if acquired := cs.VolumeLocks.TryAcquire(volID); !acquired { - util.ErrorLog(ctx, util.VolumeOperationAlreadyExistsFmt, volID) + log.ErrorLog(ctx, util.VolumeOperationAlreadyExistsFmt, volID) return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, volID) } @@ -1410,7 +1411,7 @@ func (cs *ControllerServer) ControllerExpandVolume( case errors.Is(err, ErrImageNotFound): err = status.Errorf(codes.NotFound, "volume ID %s not found", volID) case errors.Is(err, util.ErrPoolNotFound): - util.ErrorLog(ctx, "failed to get backend volume for %s: %v", volID, err) + log.ErrorLog(ctx, "failed to get backend volume for %s: %v", volID, err) err = status.Errorf(codes.NotFound, err.Error()) default: err = status.Errorf(codes.Internal, err.Error()) @@ -1430,7 +1431,7 @@ func (cs *ControllerServer) ControllerExpandVolume( // lock out volumeID for clone and delete operation if err = cs.OperationLocks.GetExpandLock(volID); err != nil { - util.ErrorLog(ctx, err.Error()) + log.ErrorLog(ctx, err.Error()) return nil, status.Error(codes.Aborted, err.Error()) } @@ -1441,10 +1442,10 @@ func (cs *ControllerServer) ControllerExpandVolume( // resize volume if required if rbdVol.VolSize < volSize { - util.DebugLog(ctx, "rbd volume %s size is %v,resizing to %v", rbdVol, rbdVol.VolSize, volSize) + log.DebugLog(ctx, "rbd volume %s size is %v,resizing to %v", rbdVol, rbdVol.VolSize, volSize) err = rbdVol.resize(volSize) if err != nil { - util.ErrorLog(ctx, "failed to resize rbd image: %s with error: %v", rbdVol, err) + log.ErrorLog(ctx, "failed to resize rbd image: %s with error: %v", rbdVol, err) return nil, status.Error(codes.Internal, err.Error()) } diff --git a/internal/rbd/driver.go b/internal/rbd/driver.go index f48daabe5..c82ee0b21 100644 --- a/internal/rbd/driver.go +++ b/internal/rbd/driver.go @@ -20,6 +20,7 @@ import ( csicommon "github.com/ceph/ceph-csi/internal/csi-common" "github.com/ceph/ceph-csi/internal/journal" "github.com/ceph/ceph-csi/internal/util" + "github.com/ceph/ceph-csi/internal/util/log" "github.com/container-storage-interface/spec/lib/go/csi" mount "k8s.io/mount-utils" @@ -107,7 +108,7 @@ func (r *Driver) Run(conf *util.Config) { // Create ceph.conf for use with CLI commands if err = util.WriteCephConfig(); err != nil { - util.FatalLogMsg("failed to write ceph configuration file (%v)", err) + log.FatalLogMsg("failed to write ceph configuration file (%v)", err) } // Use passed in instance ID, if provided for omap suffix naming @@ -128,7 +129,7 @@ func (r *Driver) Run(conf *util.Config) { // Initialize default library driver r.cd = csicommon.NewCSIDriver(conf.DriverName, util.DriverVersion, conf.NodeID) if r.cd == nil { - util.FatalLogMsg("Failed to initialize CSI Driver.") + log.FatalLogMsg("Failed to initialize CSI Driver.") } if conf.IsControllerServer || !conf.IsNodeServer { r.cd.AddControllerServiceCapabilities([]csi.ControllerServiceCapability_RPC_Type{ @@ -155,11 +156,11 @@ func (r *Driver) Run(conf *util.Config) { if conf.IsNodeServer { topology, err = util.GetTopologyFromDomainLabels(conf.DomainLabels, conf.NodeID, conf.DriverName) if err != nil { - util.FatalLogMsg(err.Error()) + log.FatalLogMsg(err.Error()) } r.ns, err = NewNodeServer(r.cd, conf.Vtype, topology) if err != nil { - util.FatalLogMsg("failed to start node server, err %v\n", err) + log.FatalLogMsg("failed to start node server, err %v\n", err) } } @@ -170,11 +171,11 @@ func (r *Driver) Run(conf *util.Config) { if !conf.IsControllerServer && !conf.IsNodeServer { topology, err = util.GetTopologyFromDomainLabels(conf.DomainLabels, conf.NodeID, conf.DriverName) if err != nil { - util.FatalLogMsg(err.Error()) + log.FatalLogMsg(err.Error()) } r.ns, err = NewNodeServer(r.cd, conf.Vtype, topology) if err != nil { - util.FatalLogMsg("failed to start node server, err %v\n", err) + log.FatalLogMsg("failed to start node server, err %v\n", err) } r.cs = NewControllerServer(r.cd) } @@ -190,14 +191,14 @@ func (r *Driver) Run(conf *util.Config) { } s.Start(conf.Endpoint, conf.HistogramOption, srv, conf.EnableGRPCMetrics) if conf.EnableGRPCMetrics { - util.WarningLogMsg("EnableGRPCMetrics is deprecated") + log.WarningLogMsg("EnableGRPCMetrics is deprecated") go util.StartMetricsServer(conf) } if conf.EnableProfiling { if !conf.EnableGRPCMetrics { go util.StartMetricsServer(conf) } - util.DebugLogMsg("Registering profiling handler") + log.DebugLogMsg("Registering profiling handler") go util.EnableProfiling() } if conf.IsNodeServer { @@ -205,7 +206,7 @@ func (r *Driver) Run(conf *util.Config) { // TODO: move the healer to csi-addons err := runVolumeHealer(r.ns, conf) if err != nil { - util.ErrorLogMsg("healer had failures, err %v\n", err) + log.ErrorLogMsg("healer had failures, err %v\n", err) } }() } diff --git a/internal/rbd/encryption.go b/internal/rbd/encryption.go index a0d1c0b83..99d35469f 100644 --- a/internal/rbd/encryption.go +++ b/internal/rbd/encryption.go @@ -23,6 +23,7 @@ import ( "strings" "github.com/ceph/ceph-csi/internal/util" + "github.com/ceph/ceph-csi/internal/util/log" librbd "github.com/ceph/go-ceph/rbd" ) @@ -65,17 +66,17 @@ const ( func (ri *rbdImage) checkRbdImageEncrypted(ctx context.Context) (rbdEncryptionState, error) { value, err := ri.MigrateMetadata(oldEncryptionMetaKey, encryptionMetaKey, string(rbdImageEncryptionUnknown)) if errors.Is(err, librbd.ErrNotFound) { - util.DebugLog(ctx, "image %s encrypted state not set", ri) + log.DebugLog(ctx, "image %s encrypted state not set", ri) return rbdImageEncryptionUnknown, nil } else if err != nil { - util.ErrorLog(ctx, "checking image %s encrypted state metadata failed: %s", ri, err) + log.ErrorLog(ctx, "checking image %s encrypted state metadata failed: %s", ri, err) return rbdImageEncryptionUnknown, err } encrypted := rbdEncryptionState(strings.TrimSpace(value)) - util.DebugLog(ctx, "image %s encrypted state metadata reports %q", ri, encrypted) + log.DebugLog(ctx, "image %s encrypted state metadata reports %q", ri, encrypted) return encrypted, nil } @@ -100,7 +101,7 @@ func (ri *rbdImage) isEncrypted() bool { func (ri *rbdImage) setupEncryption(ctx context.Context) error { err := ri.encryption.StoreNewCryptoPassphrase(ri.VolID) if err != nil { - util.ErrorLog(ctx, "failed to save encryption passphrase for "+ + log.ErrorLog(ctx, "failed to save encryption passphrase for "+ "image %s: %s", ri, err) return err @@ -108,7 +109,7 @@ func (ri *rbdImage) setupEncryption(ctx context.Context) error { err = ri.ensureEncryptionMetadataSet(rbdImageEncryptionPrepared) if err != nil { - util.ErrorLog(ctx, "failed to save encryption status, deleting "+ + log.ErrorLog(ctx, "failed to save encryption status, deleting "+ "image %s: %s", ri, err) return err @@ -185,7 +186,7 @@ func (ri *rbdImage) repairEncryptionConfig(dest *rbdImage) error { func (ri *rbdImage) encryptDevice(ctx context.Context, devicePath string) error { passphrase, err := ri.encryption.GetCryptoPassphrase(ri.VolID) if err != nil { - util.ErrorLog(ctx, "failed to get crypto passphrase for %s: %v", + log.ErrorLog(ctx, "failed to get crypto passphrase for %s: %v", ri, err) return err @@ -193,14 +194,14 @@ func (ri *rbdImage) encryptDevice(ctx context.Context, devicePath string) error if err = util.EncryptVolume(ctx, devicePath, passphrase); err != nil { err = fmt.Errorf("failed to encrypt volume %s: %w", ri, err) - util.ErrorLog(ctx, err.Error()) + log.ErrorLog(ctx, err.Error()) return err } err = ri.ensureEncryptionMetadataSet(rbdImageEncrypted) if err != nil { - util.ErrorLog(ctx, err.Error()) + log.ErrorLog(ctx, err.Error()) return err } @@ -211,7 +212,7 @@ func (ri *rbdImage) encryptDevice(ctx context.Context, devicePath string) error func (rv *rbdVolume) openEncryptedDevice(ctx context.Context, devicePath string) (string, error) { passphrase, err := rv.encryption.GetCryptoPassphrase(rv.VolID) if err != nil { - util.ErrorLog(ctx, "failed to get passphrase for encrypted device %s: %v", + log.ErrorLog(ctx, "failed to get passphrase for encrypted device %s: %v", rv, err) return "", err @@ -221,16 +222,16 @@ func (rv *rbdVolume) openEncryptedDevice(ctx context.Context, devicePath string) isOpen, err := util.IsDeviceOpen(ctx, mapperFilePath) if err != nil { - util.ErrorLog(ctx, "failed to check device %s encryption status: %s", devicePath, err) + log.ErrorLog(ctx, "failed to check device %s encryption status: %s", devicePath, err) return devicePath, err } if isOpen { - util.DebugLog(ctx, "encrypted device is already open at %s", mapperFilePath) + log.DebugLog(ctx, "encrypted device is already open at %s", mapperFilePath) } else { err = util.OpenEncryptedVolume(ctx, devicePath, mapperFile, passphrase) if err != nil { - util.ErrorLog(ctx, "failed to open device %s: %v", + log.ErrorLog(ctx, "failed to open device %s: %v", rv, err) return devicePath, err @@ -270,7 +271,7 @@ func (ri *rbdImage) ParseEncryptionOpts(ctx context.Context, volOptions map[stri // FIXME: this works only on Kubernetes, how do other CO supply metadata? ri.Owner, ok = volOptions["csi.storage.k8s.io/pvc/namespace"] if !ok { - util.DebugLog(ctx, "could not detect owner for %s", ri) + log.DebugLog(ctx, "could not detect owner for %s", ri) } encrypted, ok = volOptions["encrypted"] diff --git a/internal/rbd/nodeserver.go b/internal/rbd/nodeserver.go index 03ddfee6f..b9b41f519 100644 --- a/internal/rbd/nodeserver.go +++ b/internal/rbd/nodeserver.go @@ -27,6 +27,7 @@ import ( csicommon "github.com/ceph/ceph-csi/internal/csi-common" "github.com/ceph/ceph-csi/internal/journal" "github.com/ceph/ceph-csi/internal/util" + "github.com/ceph/ceph-csi/internal/util/log" librbd "github.com/ceph/go-ceph/rbd" "github.com/container-storage-interface/spec/lib/go/csi" @@ -131,7 +132,7 @@ func isStaticVolume(parameters map[string]string) bool { func healerStageTransaction(ctx context.Context, cr *util.Credentials, volOps *rbdVolume, metaDataPath string) error { imgInfo, err := lookupRBDImageMetadataStash(metaDataPath) if err != nil { - util.ErrorLog(ctx, "failed to find image metadata, at stagingPath: %s, err: %v", metaDataPath, err) + log.ErrorLog(ctx, "failed to find image metadata, at stagingPath: %s, err: %v", metaDataPath, err) return err } @@ -143,7 +144,7 @@ func healerStageTransaction(ctx context.Context, cr *util.Credentials, volOps *r if err != nil { return err } - util.DebugLog(ctx, "rbd volID: %s was successfully attached to device: %s", volOps.VolID, devicePath) + log.DebugLog(ctx, "rbd volID: %s was successfully attached to device: %s", volOps.VolID, devicePath) return nil } @@ -175,7 +176,7 @@ func (ns *NodeServer) NodeStageVolume( // MULTI_NODE_MULTI_WRITER is supported by default for Block access type volumes if req.VolumeCapability.AccessMode.Mode == csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER { if !isBlock { - util.WarningLog( + log.WarningLog( ctx, "MULTI_NODE_MULTI_WRITER currently only supported with volumes of access type `block`,"+ "invalid AccessMode for volume: %v", @@ -200,7 +201,7 @@ func (ns *NodeServer) NodeStageVolume( defer cr.DeleteCredentials() if acquired := ns.VolumeLocks.TryAcquire(volID); !acquired { - util.ErrorLog(ctx, util.VolumeOperationAlreadyExistsFmt, volID) + log.ErrorLog(ctx, util.VolumeOperationAlreadyExistsFmt, volID) return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, volID) } @@ -217,7 +218,7 @@ func (ns *NodeServer) NodeStageVolume( if err != nil { return nil, status.Error(codes.Internal, err.Error()) } else if !isNotMnt { - util.DebugLog(ctx, "rbd: volume %s is already mounted to %s, skipping", volID, stagingTargetPath) + log.DebugLog(ctx, "rbd: volume %s is already mounted to %s, skipping", volID, stagingTargetPath) return &csi.NodeStageVolumeResponse{}, nil } @@ -254,7 +255,7 @@ func (ns *NodeServer) NodeStageVolume( j, connErr := volJournal.Connect(volOptions.Monitors, volOptions.RadosNamespace, cr) if connErr != nil { - util.ErrorLog(ctx, "failed to establish cluster connection: %v", connErr) + log.ErrorLog(ctx, "failed to establish cluster connection: %v", connErr) return nil, status.Error(codes.Internal, connErr.Error()) } @@ -281,7 +282,7 @@ func (ns *NodeServer) NodeStageVolume( err = volOptions.Connect(cr) if err != nil { - util.ErrorLog(ctx, "failed to connect to volume %s: %v", volOptions, err) + log.ErrorLog(ctx, "failed to connect to volume %s: %v", volOptions, err) return nil, status.Error(codes.Internal, err.Error()) } @@ -316,7 +317,7 @@ func (ns *NodeServer) NodeStageVolume( return nil, status.Error(codes.Internal, err.Error()) } - util.DebugLog( + log.DebugLog( ctx, "rbd: successfully mounted volume %s to stagingTargetPath %s", volID, @@ -346,7 +347,7 @@ func (ns *NodeServer) stageTransaction( // Allow image to be mounted on multiple nodes if it is ROX if req.VolumeCapability.AccessMode.Mode == csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY { - util.ExtendedLog(ctx, "setting disableInUseChecks on rbd volume to: %v", req.GetVolumeId) + log.ExtendedLog(ctx, "setting disableInUseChecks on rbd volume to: %v", req.GetVolumeId) volOptions.DisableInUseChecks = true volOptions.readOnly = true } @@ -382,7 +383,7 @@ func (ns *NodeServer) stageTransaction( } transaction.devicePath = devicePath - util.DebugLog(ctx, "rbd image: %s/%s was successfully mapped at %s\n", + log.DebugLog(ctx, "rbd image: %s/%s was successfully mapped at %s\n", req.GetVolumeId(), volOptions.Pool, devicePath) // userspace mounters like nbd need the device path as a reference while @@ -440,7 +441,7 @@ func (ns *NodeServer) undoStagingTransaction( if transaction.isMounted { err = ns.mounter.Unmount(stagingTargetPath) if err != nil { - util.ErrorLog(ctx, "failed to unmount stagingtargetPath: %s with error: %v", stagingTargetPath, err) + log.ErrorLog(ctx, "failed to unmount stagingtargetPath: %s with error: %v", stagingTargetPath, err) return } @@ -450,7 +451,7 @@ func (ns *NodeServer) undoStagingTransaction( if transaction.isStagePathCreated { err = os.Remove(stagingTargetPath) if err != nil { - util.ErrorLog(ctx, "failed to remove stagingtargetPath: %s with error: %v", stagingTargetPath, err) + log.ErrorLog(ctx, "failed to remove stagingtargetPath: %s with error: %v", stagingTargetPath, err) // continue on failure to unmap the image, as leaving stale images causes more issues than a stale // file/directory } @@ -462,7 +463,7 @@ func (ns *NodeServer) undoStagingTransaction( if transaction.devicePath != "" { err = detachRBDDevice(ctx, transaction.devicePath, volID, volOptions.UnmapOptions, transaction.isEncrypted) if err != nil { - util.ErrorLog( + log.ErrorLog( ctx, "failed to unmap rbd device: %s for volume %s with error: %v", transaction.devicePath, @@ -475,7 +476,7 @@ func (ns *NodeServer) undoStagingTransaction( // Cleanup the stashed image metadata if err = cleanupRBDImageMetadataStash(req.GetStagingTargetPath()); err != nil { - util.ErrorLog(ctx, "failed to cleanup image metadata stash (%v)", err) + log.ErrorLog(ctx, "failed to cleanup image metadata stash (%v)", err) return } @@ -486,12 +487,12 @@ func (ns *NodeServer) createStageMountPoint(ctx context.Context, mountPath strin // #nosec:G304, intentionally creating file mountPath, not a security issue pathFile, err := os.OpenFile(mountPath, os.O_CREATE|os.O_RDWR, 0o600) if err != nil { - util.ErrorLog(ctx, "failed to create mountPath:%s with error: %v", mountPath, err) + log.ErrorLog(ctx, "failed to create mountPath:%s with error: %v", mountPath, err) return status.Error(codes.Internal, err.Error()) } if err = pathFile.Close(); err != nil { - util.ErrorLog(ctx, "failed to close mountPath:%s with error: %v", mountPath, err) + log.ErrorLog(ctx, "failed to close mountPath:%s with error: %v", mountPath, err) return status.Error(codes.Internal, err.Error()) } @@ -502,7 +503,7 @@ func (ns *NodeServer) createStageMountPoint(ctx context.Context, mountPath strin err := os.Mkdir(mountPath, 0o750) if err != nil { if !os.IsExist(err) { - util.ErrorLog(ctx, "failed to create mountPath:%s with error: %v", mountPath, err) + log.ErrorLog(ctx, "failed to create mountPath:%s with error: %v", mountPath, err) return status.Error(codes.Internal, err.Error()) } @@ -545,7 +546,7 @@ func (ns *NodeServer) NodePublishVolume( return nil, err } - util.DebugLog(ctx, "rbd: successfully mounted stagingPath %s to targetPath %s", stagingPath, targetPath) + log.DebugLog(ctx, "rbd: successfully mounted stagingPath %s to targetPath %s", stagingPath, targetPath) return &csi.NodePublishVolumeResponse{}, nil } @@ -570,7 +571,7 @@ func (ns *NodeServer) mountVolumeToStagePath( // the first time). existingFormat, err := diskMounter.GetDiskFormat(devicePath) if err != nil { - util.ErrorLog(ctx, "failed to get disk format for path %s, error: %v", devicePath, err) + log.ErrorLog(ctx, "failed to get disk format for path %s, error: %v", devicePath, err) return readOnly, err } @@ -610,7 +611,7 @@ func (ns *NodeServer) mountVolumeToStagePath( if len(args) > 0 { cmdOut, cmdErr := diskMounter.Exec.Command("mkfs."+fsType, args...).CombinedOutput() if cmdErr != nil { - util.ErrorLog(ctx, "failed to run mkfs error: %v, output: %v", cmdErr, string(cmdOut)) + log.ErrorLog(ctx, "failed to run mkfs error: %v, output: %v", cmdErr, string(cmdOut)) return readOnly, cmdErr } @@ -624,7 +625,7 @@ func (ns *NodeServer) mountVolumeToStagePath( err = diskMounter.FormatAndMount(devicePath, stagingPath, fsType, opt) } if err != nil { - util.ErrorLog(ctx, + log.ErrorLog(ctx, "failed to mount device path (%s) to staging path (%s) for volume "+ "(%s) error: %s Check dmesg logs if required.", devicePath, @@ -646,7 +647,7 @@ func (ns *NodeServer) mountVolume(ctx context.Context, stagingPath string, req * mountOptions = csicommon.ConstructMountOptions(mountOptions, req.GetVolumeCapability()) - util.DebugLog(ctx, "target %v\nisBlock %v\nfstype %v\nstagingPath %v\nreadonly %v\nmountflags %v\n", + log.DebugLog(ctx, "target %v\nisBlock %v\nfstype %v\nstagingPath %v\nreadonly %v\nmountflags %v\n", targetPath, isBlock, fsType, stagingPath, readOnly, mountOptions) if readOnly { @@ -672,12 +673,12 @@ func (ns *NodeServer) createTargetMountPath(ctx context.Context, mountPath strin // #nosec pathFile, e := os.OpenFile(mountPath, os.O_CREATE|os.O_RDWR, 0o750) if e != nil { - util.DebugLog(ctx, "Failed to create mountPath:%s with error: %v", mountPath, err) + log.DebugLog(ctx, "Failed to create mountPath:%s with error: %v", mountPath, err) return notMnt, status.Error(codes.Internal, e.Error()) } if err = pathFile.Close(); err != nil { - util.DebugLog(ctx, "Failed to close mountPath:%s with error: %v", mountPath, err) + log.DebugLog(ctx, "Failed to close mountPath:%s with error: %v", mountPath, err) return notMnt, status.Error(codes.Internal, err.Error()) } @@ -708,7 +709,7 @@ func (ns *NodeServer) NodeUnpublishVolume( if err != nil { if os.IsNotExist(err) { // targetPath has already been deleted - util.DebugLog(ctx, "targetPath: %s has already been deleted", targetPath) + log.DebugLog(ctx, "targetPath: %s has already been deleted", targetPath) return &csi.NodeUnpublishVolumeResponse{}, nil } @@ -731,7 +732,7 @@ func (ns *NodeServer) NodeUnpublishVolume( return nil, status.Error(codes.Internal, err.Error()) } - util.DebugLog(ctx, "rbd: successfully unbound volume %s from %s", req.GetVolumeId(), targetPath) + log.DebugLog(ctx, "rbd: successfully unbound volume %s from %s", req.GetVolumeId(), targetPath) return &csi.NodeUnpublishVolumeResponse{}, nil } @@ -761,7 +762,7 @@ func (ns *NodeServer) NodeUnstageVolume( volID := req.GetVolumeId() if acquired := ns.VolumeLocks.TryAcquire(volID); !acquired { - util.ErrorLog(ctx, util.VolumeOperationAlreadyExistsFmt, volID) + log.ErrorLog(ctx, util.VolumeOperationAlreadyExistsFmt, volID) return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, volID) } @@ -782,11 +783,11 @@ func (ns *NodeServer) NodeUnstageVolume( // Unmounting the image err = ns.mounter.Unmount(stagingTargetPath) if err != nil { - util.ExtendedLog(ctx, "failed to unmount targetPath: %s with error: %v", stagingTargetPath, err) + log.ExtendedLog(ctx, "failed to unmount targetPath: %s with error: %v", stagingTargetPath, err) return nil, status.Error(codes.Internal, err.Error()) } - util.DebugLog(ctx, "successfully unmounted volume (%s) from staging path (%s)", + log.DebugLog(ctx, "successfully unmounted volume (%s) from staging path (%s)", req.GetVolumeId(), stagingTargetPath) } @@ -795,7 +796,7 @@ func (ns *NodeServer) NodeUnstageVolume( // keeps invoking Unstage. Hence any errors removing files within this path is a critical // error if !os.IsNotExist(err) { - util.ErrorLog(ctx, "failed to remove staging target path (%s): (%v)", stagingTargetPath, err) + log.ErrorLog(ctx, "failed to remove staging target path (%s): (%v)", stagingTargetPath, err) return nil, status.Error(codes.Internal, err.Error()) } @@ -803,7 +804,7 @@ func (ns *NodeServer) NodeUnstageVolume( imgInfo, err := lookupRBDImageMetadataStash(stagingParentPath) if err != nil { - util.UsefulLog(ctx, "failed to find image metadata: %v", err) + log.UsefulLog(ctx, "failed to find image metadata: %v", err) // It is an error if it was mounted, as we should have found the image metadata file with // no errors if !notMnt { @@ -833,7 +834,7 @@ func (ns *NodeServer) NodeUnstageVolume( logDir: imgInfo.LogDir, } if err = detachRBDImageOrDeviceSpec(ctx, dArgs); err != nil { - util.ErrorLog( + log.ErrorLog( ctx, "error unmapping volume (%s) from staging path (%s): (%v)", req.GetVolumeId(), @@ -843,10 +844,10 @@ func (ns *NodeServer) NodeUnstageVolume( return nil, status.Error(codes.Internal, err.Error()) } - util.DebugLog(ctx, "successfully unmapped volume (%s)", req.GetVolumeId()) + log.DebugLog(ctx, "successfully unmapped volume (%s)", req.GetVolumeId()) if err = cleanupRBDImageMetadataStash(stagingParentPath); err != nil { - util.ErrorLog(ctx, "failed to cleanup image metadata stash (%v)", err) + log.ErrorLog(ctx, "failed to cleanup image metadata stash (%v)", err) return nil, status.Error(codes.Internal, err.Error()) } @@ -877,7 +878,7 @@ func (ns *NodeServer) NodeExpandVolume( } if acquired := ns.VolumeLocks.TryAcquire(volumeID); !acquired { - util.ErrorLog(ctx, util.VolumeOperationAlreadyExistsFmt, volumeID) + log.ErrorLog(ctx, util.VolumeOperationAlreadyExistsFmt, volumeID) return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, volumeID) } @@ -885,7 +886,7 @@ func (ns *NodeServer) NodeExpandVolume( imgInfo, err := lookupRBDImageMetadataStash(volumePath) if err != nil { - util.ErrorLog(ctx, "failed to find image metadata: %v", err) + log.ErrorLog(ctx, "failed to find image metadata: %v", err) } devicePath, found := findDeviceMappingImage( ctx, @@ -903,7 +904,7 @@ func (ns *NodeServer) NodeExpandVolume( // The volume is encrypted, resize an active mapping err = util.ResizeEncryptedVolume(ctx, mapperFile) if err != nil { - util.ErrorLog(ctx, "failed to resize device %s, mapper %s: %w", + log.ErrorLog(ctx, "failed to resize device %s, mapper %s: %w", devicePath, mapperFile, err) return nil, status.Errorf(codes.Internal, @@ -966,7 +967,7 @@ func (ns *NodeServer) processEncryptedDevice( imageSpec := volOptions.String() encrypted, err := volOptions.checkRbdImageEncrypted(ctx) if err != nil { - util.ErrorLog(ctx, "failed to get encryption status for rbd image %s: %v", + log.ErrorLog(ctx, "failed to get encryption status for rbd image %s: %v", imageSpec, err) return "", err @@ -982,7 +983,7 @@ func (ns *NodeServer) processEncryptedDevice( // continue with the common process to crypt-format the device. err = volOptions.setupEncryption(ctx) if err != nil { - util.ErrorLog(ctx, "failed to setup encryption for rbd"+ + log.ErrorLog(ctx, "failed to setup encryption for rbd"+ "image %s: %v", imageSpec, err) return "", err @@ -1006,7 +1007,7 @@ func (ns *NodeServer) processEncryptedDevice( return "", fmt.Errorf("failed to encrypt rbd image %s: %w", imageSpec, err) } case "crypt", "crypto_LUKS": - util.WarningLog(ctx, "rbd image %s is encrypted, but encryption state was not updated", + log.WarningLog(ctx, "rbd image %s is encrypted, but encryption state was not updated", imageSpec) err = volOptions.ensureEncryptionMetadataSet(rbdImageEncrypted) if err != nil { @@ -1092,7 +1093,7 @@ func blockNodeGetVolumeStats(ctx context.Context, targetPath string) (*csi.NodeG lsblkSize, _, err := util.ExecCommand(ctx, "/bin/lsblk", args...) if err != nil { err = fmt.Errorf("lsblk %v returned an error: %w", args, err) - util.ErrorLog(ctx, err.Error()) + log.ErrorLog(ctx, err.Error()) return nil, status.Error(codes.Internal, err.Error()) } @@ -1100,7 +1101,7 @@ func blockNodeGetVolumeStats(ctx context.Context, targetPath string) (*csi.NodeG size, err := strconv.ParseInt(strings.TrimSpace(lsblkSize), 10, 64) if err != nil { err = fmt.Errorf("failed to convert %q to bytes: %w", lsblkSize, err) - util.ErrorLog(ctx, err.Error()) + log.ErrorLog(ctx, err.Error()) return nil, status.Error(codes.Internal, err.Error()) } diff --git a/internal/rbd/rbd_attach.go b/internal/rbd/rbd_attach.go index 42dcc0201..d5b4c6062 100644 --- a/internal/rbd/rbd_attach.go +++ b/internal/rbd/rbd_attach.go @@ -27,6 +27,7 @@ import ( "time" "github.com/ceph/ceph-csi/internal/util" + "github.com/ceph/ceph-csi/internal/util/log" "k8s.io/apimachinery/pkg/util/wait" ) @@ -161,7 +162,7 @@ func findDeviceMappingImage(ctx context.Context, pool, namespace, image string, rbdDeviceList, err := rbdGetDeviceList(ctx, accessType) if err != nil { - util.WarningLog(ctx, "failed to determine if image (%s) is mapped to a device (%v)", imageSpec, err) + log.WarningLog(ctx, "failed to determine if image (%s) is mapped to a device (%v)", imageSpec, err) return "", false } @@ -199,17 +200,17 @@ func checkRbdNbdTools() bool { // try to load the module _, _, err = util.ExecCommand(context.TODO(), "modprobe", moduleNbd) if err != nil { - util.ExtendedLogMsg("rbd-nbd: nbd modprobe failed with error %v", err) + log.ExtendedLogMsg("rbd-nbd: nbd modprobe failed with error %v", err) return false } } if _, _, err := util.ExecCommand(context.TODO(), rbdTonbd, "--version"); err != nil { - util.ExtendedLogMsg("rbd-nbd: running rbd-nbd --version failed with error %v", err) + log.ExtendedLogMsg("rbd-nbd: running rbd-nbd --version failed with error %v", err) return false } - util.ExtendedLogMsg("rbd-nbd tools were found.") + log.ExtendedLogMsg("rbd-nbd tools were found.") return true } @@ -305,7 +306,7 @@ func createPath(ctx context.Context, volOpt *rbdVolume, device string, cr *util. isNbd := false imagePath := volOpt.String() - util.TraceLog(ctx, "rbd: map mon %s", volOpt.Monitors) + log.TraceLog(ctx, "rbd: map mon %s", volOpt.Monitors) mapArgs := []string{ "--id", cr.ID, @@ -321,7 +322,7 @@ func createPath(ctx context.Context, volOpt *rbdVolume, device string, cr *util. // check if the image should stay thick-provisioned isThick, err := volOpt.isThickProvisioned() if err != nil { - util.WarningLog(ctx, "failed to detect if image %q is thick-provisioned: %v", volOpt, err) + log.WarningLog(ctx, "failed to detect if image %q is thick-provisioned: %v", volOpt, err) } if isNbd { @@ -347,7 +348,7 @@ func createPath(ctx context.Context, volOpt *rbdVolume, device string, cr *util. // Execute map stdout, stderr, err := util.ExecCommand(ctx, cli, mapArgs...) if err != nil { - util.WarningLog(ctx, "rbd: map error %v, rbd output: %s", err, stderr) + log.WarningLog(ctx, "rbd: map error %v, rbd output: %s", err, stderr) // unmap rbd image if connection timeout if strings.Contains(err.Error(), rbdMapConnectionTimeout) { dArgs := detachRBDImageArgs{ @@ -361,7 +362,7 @@ func createPath(ctx context.Context, volOpt *rbdVolume, device string, cr *util. } detErr := detachRBDImageOrDeviceSpec(ctx, dArgs) if detErr != nil { - util.WarningLog(ctx, "rbd: %s unmap error %v", imagePath, detErr) + log.WarningLog(ctx, "rbd: %s unmap error %v", imagePath, detErr) } } @@ -381,7 +382,7 @@ func waitForrbdImage(ctx context.Context, backoff wait.Backoff, volOptions *rbdV return false, fmt.Errorf("fail to check rbd image status: (%w)", err) } if (volOptions.DisableInUseChecks) && (used) { - util.UsefulLog(ctx, "valid multi-node attach requested, ignoring watcher in-use result") + log.UsefulLog(ctx, "valid multi-node attach requested, ignoring watcher in-use result") return used, nil } @@ -423,7 +424,7 @@ func detachRBDImageOrDeviceSpec( mapperFile, mapperPath := util.VolumeMapper(dArgs.volumeID) mappedDevice, mapper, err := util.DeviceEncryptionStatus(ctx, mapperPath) if err != nil { - util.ErrorLog(ctx, "error determining LUKS device on %s, %s: %s", + log.ErrorLog(ctx, "error determining LUKS device on %s, %s: %s", mapperPath, dArgs.imageOrDeviceSpec, err) return err @@ -432,7 +433,7 @@ func detachRBDImageOrDeviceSpec( // mapper found, so it is open Luks device err = util.CloseEncryptedVolume(ctx, mapperFile) if err != nil { - util.ErrorLog(ctx, "error closing LUKS device on %s, %s: %s", + log.ErrorLog(ctx, "error closing LUKS device on %s, %s: %s", mapperPath, dArgs.imageOrDeviceSpec, err) return err @@ -452,7 +453,7 @@ func detachRBDImageOrDeviceSpec( (strings.Contains(stderr, fmt.Sprintf(rbdUnmapCmdkRbdMissingMap, dArgs.imageOrDeviceSpec)) || strings.Contains(stderr, fmt.Sprintf(rbdUnmapCmdNbdMissingMap, dArgs.imageOrDeviceSpec))) { // Devices found not to be mapped are treated as a successful detach - util.TraceLog(ctx, "image or device spec (%s) not mapped", dArgs.imageOrDeviceSpec) + log.TraceLog(ctx, "image or device spec (%s) not mapped", dArgs.imageOrDeviceSpec) return nil } @@ -462,7 +463,7 @@ func detachRBDImageOrDeviceSpec( if dArgs.isNbd && dArgs.logDir != "" { logFile := getCephClientLogFileName(dArgs.volumeID, dArgs.logDir, "rbd-nbd") if err = os.Remove(logFile); err != nil { - util.WarningLog(ctx, "failed to remove logfile: %s, error: %v", + log.WarningLog(ctx, "failed to remove logfile: %s, error: %v", logFile, err) } } diff --git a/internal/rbd/rbd_healer.go b/internal/rbd/rbd_healer.go index 73d5677dc..05cb941a4 100644 --- a/internal/rbd/rbd_healer.go +++ b/internal/rbd/rbd_healer.go @@ -21,6 +21,7 @@ import ( "sync" "github.com/ceph/ceph-csi/internal/util" + "github.com/ceph/ceph-csi/internal/util/log" "github.com/container-storage-interface/spec/lib/go/csi" v1 "k8s.io/api/core/v1" @@ -56,7 +57,7 @@ func getSecret(c *k8s.Clientset, ns, name string) (map[string]string, error) { secret, err := c.CoreV1().Secrets(ns).Get(context.TODO(), name, metav1.GetOptions{}) if err != nil { - util.ErrorLogMsg("get secret failed, err: %v", err) + log.ErrorLogMsg("get secret failed, err: %v", err) return nil, err } @@ -74,14 +75,14 @@ func callNodeStageVolume(ns *NodeServer, c *k8s.Clientset, pv *v1.PersistentVolu volID := pv.Spec.PersistentVolumeSource.CSI.VolumeHandle stagingParentPath := stagingPath + pv.Name + "/globalmount" - util.DefaultLog("sending nodeStageVolume for volID: %s, stagingPath: %s", + log.DefaultLog("sending nodeStageVolume for volID: %s, stagingPath: %s", volID, stagingParentPath) deviceSecret, err := getSecret(c, pv.Spec.PersistentVolumeSource.CSI.NodeStageSecretRef.Namespace, pv.Spec.PersistentVolumeSource.CSI.NodeStageSecretRef.Name) if err != nil { - util.ErrorLogMsg("getSecret failed for volID: %s, err: %v", volID, err) + log.ErrorLogMsg("getSecret failed for volID: %s, err: %v", volID, err) return err } @@ -116,7 +117,7 @@ func callNodeStageVolume(ns *NodeServer, c *k8s.Clientset, pv *v1.PersistentVolu _, err = ns.NodeStageVolume(context.TODO(), req) if err != nil { - util.ErrorLogMsg("nodeStageVolume request failed, volID: %s, stagingPath: %s, err: %v", + log.ErrorLogMsg("nodeStageVolume request failed, volID: %s, stagingPath: %s, err: %v", volID, stagingParentPath, err) return err @@ -130,7 +131,7 @@ func runVolumeHealer(ns *NodeServer, conf *util.Config) error { c := util.NewK8sClient() val, err := c.StorageV1().VolumeAttachments().List(context.TODO(), metav1.ListOptions{}) if err != nil { - util.ErrorLogMsg("list volumeAttachments failed, err: %v", err) + log.ErrorLogMsg("list volumeAttachments failed, err: %v", err) return err } @@ -147,7 +148,7 @@ func runVolumeHealer(ns *NodeServer, conf *util.Config) error { if err != nil { // skip if volume doesn't exist if !apierrors.IsNotFound(err) { - util.ErrorLogMsg("get persistentVolumes failed for pv: %s, err: %v", pvName, err) + log.ErrorLogMsg("get persistentVolumes failed for pv: %s, err: %v", pvName, err) } continue @@ -167,7 +168,7 @@ func runVolumeHealer(ns *NodeServer, conf *util.Config) error { if err != nil { // skip if volume attachment doesn't exist if !apierrors.IsNotFound(err) { - util.ErrorLogMsg("get volumeAttachments failed for volumeAttachment: %s, volID: %s, err: %v", + log.ErrorLogMsg("get volumeAttachments failed for volumeAttachment: %s, volID: %s, err: %v", val.Items[i].Name, pv.Spec.PersistentVolumeSource.CSI.VolumeHandle, err) } @@ -192,7 +193,7 @@ func runVolumeHealer(ns *NodeServer, conf *util.Config) error { for s := range channel { if s != nil { - util.ErrorLogMsg("callNodeStageVolume failed, err: %v", s) + log.ErrorLogMsg("callNodeStageVolume failed, err: %v", s) } } diff --git a/internal/rbd/rbd_journal.go b/internal/rbd/rbd_journal.go index 6987723ad..7408b7bdf 100644 --- a/internal/rbd/rbd_journal.go +++ b/internal/rbd/rbd_journal.go @@ -23,6 +23,7 @@ import ( "github.com/ceph/ceph-csi/internal/journal" "github.com/ceph/ceph-csi/internal/util" + "github.com/ceph/ceph-csi/internal/util/log" ) const ( @@ -167,7 +168,7 @@ func checkSnapCloneExists( err = parentVol.deleteSnapshot(ctx, rbdSnap) if err != nil { if !errors.Is(err, ErrSnapNotFound) { - util.ErrorLog(ctx, "failed to delete snapshot %s: %v", rbdSnap, err) + log.ErrorLog(ctx, "failed to delete snapshot %s: %v", rbdSnap, err) return false, err } @@ -198,7 +199,7 @@ func checkSnapCloneExists( // create snapshot sErr := vol.createSnapshot(ctx, rbdSnap) if sErr != nil { - util.ErrorLog(ctx, "failed to create snapshot %s: %v", rbdSnap, sErr) + log.ErrorLog(ctx, "failed to create snapshot %s: %v", rbdSnap, sErr) err = undoSnapshotCloning(ctx, parentVol, rbdSnap, vol, cr) return false, err @@ -211,21 +212,21 @@ func checkSnapCloneExists( if vol.ImageID == "" { sErr := vol.getImageID() if sErr != nil { - util.ErrorLog(ctx, "failed to get image id %s: %v", vol, sErr) + log.ErrorLog(ctx, "failed to get image id %s: %v", vol, sErr) err = undoSnapshotCloning(ctx, parentVol, rbdSnap, vol, cr) return false, err } sErr = j.StoreImageID(ctx, vol.JournalPool, vol.ReservedID, vol.ImageID) if sErr != nil { - util.ErrorLog(ctx, "failed to store volume id %s: %v", vol, sErr) + log.ErrorLog(ctx, "failed to store volume id %s: %v", vol, sErr) err = undoSnapshotCloning(ctx, parentVol, rbdSnap, vol, cr) return false, err } } - util.DebugLog(ctx, "found existing image (%s) with name (%s) for request (%s)", + log.DebugLog(ctx, "found existing image (%s) with name (%s) for request (%s)", rbdSnap.VolID, rbdSnap.RbdSnapName, rbdSnap.RequestName) return true, nil @@ -335,13 +336,13 @@ func (rv *rbdVolume) Exists(ctx context.Context, parentVol *rbdVolume) (bool, er if parentVol != nil && parentVol.isEncrypted() { err = parentVol.copyEncryptionConfig(&rv.rbdImage) if err != nil { - util.ErrorLog(ctx, err.Error()) + log.ErrorLog(ctx, err.Error()) return false, err } } - util.DebugLog(ctx, "found existing volume (%s) with image name (%s) for request (%s)", + log.DebugLog(ctx, "found existing volume (%s) with image name (%s) for request (%s)", rv.VolID, rv.RbdImageName, rv.RequestName) return true, nil @@ -357,13 +358,13 @@ func (rv *rbdVolume) repairImageID(ctx context.Context, j *journal.Connection) e err := rv.getImageID() if err != nil { - util.ErrorLog(ctx, "failed to get image id %s: %v", rv, err) + log.ErrorLog(ctx, "failed to get image id %s: %v", rv, err) return err } err = j.StoreImageID(ctx, rv.JournalPool, rv.ReservedID, rv.ImageID) if err != nil { - util.ErrorLog(ctx, "failed to store volume id %s: %v", rv, err) + log.ErrorLog(ctx, "failed to store volume id %s: %v", rv, err) return err } @@ -405,7 +406,7 @@ func reserveSnap(ctx context.Context, rbdSnap *rbdSnapshot, rbdVol *rbdVolume, c return err } - util.DebugLog(ctx, "generated Volume ID (%s) and image name (%s) for request name (%s)", + log.DebugLog(ctx, "generated Volume ID (%s) and image name (%s) for request name (%s)", rbdSnap.VolID, rbdSnap.RbdSnapName, rbdSnap.RequestName) return nil @@ -481,7 +482,7 @@ func reserveVol(ctx context.Context, rbdVol *rbdVolume, rbdSnap *rbdSnapshot, cr return err } - util.DebugLog(ctx, "generated Volume ID (%s) and image name (%s) for request name (%s)", + log.DebugLog(ctx, "generated Volume ID (%s) and image name (%s) for request name (%s)", rbdVol.VolID, rbdVol.RbdImageName, rbdVol.RequestName) return nil @@ -566,7 +567,7 @@ func RegenerateJournal( rbdVol.Monitors, _, err = util.GetMonsAndClusterID(options) if err != nil { - util.ErrorLog(ctx, "failed getting mons (%s)", err) + log.ErrorLog(ctx, "failed getting mons (%s)", err) return "", err } @@ -634,7 +635,7 @@ func RegenerateJournal( undoErr := j.UndoReservation(ctx, rbdVol.JournalPool, rbdVol.Pool, rbdVol.RbdImageName, rbdVol.RequestName) if undoErr != nil { - util.ErrorLog(ctx, "failed to undo reservation %s: %v", rbdVol, undoErr) + log.ErrorLog(ctx, "failed to undo reservation %s: %v", rbdVol, undoErr) } } }() @@ -644,7 +645,7 @@ func RegenerateJournal( return "", err } - util.DebugLog(ctx, "re-generated Volume ID (%s) and image name (%s) for request name (%s)", + log.DebugLog(ctx, "re-generated Volume ID (%s) and image name (%s) for request name (%s)", rbdVol.VolID, rbdVol.RbdImageName, rbdVol.RequestName) if rbdVol.ImageID == "" { err = rbdVol.storeImageID(ctx, j) @@ -660,13 +661,13 @@ func RegenerateJournal( func (rv *rbdVolume) storeImageID(ctx context.Context, j *journal.Connection) error { err := rv.getImageID() if err != nil { - util.ErrorLog(ctx, "failed to get image id %s: %v", rv, err) + log.ErrorLog(ctx, "failed to get image id %s: %v", rv, err) return err } err = j.StoreImageID(ctx, rv.JournalPool, rv.ReservedID, rv.ImageID) if err != nil { - util.ErrorLog(ctx, "failed to store volume id %s: %v", rv, err) + log.ErrorLog(ctx, "failed to store volume id %s: %v", rv, err) return err } diff --git a/internal/rbd/rbd_util.go b/internal/rbd/rbd_util.go index ad5727629..e02af3446 100644 --- a/internal/rbd/rbd_util.go +++ b/internal/rbd/rbd_util.go @@ -29,6 +29,7 @@ import ( "time" "github.com/ceph/ceph-csi/internal/util" + "github.com/ceph/ceph-csi/internal/util/log" "github.com/ceph/go-ceph/rados" librbd "github.com/ceph/go-ceph/rbd" @@ -241,7 +242,7 @@ func createImage(ctx context.Context, pOpts *rbdVolume, cr *util.Credentials) er return fmt.Errorf("failed to set data pool: %w", err) } } - util.DebugLog(ctx, logMsg, + log.DebugLog(ctx, logMsg, pOpts, volSzMiB, pOpts.imageFeatureSet.Names(), pOpts.Monitors) if pOpts.imageFeatureSet != 0 { @@ -492,7 +493,7 @@ func isNotMountPoint(mounter mount.Interface, stagingTargetPath string) (bool, e func addRbdManagerTask(ctx context.Context, pOpts *rbdVolume, arg []string) (bool, error) { args := []string{"rbd", "task", "add"} args = append(args, arg...) - util.DebugLog( + log.DebugLog( ctx, "executing %v for image (%s) using mon %s, pool %s", args, @@ -505,17 +506,17 @@ func addRbdManagerTask(ctx context.Context, pOpts *rbdVolume, arg []string) (boo switch { case strings.Contains(stderr, rbdTaskRemoveCmdInvalidString1) && strings.Contains(stderr, rbdTaskRemoveCmdInvalidString2): - util.WarningLog( + log.WarningLog( ctx, "cluster with cluster ID (%s) does not support Ceph manager based rbd commands"+ "(minimum ceph version required is v14.2.3)", pOpts.ClusterID) supported = false case strings.HasPrefix(stderr, rbdTaskRemoveCmdAccessDeniedMessage): - util.WarningLog(ctx, "access denied to Ceph MGR-based rbd commands on cluster ID (%s)", pOpts.ClusterID) + log.WarningLog(ctx, "access denied to Ceph MGR-based rbd commands on cluster ID (%s)", pOpts.ClusterID) supported = false default: - util.WarningLog(ctx, "uncaught error while scheduling a task (%v): %s", err, stderr) + log.WarningLog(ctx, "uncaught error while scheduling a task (%v): %s", err, stderr) } } if err != nil { @@ -539,7 +540,7 @@ func (rv *rbdVolume) getTrashPath() string { func deleteImage(ctx context.Context, pOpts *rbdVolume, cr *util.Credentials) error { image := pOpts.RbdImageName - util.DebugLog(ctx, "rbd: delete %s using mon %s, pool %s", image, pOpts.Monitors, pOpts.Pool) + log.DebugLog(ctx, "rbd: delete %s using mon %s, pool %s", image, pOpts.Monitors, pOpts.Pool) // Support deleting the older rbd images whose imageID is not stored in omap err := pOpts.getImageID() @@ -548,9 +549,9 @@ func deleteImage(ctx context.Context, pOpts *rbdVolume, cr *util.Credentials) er } if pOpts.isEncrypted() { - util.DebugLog(ctx, "rbd: going to remove DEK for %q", pOpts) + log.DebugLog(ctx, "rbd: going to remove DEK for %q", pOpts) if err = pOpts.encryption.RemoveDEK(pOpts.VolID); err != nil { - util.WarningLog(ctx, "failed to clean the passphrase for volume %s: %s", pOpts.VolID, err) + log.WarningLog(ctx, "failed to clean the passphrase for volume %s: %s", pOpts.VolID, err) } } @@ -562,7 +563,7 @@ func deleteImage(ctx context.Context, pOpts *rbdVolume, cr *util.Credentials) er rbdImage := librbd.GetImage(pOpts.ioctx, image) err = rbdImage.Trash(0) if err != nil { - util.ErrorLog(ctx, "failed to delete rbd image: %s, error: %v", pOpts, err) + log.ErrorLog(ctx, "failed to delete rbd image: %s, error: %v", pOpts, err) return err } @@ -578,7 +579,7 @@ func deleteImage(ctx context.Context, pOpts *rbdVolume, cr *util.Credentials) er } rbdCephMgrSupported, err := addRbdManagerTask(ctx, pOpts, args) if rbdCephMgrSupported && err != nil { - util.ErrorLog(ctx, "failed to add task to delete rbd image: %s, %v", pOpts, err) + log.ErrorLog(ctx, "failed to add task to delete rbd image: %s, %v", pOpts, err) return err } @@ -586,7 +587,7 @@ func deleteImage(ctx context.Context, pOpts *rbdVolume, cr *util.Credentials) er if !rbdCephMgrSupported { err = librbd.TrashRemove(pOpts.ioctx, pOpts.ImageID, true) if err != nil { - util.ErrorLog(ctx, "failed to delete rbd image: %s, %v", pOpts, err) + log.ErrorLog(ctx, "failed to delete rbd image: %s, %v", pOpts, err) return err } @@ -625,7 +626,7 @@ func (rv *rbdVolume) getCloneDepth(ctx context.Context) (uint, error) { if errors.Is(err, ErrImageNotFound) { return depth, nil } - util.ErrorLog(ctx, "failed to check depth on image %s: %s", &vol, err) + log.ErrorLog(ctx, "failed to check depth on image %s: %s", &vol, err) return depth, err } @@ -654,7 +655,7 @@ func flattenClonedRbdImages( defer rv.Destroy() err := rv.Connect(cr) if err != nil { - util.ErrorLog(ctx, "failed to open connection %s; err %v", rv, err) + log.ErrorLog(ctx, "failed to open connection %s; err %v", rv, err) return err } @@ -680,7 +681,7 @@ func flattenClonedRbdImages( rv.RbdImageName = snapName.origSnapName err = rv.flattenRbdImage(ctx, cr, true, rbdHardMaxCloneDepth, rbdSoftMaxCloneDepth) if err != nil { - util.ErrorLog(ctx, "failed to flatten %s; err %v", rv, err) + log.ErrorLog(ctx, "failed to flatten %s; err %v", rv, err) continue } @@ -703,7 +704,7 @@ func (rv *rbdVolume) flattenRbdImage( if err != nil { return err } - util.ExtendedLog( + log.ExtendedLog( ctx, "clone depth is (%d), configured softlimit (%d) and hardlimit (%d) for %s", depth, @@ -724,7 +725,7 @@ func (rv *rbdVolume) flattenRbdImage( if strings.Contains(err.Error(), rbdFlattenNoParent) { return nil } - util.ErrorLog(ctx, "failed to add task flatten for %s : %v", rv, err) + log.ErrorLog(ctx, "failed to add task flatten for %s : %v", rv, err) return err } @@ -733,7 +734,7 @@ func (rv *rbdVolume) flattenRbdImage( } } if !supported { - util.ErrorLog( + log.ErrorLog( ctx, "task manager does not support flatten,image will be flattened once hardlimit is reached: %v", err) @@ -744,7 +745,7 @@ func (rv *rbdVolume) flattenRbdImage( } err := rv.flatten() if err != nil { - util.ErrorLog(ctx, "rbd failed to flatten image %s %s: %v", rv.Pool, rv.RbdImageName, err) + log.ErrorLog(ctx, "rbd failed to flatten image %s %s: %v", rv.Pool, rv.RbdImageName, err) return err } @@ -826,7 +827,7 @@ func (rv *rbdVolume) checkImageChainHasFeature(ctx context.Context, feature uint if errors.Is(err, ErrImageNotFound) { return false, nil } - util.ErrorLog(ctx, "failed to get image info for %s: %s", vol.String(), err) + log.ErrorLog(ctx, "failed to get image info for %s: %s", vol.String(), err) return false, err } @@ -856,7 +857,7 @@ func genSnapFromSnapID( err := vi.DecomposeCSIID(rbdSnap.VolID) if err != nil { - util.ErrorLog(ctx, "error decoding snapshot ID (%s) (%s)", err, rbdSnap.VolID) + log.ErrorLog(ctx, "error decoding snapshot ID (%s) (%s)", err, rbdSnap.VolID) return err } @@ -866,7 +867,7 @@ func genSnapFromSnapID( rbdSnap.Monitors, _, err = util.GetMonsAndClusterID(options) if err != nil { - util.ErrorLog(ctx, "failed getting mons (%s)", err) + log.ErrorLog(ctx, "failed getting mons (%s)", err) return err } @@ -955,7 +956,7 @@ func generateVolumeFromVolumeID( rbdVol.Monitors, _, err = util.GetMonsAndClusterID(options) if err != nil { - util.ErrorLog(ctx, "failed getting mons (%s)", err) + log.ErrorLog(ctx, "failed getting mons (%s)", err) return rbdVol, err } @@ -1073,7 +1074,7 @@ func genVolFromVolID( for i := range pvlist.Items { if pvlist.Items[i].Spec.CSI != nil && pvlist.Items[i].Spec.CSI.VolumeHandle == volumeID { if v, ok := pvlist.Items[i].Annotations[PVVolumeHandleAnnotationKey]; ok { - util.UsefulLog(ctx, "found new volumeID %s for existing volumeID %s", v, volumeID) + log.UsefulLog(ctx, "found new volumeID %s for existing volumeID %s", v, volumeID) err = vi.DecomposeCSIID(v) if err != nil { return vol, fmt.Errorf("%w: error decoding volume ID (%s) (%s)", @@ -1108,7 +1109,7 @@ func generateVolumeFromMapping( continue } - util.DebugLog(ctx, + log.DebugLog(ctx, "found new clusterID mapping %s for existing clusterID %s", mappedClusterID, vi.ClusterID) @@ -1121,7 +1122,7 @@ func generateVolumeFromMapping( if mappedPoolID == "" { continue } - util.DebugLog(ctx, + log.DebugLog(ctx, "found new poolID mapping %s for existing pooID %s", mappedPoolID, poolID) @@ -1181,7 +1182,7 @@ func genVolFromVolumeOptions( rbdVol.Monitors, rbdVol.ClusterID, err = util.GetMonsAndClusterID(volOptions) if err != nil { - util.ErrorLog(ctx, "failed getting mons (%s)", err) + log.ErrorLog(ctx, "failed getting mons (%s)", err) return nil, err } @@ -1196,12 +1197,12 @@ func genVolFromVolumeOptions( // if no image features is provided, it results in empty string // which disable all RBD image features as we expected if err = rbdVol.validateImageFeatures(volOptions["imageFeatures"]); err != nil { - util.ErrorLog(ctx, "failed to validate image features %v", err) + log.ErrorLog(ctx, "failed to validate image features %v", err) return nil, err } - util.ExtendedLog( + log.ExtendedLog( ctx, "setting disableInUseChecks: %t image features: %v mounter: %s", disableInUseChecks, @@ -1258,7 +1259,7 @@ func genSnapFromOptions(ctx context.Context, rbdVol *rbdVolume, snapOptions map[ rbdSnap.Monitors, rbdSnap.ClusterID, err = util.GetMonsAndClusterID(snapOptions) if err != nil { - util.ErrorLog(ctx, "failed getting mons (%s)", err) + log.ErrorLog(ctx, "failed getting mons (%s)", err) return nil, err } @@ -1276,7 +1277,7 @@ func (rv *rbdVolume) hasSnapshotFeature() bool { } func (rv *rbdVolume) createSnapshot(ctx context.Context, pOpts *rbdSnapshot) error { - util.DebugLog(ctx, "rbd: snap create %s using mon %s", pOpts, pOpts.Monitors) + log.DebugLog(ctx, "rbd: snap create %s using mon %s", pOpts, pOpts.Monitors) image, err := rv.open() if err != nil { return err @@ -1289,7 +1290,7 @@ func (rv *rbdVolume) createSnapshot(ctx context.Context, pOpts *rbdSnapshot) err } func (rv *rbdVolume) deleteSnapshot(ctx context.Context, pOpts *rbdSnapshot) error { - util.DebugLog(ctx, "rbd: snap rm %s using mon %s", pOpts, pOpts.Monitors) + log.DebugLog(ctx, "rbd: snap rm %s using mon %s", pOpts, pOpts.Monitors) image, err := rv.open() if err != nil { return err @@ -1335,7 +1336,7 @@ func (rv *rbdVolume) cloneRbdImageFromSnapshot( } } - util.DebugLog(ctx, logMsg, + log.DebugLog(ctx, logMsg, pSnapOpts, rv, rv.imageFeatureSet.Names(), rv.Monitors) if rv.imageFeatureSet != 0 { @@ -1373,7 +1374,7 @@ func (rv *rbdVolume) cloneRbdImageFromSnapshot( if deleteClone { err = librbd.RemoveImage(rv.ioctx, rv.RbdImageName) if err != nil { - util.ErrorLog(ctx, "failed to delete temporary image %q: %v", rv, err) + log.ErrorLog(ctx, "failed to delete temporary image %q: %v", rv, err) } } }() diff --git a/internal/rbd/replicationcontrollerserver.go b/internal/rbd/replicationcontrollerserver.go index feb656cb4..a72348ae8 100644 --- a/internal/rbd/replicationcontrollerserver.go +++ b/internal/rbd/replicationcontrollerserver.go @@ -26,6 +26,7 @@ import ( "time" "github.com/ceph/ceph-csi/internal/util" + "github.com/ceph/ceph-csi/internal/util/log" librbd "github.com/ceph/go-ceph/rbd" "github.com/ceph/go-ceph/rbd/admin" @@ -91,7 +92,7 @@ type ReplicationServer struct { func getForceOption(ctx context.Context, parameters map[string]string) (bool, error) { val, ok := parameters[forceKey] if !ok { - util.WarningLog(ctx, "%s is not set in parameters, setting to default (%v)", forceKey, false) + log.WarningLog(ctx, "%s is not set in parameters, setting to default (%v)", forceKey, false) return false, nil } @@ -108,7 +109,7 @@ func getForceOption(ctx context.Context, parameters map[string]string) (bool, er func getMirroringMode(ctx context.Context, parameters map[string]string) (librbd.ImageMirrorMode, error) { val, ok := parameters[imageMirroringKey] if !ok { - util.WarningLog( + log.WarningLog( ctx, "%s is not set in parameters, setting to mirroringMode to default (%s)", imageMirroringKey, @@ -206,7 +207,7 @@ func (rs *ReplicationServer) EnableVolumeReplication(ctx context.Context, } if acquired := rs.VolumeLocks.TryAcquire(volumeID); !acquired { - util.ErrorLog(ctx, util.VolumeOperationAlreadyExistsFmt, volumeID) + log.ErrorLog(ctx, util.VolumeOperationAlreadyExistsFmt, volumeID) return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, volumeID) } @@ -234,7 +235,7 @@ func (rs *ReplicationServer) EnableVolumeReplication(ctx context.Context, mirroringInfo, err := rbdVol.getImageMirroringInfo() if err != nil { - util.ErrorLog(ctx, err.Error()) + log.ErrorLog(ctx, err.Error()) return nil, status.Error(codes.Internal, err.Error()) } @@ -242,7 +243,7 @@ func (rs *ReplicationServer) EnableVolumeReplication(ctx context.Context, if mirroringInfo.State != librbd.MirrorImageEnabled { err = rbdVol.enableImageMirroring(mirroringMode) if err != nil { - util.ErrorLog(ctx, err.Error()) + log.ErrorLog(ctx, err.Error()) return nil, status.Error(codes.Internal, err.Error()) } @@ -253,7 +254,7 @@ func (rs *ReplicationServer) EnableVolumeReplication(ctx context.Context, if err != nil { return nil, err } - util.DebugLog( + log.DebugLog( ctx, "Added scheduling at interval %s, start time %s for volume %s", interval, @@ -281,7 +282,7 @@ func (rs *ReplicationServer) DisableVolumeReplication(ctx context.Context, defer cr.DeleteCredentials() if acquired := rs.VolumeLocks.TryAcquire(volumeID); !acquired { - util.ErrorLog(ctx, util.VolumeOperationAlreadyExistsFmt, volumeID) + log.ErrorLog(ctx, util.VolumeOperationAlreadyExistsFmt, volumeID) return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, volumeID) } @@ -309,7 +310,7 @@ func (rs *ReplicationServer) DisableVolumeReplication(ctx context.Context, mirroringInfo, err := rbdVol.getImageMirroringInfo() if err != nil { - util.ErrorLog(ctx, err.Error()) + log.ErrorLog(ctx, err.Error()) return nil, status.Error(codes.Internal, err.Error()) } @@ -394,7 +395,7 @@ func (rs *ReplicationServer) PromoteVolume(ctx context.Context, defer cr.DeleteCredentials() if acquired := rs.VolumeLocks.TryAcquire(volumeID); !acquired { - util.ErrorLog(ctx, util.VolumeOperationAlreadyExistsFmt, volumeID) + log.ErrorLog(ctx, util.VolumeOperationAlreadyExistsFmt, volumeID) return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, volumeID) } @@ -417,7 +418,7 @@ func (rs *ReplicationServer) PromoteVolume(ctx context.Context, mirroringInfo, err := rbdVol.getImageMirroringInfo() if err != nil { - util.ErrorLog(ctx, err.Error()) + log.ErrorLog(ctx, err.Error()) return nil, status.Error(codes.Internal, err.Error()) } @@ -434,7 +435,7 @@ func (rs *ReplicationServer) PromoteVolume(ctx context.Context, if !mirroringInfo.Primary { err = rbdVol.promoteImage(req.Force) if err != nil { - util.ErrorLog(ctx, err.Error()) + log.ErrorLog(ctx, err.Error()) // In case of the DR the image on the primary site cannot be // demoted as the cluster is down, during failover the image need // to be force promoted. RBD returns `Device or resource busy` @@ -470,7 +471,7 @@ func (rs *ReplicationServer) DemoteVolume(ctx context.Context, defer cr.DeleteCredentials() if acquired := rs.VolumeLocks.TryAcquire(volumeID); !acquired { - util.ErrorLog(ctx, util.VolumeOperationAlreadyExistsFmt, volumeID) + log.ErrorLog(ctx, util.VolumeOperationAlreadyExistsFmt, volumeID) return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, volumeID) } @@ -492,7 +493,7 @@ func (rs *ReplicationServer) DemoteVolume(ctx context.Context, } mirroringInfo, err := rbdVol.getImageMirroringInfo() if err != nil { - util.ErrorLog(ctx, err.Error()) + log.ErrorLog(ctx, err.Error()) return nil, status.Error(codes.Internal, err.Error()) } @@ -509,7 +510,7 @@ func (rs *ReplicationServer) DemoteVolume(ctx context.Context, if mirroringInfo.Primary { err = rbdVol.demoteImage() if err != nil { - util.ErrorLog(ctx, err.Error()) + log.ErrorLog(ctx, err.Error()) return nil, status.Error(codes.Internal, err.Error()) } @@ -525,7 +526,7 @@ func checkRemoteSiteStatus(ctx context.Context, mirrorStatus *librbd.GlobalMirro for _, s := range mirrorStatus.SiteStatuses { if s.MirrorUUID != "" { if imageMirroringState(s.State.String()) != unknown && !s.Up { - util.UsefulLog( + log.UsefulLog( ctx, "peer site mirrorUUID=%s, mirroring state=%s, description=%s and lastUpdate=%s", s.MirrorUUID, @@ -559,7 +560,7 @@ func (rs *ReplicationServer) ResyncVolume(ctx context.Context, defer cr.DeleteCredentials() if acquired := rs.VolumeLocks.TryAcquire(volumeID); !acquired { - util.ErrorLog(ctx, util.VolumeOperationAlreadyExistsFmt, volumeID) + log.ErrorLog(ctx, util.VolumeOperationAlreadyExistsFmt, volumeID) return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, volumeID) } @@ -583,7 +584,7 @@ func (rs *ReplicationServer) ResyncVolume(ctx context.Context, if err != nil { // in case of Resync the image will get deleted and gets recreated and // it takes time for this operation. - util.ErrorLog(ctx, err.Error()) + log.ErrorLog(ctx, err.Error()) return nil, status.Error(codes.Aborted, err.Error()) } @@ -608,7 +609,7 @@ func (rs *ReplicationServer) ResyncVolume(ctx context.Context, return resp, nil } - util.ErrorLog(ctx, err.Error()) + log.ErrorLog(ctx, err.Error()) return nil, status.Error(codes.Internal, err.Error()) } @@ -616,7 +617,7 @@ func (rs *ReplicationServer) ResyncVolume(ctx context.Context, localStatus, err := mirrorStatus.LocalStatus() if err != nil { - util.ErrorLog(ctx, err.Error()) + log.ErrorLog(ctx, err.Error()) return nil, fmt.Errorf("failed to get local status: %w", err) } @@ -639,7 +640,7 @@ func (rs *ReplicationServer) ResyncVolume(ctx context.Context, if strings.Contains(localStatus.State.String(), string(errorState)) { err = rbdVol.resyncImage() if err != nil { - util.ErrorLog(ctx, err.Error()) + log.ErrorLog(ctx, err.Error()) return nil, status.Error(codes.Internal, err.Error()) } @@ -647,7 +648,7 @@ func (rs *ReplicationServer) ResyncVolume(ctx context.Context, // convert the last update time to UTC lastUpdateTime := time.Unix(localStatus.LastUpdate, 0).UTC() - util.UsefulLog( + log.UsefulLog( ctx, "image mirroring state=%s, description=%s and lastUpdate=%s", localStatus.State.String(), diff --git a/internal/rbd/snapshot.go b/internal/rbd/snapshot.go index 29981ceb3..f15f41db1 100644 --- a/internal/rbd/snapshot.go +++ b/internal/rbd/snapshot.go @@ -21,6 +21,7 @@ import ( "fmt" "github.com/ceph/ceph-csi/internal/util" + "github.com/ceph/ceph-csi/internal/util/log" ) func createRBDClone( @@ -31,7 +32,7 @@ func createRBDClone( // create snapshot err := parentVol.createSnapshot(ctx, snap) if err != nil { - util.ErrorLog(ctx, "failed to create snapshot %s: %v", snap, err) + log.ErrorLog(ctx, "failed to create snapshot %s: %v", snap, err) return err } @@ -40,7 +41,7 @@ func createRBDClone( // create clone image and delete snapshot err = cloneRbdVol.cloneRbdImageFromSnapshot(ctx, snap, parentVol) if err != nil { - util.ErrorLog( + log.ErrorLog( ctx, "failed to clone rbd image %s from snapshot %s: %v", cloneRbdVol.RbdImageName, @@ -54,10 +55,10 @@ func createRBDClone( } errSnap := parentVol.deleteSnapshot(ctx, snap) if errSnap != nil { - util.ErrorLog(ctx, "failed to delete snapshot: %v", errSnap) + log.ErrorLog(ctx, "failed to delete snapshot: %v", errSnap) delErr := deleteImage(ctx, cloneRbdVol, cr) if delErr != nil { - util.ErrorLog(ctx, "failed to delete rbd image: %s with error: %v", cloneRbdVol, delErr) + log.ErrorLog(ctx, "failed to delete rbd image: %s with error: %v", cloneRbdVol, delErr) } return err @@ -65,10 +66,10 @@ func createRBDClone( err = cloneRbdVol.getImageInfo() if err != nil { - util.ErrorLog(ctx, "failed to get rbd image: %s details with error: %v", cloneRbdVol, err) + log.ErrorLog(ctx, "failed to get rbd image: %s details with error: %v", cloneRbdVol, err) delErr := deleteImage(ctx, cloneRbdVol, cr) if delErr != nil { - util.ErrorLog(ctx, "failed to delete rbd image: %s with error: %v", cloneRbdVol, delErr) + log.ErrorLog(ctx, "failed to delete rbd image: %s with error: %v", cloneRbdVol, delErr) } return err @@ -88,7 +89,7 @@ func cleanUpSnapshot( err := parentVol.deleteSnapshot(ctx, rbdSnap) if err != nil { if !errors.Is(err, ErrSnapNotFound) { - util.ErrorLog(ctx, "failed to delete snapshot %q: %v", rbdSnap, err) + log.ErrorLog(ctx, "failed to delete snapshot %q: %v", rbdSnap, err) return err } @@ -98,7 +99,7 @@ func cleanUpSnapshot( err := deleteImage(ctx, rbdVol, cr) if err != nil { if !errors.Is(err, ErrImageNotFound) { - util.ErrorLog(ctx, "failed to delete rbd image %q with error: %v", rbdVol, err) + log.ErrorLog(ctx, "failed to delete rbd image %q with error: %v", rbdVol, err) return err } @@ -134,7 +135,7 @@ func undoSnapshotCloning( cr *util.Credentials) error { err := cleanUpSnapshot(ctx, parentVol, rbdSnap, cloneVol, cr) if err != nil { - util.ErrorLog(ctx, "failed to clean up %s or %s: %v", cloneVol, rbdSnap, err) + log.ErrorLog(ctx, "failed to clean up %s or %s: %v", cloneVol, rbdSnap, err) return err } diff --git a/internal/util/cephcmds.go b/internal/util/cephcmds.go index a3fe570cf..a405f4973 100644 --- a/internal/util/cephcmds.go +++ b/internal/util/cephcmds.go @@ -23,6 +23,8 @@ import ( "fmt" "os/exec" + "github.com/ceph/ceph-csi/internal/util/log" + "github.com/ceph/go-ceph/rados" ) @@ -50,14 +52,14 @@ func ExecCommand(ctx context.Context, program string, args ...string) (string, s if err != nil { err = fmt.Errorf("an error (%w) occurred while running %s args: %v", err, program, sanitizedArgs) if ctx != context.TODO() { - UsefulLog(ctx, "%s", err) + log.UsefulLog(ctx, "%s", err) } return stdout, stderr, err } if ctx != context.TODO() { - UsefulLog(ctx, "command succeeded: %s %v", program, sanitizedArgs) + log.UsefulLog(ctx, "command succeeded: %s %v", program, sanitizedArgs) } return stdout, stderr, nil @@ -151,7 +153,7 @@ func CreateObject(ctx context.Context, monitors string, cr *Credentials, poolNam if errors.Is(err, rados.ErrObjectExists) { return JoinErrors(ErrObjectExists, err) } else if err != nil { - ErrorLog(ctx, "failed creating omap (%s) in pool (%s): (%v)", objectName, poolName, err) + log.ErrorLog(ctx, "failed creating omap (%s) in pool (%s): (%v)", objectName, poolName, err) return err } @@ -187,7 +189,7 @@ func RemoveObject(ctx context.Context, monitors string, cr *Credentials, poolNam if errors.Is(err, rados.ErrNotFound) { return JoinErrors(ErrObjectNotFound, err) } else if err != nil { - ErrorLog(ctx, "failed removing omap (%s) in pool (%s): (%v)", oMapName, poolName, err) + log.ErrorLog(ctx, "failed removing omap (%s) in pool (%s): (%v)", oMapName, poolName, err) return err } diff --git a/internal/util/crypto.go b/internal/util/crypto.go index acd0666b6..79821e328 100644 --- a/internal/util/crypto.go +++ b/internal/util/crypto.go @@ -25,6 +25,8 @@ import ( "path" "strconv" "strings" + + "github.com/ceph/ceph-csi/internal/util/log" ) const ( @@ -262,7 +264,7 @@ func VolumeMapper(volumeID string) (mapperFile, mapperFilePath string) { // EncryptVolume encrypts provided device with LUKS. func EncryptVolume(ctx context.Context, devicePath, passphrase string) error { - DebugLog(ctx, "Encrypting device %s with LUKS", devicePath) + log.DebugLog(ctx, "Encrypting device %s with LUKS", devicePath) if _, _, err := LuksFormat(devicePath, passphrase); err != nil { return fmt.Errorf("failed to encrypt device %s with LUKS: %w", devicePath, err) } @@ -272,10 +274,10 @@ func EncryptVolume(ctx context.Context, devicePath, passphrase string) error { // OpenEncryptedVolume opens volume so that it can be used by the client. func OpenEncryptedVolume(ctx context.Context, devicePath, mapperFile, passphrase string) error { - DebugLog(ctx, "Opening device %s with LUKS on %s", devicePath, mapperFile) + log.DebugLog(ctx, "Opening device %s with LUKS on %s", devicePath, mapperFile) _, stderr, err := LuksOpen(devicePath, mapperFile, passphrase) if err != nil { - ErrorLog(ctx, "failed to open LUKS device %q: %s", devicePath, stderr) + log.ErrorLog(ctx, "failed to open LUKS device %q: %s", devicePath, stderr) } return err @@ -283,10 +285,10 @@ func OpenEncryptedVolume(ctx context.Context, devicePath, mapperFile, passphrase // ResizeEncryptedVolume resizes encrypted volume so that it can be used by the client. func ResizeEncryptedVolume(ctx context.Context, mapperFile string) error { - DebugLog(ctx, "Resizing LUKS device %s", mapperFile) + log.DebugLog(ctx, "Resizing LUKS device %s", mapperFile) _, stderr, err := LuksResize(mapperFile) if err != nil { - ErrorLog(ctx, "failed to resize LUKS device %s: %s", mapperFile, stderr) + log.ErrorLog(ctx, "failed to resize LUKS device %s: %s", mapperFile, stderr) } return err @@ -294,7 +296,7 @@ func ResizeEncryptedVolume(ctx context.Context, mapperFile string) error { // CloseEncryptedVolume closes encrypted volume so it can be detached. func CloseEncryptedVolume(ctx context.Context, mapperFile string) error { - DebugLog(ctx, "Closing LUKS device %s", mapperFile) + log.DebugLog(ctx, "Closing LUKS device %s", mapperFile) _, _, err := LuksClose(mapperFile) return err @@ -317,7 +319,7 @@ func DeviceEncryptionStatus(ctx context.Context, devicePath string) (mappedDevic mapPath := strings.TrimPrefix(devicePath, mapperFilePathPrefix+"/") stdout, _, err := LuksStatus(mapPath) if err != nil { - DebugLog(ctx, "device %s is not an active LUKS device: %v", devicePath, err) + log.DebugLog(ctx, "device %s is not an active LUKS device: %v", devicePath, err) return devicePath, "", nil } diff --git a/internal/util/httpserver.go b/internal/util/httpserver.go index d193e4b19..676cf18b3 100644 --- a/internal/util/httpserver.go +++ b/internal/util/httpserver.go @@ -8,6 +8,8 @@ import ( runtime_pprof "runtime/pprof" "strconv" + "github.com/ceph/ceph-csi/internal/util/log" + "github.com/prometheus/client_golang/prometheus/promhttp" ) @@ -24,13 +26,13 @@ func StartMetricsServer(c *Config) { http.Handle(c.MetricsPath, promhttp.Handler()) err := http.ListenAndServe(addr, nil) if err != nil { - FatalLogMsg("failed to listen on address %v: %s", addr, err) + log.FatalLogMsg("failed to listen on address %v: %s", addr, err) } } func addPath(name string, handler http.Handler) { http.Handle(name, handler) - DebugLogMsg("DEBUG: registered profiling handler on /debug/pprof/%s\n", name) + log.DebugLogMsg("DEBUG: registered profiling handler on /debug/pprof/%s\n", name) } // EnableProfiling enables golang profiling. diff --git a/internal/util/idlocker.go b/internal/util/idlocker.go index 165c44263..694f92eb1 100644 --- a/internal/util/idlocker.go +++ b/internal/util/idlocker.go @@ -17,6 +17,8 @@ import ( "fmt" "sync" + "github.com/ceph/ceph-csi/internal/util/log" + "k8s.io/apimachinery/pkg/util/sets" ) @@ -240,6 +242,6 @@ func (ol *OperationLock) release(op operation, volumeID string) { } } default: - ErrorLogMsg("%v operation not supported", op) + log.ErrorLogMsg("%v operation not supported", op) } } diff --git a/internal/util/k8s.go b/internal/util/k8s.go index f175e959e..4eca31be5 100644 --- a/internal/util/k8s.go +++ b/internal/util/k8s.go @@ -19,6 +19,8 @@ package util import ( "os" + "github.com/ceph/ceph-csi/internal/util/log" + k8s "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" @@ -32,17 +34,17 @@ func NewK8sClient() *k8s.Clientset { if cPath != "" { cfg, err = clientcmd.BuildConfigFromFlags("", cPath) if err != nil { - FatalLogMsg("Failed to get cluster config with error: %v\n", err) + log.FatalLogMsg("Failed to get cluster config with error: %v\n", err) } } else { cfg, err = rest.InClusterConfig() if err != nil { - FatalLogMsg("Failed to get cluster config with error: %v\n", err) + log.FatalLogMsg("Failed to get cluster config with error: %v\n", err) } } client, err := k8s.NewForConfig(cfg) if err != nil { - FatalLogMsg("Failed to create client with error: %v\n", err) + log.FatalLogMsg("Failed to create client with error: %v\n", err) } return client diff --git a/internal/util/log.go b/internal/util/log/log.go similarity index 99% rename from internal/util/log.go rename to internal/util/log/log.go index 2f1dbd734..d032e7d63 100644 --- a/internal/util/log.go +++ b/internal/util/log/log.go @@ -11,7 +11,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package util +package log import ( "context" diff --git a/internal/util/topology.go b/internal/util/topology.go index 5a1fadda6..3e782d0c1 100644 --- a/internal/util/topology.go +++ b/internal/util/topology.go @@ -22,6 +22,8 @@ import ( "fmt" "strings" + "github.com/ceph/ceph-csi/internal/util/log" + "github.com/container-storage-interface/spec/lib/go/csi" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -59,7 +61,7 @@ func GetTopologyFromDomainLabels(domainLabels, nodeName, driverName string) (map // Convert passed in labels to a map, and check for uniqueness labelsToRead := strings.SplitN(domainLabels, labelSeparator, -1) - DefaultLog("passed in node labels for processing: %+v", labelsToRead) + log.DefaultLog("passed in node labels for processing: %+v", labelsToRead) labelsIn := make(map[string]bool) labelCount := 0 @@ -106,7 +108,7 @@ func GetTopologyFromDomainLabels(domainLabels, nodeName, driverName string) (map return nil, fmt.Errorf("missing domain labels %v on node %q", missingLabels, nodeName) } - DefaultLog("list of domains processed: %+v", domainMap) + log.DefaultLog("list of domains processed: %+v", domainMap) topology := make(map[string]string) for domain, value := range domainMap { diff --git a/internal/util/util.go b/internal/util/util.go index 9a975c8d3..62dcf2b2f 100644 --- a/internal/util/util.go +++ b/internal/util/util.go @@ -26,6 +26,8 @@ import ( "strings" "time" + "github.com/ceph/ceph-csi/internal/util/log" + "golang.org/x/sys/unix" "k8s.io/apimachinery/pkg/util/validation" "k8s.io/cloud-provider/volume/helpers" @@ -216,7 +218,7 @@ func parseKernelRelease(release string) (int, int, int, int, error) { func CheckKernelSupport(release string, supportedVersions []KernelVersion) bool { version, patchlevel, sublevel, extraversion, err := parseKernelRelease(release) if err != nil { - ErrorLogMsg("%v", err) + log.ErrorLogMsg("%v", err) return false } @@ -242,7 +244,7 @@ func CheckKernelSupport(release string, supportedVersions []KernelVersion) bool } } } - ErrorLogMsg("kernel %s does not support required features", release) + log.ErrorLogMsg("kernel %s does not support required features", release) return false } From 3417fe86e60b3745294edc521048d2be352f4e35 Mon Sep 17 00:00:00 2001 From: Humble Chirammal Date: Wed, 25 Aug 2021 15:07:49 +0530 Subject: [PATCH 04/17] doc: update support matrix for deprecated ceph csi releases As discussed in https://github.com/ceph/ceph-csi/issues/2438 we are marking ceph csi release support to N.(x-1) release versions. N = latest major release x = latest minor release This address the release version support matrix based on the same. Signed-off-by: Humble Chirammal --- README.md | 47 ++++++++++++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 18f76d13a..f5989a4b1 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,13 @@ support other orchestration environments in the future. NOTE: -- **`csiv0.3`** is deprecated with release of **`csi v1.1.0`** +The supported window of Ceph CSI versions is known as "N.(x-1)": +(N (Latest major release) . (x (Latest minor release) - 1)). + +For example, if Ceph CSI latest major version is `3.4.0` today, support is +provided for the versions above `3.3.0`. If users are running an unsupported +Ceph CSI version, they will be asked to upgrade when requesting support for the +cluster. ## Support Matrix @@ -106,24 +112,27 @@ in the Kubernetes documentation. | v3.4.0 (Release) | quay.io/cephcsi/cephcsi | v3.4.0 | | v3.3.1 (Release) | quay.io/cephcsi/cephcsi | v3.3.1 | | v3.3.0 (Release) | quay.io/cephcsi/cephcsi | v3.3.0 | -| v3.2.2 (Release) | quay.io/cephcsi/cephcsi | v3.2.2 | -| v3.2.1 (Release) | quay.io/cephcsi/cephcsi | v3.2.1 | -| v3.2.0 (Release) | quay.io/cephcsi/cephcsi | v3.2.0 | -| v3.1.2 (Release) | quay.io/cephcsi/cephcsi | v3.1.2 | -| v3.1.1 (Release) | quay.io/cephcsi/cephcsi | v3.1.1 | -| v3.1.0 (Release) | quay.io/cephcsi/cephcsi | v3.1.0 | -| v3.0.0 (Release) | quay.io/cephcsi/cephcsi | v3.0.0 | -| v2.1.2 (Release) | quay.io/cephcsi/cephcsi | v2.1.2 | -| v2.1.1 (Release) | quay.io/cephcsi/cephcsi | v2.1.1 | -| v2.1.0 (Release) | quay.io/cephcsi/cephcsi | v2.1.0 | -| v2.0.1 (Release) | quay.io/cephcsi/cephcsi | v2.0.1 | -| v2.0.0 (Release) | quay.io/cephcsi/cephcsi | v2.0.0 | -| v1.2.2 (Release) | quay.io/cephcsi/cephcsi | v1.2.2 | -| v1.2.1 (Release) | quay.io/cephcsi/cephcsi | v1.2.1 | -| v1.2.0 (Release) | quay.io/cephcsi/cephcsi | v1.2.0 | -| v1.1.0 (Release) | quay.io/cephcsi/cephcsi | v1.1.0 | -| v1.0.0 (Branch) | quay.io/cephcsi/cephfsplugin | v1.0.0 | -| v1.0.0 (Branch) | quay.io/cephcsi/rbdplugin | v1.0.0 | + +| Deprecated Ceph CSI Release/Branch | Container image name | Image Tag | +| ----------------------- | --------------------------------| --------- | +| v3.2.2 (Release) | quay.io/cephcsi/cephcsi | v3.2.2 | +| v3.2.1 (Release) | quay.io/cephcsi/cephcsi | v3.2.1 | +| v3.2.0 (Release) | quay.io/cephcsi/cephcsi | v3.2.0 | +| v3.1.2 (Release) | quay.io/cephcsi/cephcsi | v3.1.2 | +| v3.1.1 (Release) | quay.io/cephcsi/cephcsi | v3.1.1 | +| v3.1.0 (Release) | quay.io/cephcsi/cephcsi | v3.1.0 | +| v3.0.0 (Release) | quay.io/cephcsi/cephcsi | v3.0.0 | +| v2.1.2 (Release) | quay.io/cephcsi/cephcsi | v2.1.2 | +| v2.1.1 (Release) | quay.io/cephcsi/cephcsi | v2.1.1 | +| v2.1.0 (Release) | quay.io/cephcsi/cephcsi | v2.1.0 | +| v2.0.1 (Release) | quay.io/cephcsi/cephcsi | v2.0.1 | +| v2.0.0 (Release) | quay.io/cephcsi/cephcsi | v2.0.0 | +| v1.2.2 (Release) | quay.io/cephcsi/cephcsi | v1.2.2 | +| v1.2.1 (Release) | quay.io/cephcsi/cephcsi | v1.2.1 | +| v1.2.0 (Release) | quay.io/cephcsi/cephcsi | v1.2.0 | +| v1.1.0 (Release) | quay.io/cephcsi/cephcsi | v1.1.0 | +| v1.0.0 (Branch) | quay.io/cephcsi/cephfsplugin | v1.0.0 | +| v1.0.0 (Branch) | quay.io/cephcsi/rbdplugin | v1.0.0 | ## Contributing to this repo From 8ea495ab819ffbcadb13e866370f3101b59be169 Mon Sep 17 00:00:00 2001 From: Humble Chirammal Date: Thu, 26 Aug 2021 15:20:27 +0530 Subject: [PATCH 05/17] rbd: skip volumeattachment processing if pv marked for deletion if the volumeattachment has been fetched but marked for deletion the nbd healer dont want to process further on this pv. This patch adds a check for pv is marked for deletion and if so, make the healer skip processing the same Signed-off-by: Humble Chirammal --- internal/rbd/rbd_healer.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/internal/rbd/rbd_healer.go b/internal/rbd/rbd_healer.go index 05cb941a4..9a893764f 100644 --- a/internal/rbd/rbd_healer.go +++ b/internal/rbd/rbd_healer.go @@ -153,9 +153,8 @@ func runVolumeHealer(ns *NodeServer, conf *util.Config) error { continue } - // TODO: check with pv delete annotations, for eg: what happens when the pv is marked for delete - // skip this volumeattachment if its pv is not bound - if pv.Status.Phase != v1.VolumeBound { + // skip this volumeattachment if its pv is not bound or marked for deletion + if pv.Status.Phase != v1.VolumeBound || pv.DeletionTimestamp != nil { continue } // skip if mounter is not rbd-nbd From dbf2eb3905038f9c384f49d06eea857ccff126fd Mon Sep 17 00:00:00 2001 From: Rakshith R Date: Thu, 26 Aug 2021 16:15:02 +0530 Subject: [PATCH 06/17] e2e: ignore lines with 'Warning' in isAlreadyExistsCliError() Signed-off-by: Rakshith R --- e2e/errors.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/e2e/errors.go b/e2e/errors.go index 31aad8ad0..fc84c8d35 100644 --- a/e2e/errors.go +++ b/e2e/errors.go @@ -103,6 +103,10 @@ func isAlreadyExistsCLIError(err error) bool { if strings.TrimSuffix(s, "\n") == "" { continue } + // Ignore warnings + if strings.Contains(s, "Warning") { + continue + } // Resource already exists error message if !strings.Contains(s, "Error from server (AlreadyExists)") { return false From f9d4972444f8be1ad5f180bf076077e8b46ac884 Mon Sep 17 00:00:00 2001 From: Rakshith R Date: Thu, 26 Aug 2021 17:50:35 +0530 Subject: [PATCH 07/17] e2e: fix log msg in retryKubectlInput() e2elog.Logf("waiting for kubectl (%s -f $q args %s) to finish", action, args) changed to e2elog.Logf("waiting for kubectl (%s -f args %s) to finish", action, args) Signed-off-by: Rakshith R --- e2e/utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/utils.go b/e2e/utils.go index adf9f2fd9..7b2a22db1 100644 --- a/e2e/utils.go +++ b/e2e/utils.go @@ -1261,7 +1261,7 @@ func (ka kubectlAction) String() string { // no error occurred, or the timeout passed. func retryKubectlInput(namespace string, action kubectlAction, data string, t int, args ...string) error { timeout := time.Duration(t) * time.Minute - e2elog.Logf("waiting for kubectl (%s -f %q args %s) to finish", action, args) + e2elog.Logf("waiting for kubectl (%s -f args %s) to finish", action, args) start := time.Now() return wait.PollImmediate(poll, timeout, func() (bool, error) { From c17b3f69bddb2cbe7f8351845052afb612a9e5eb Mon Sep 17 00:00:00 2001 From: Niels de Vos Date: Thu, 26 Aug 2021 17:57:12 +0200 Subject: [PATCH 08/17] ci: add dependabot config for updating vendored packages Vendored dependencies need updating on regular basis. This is currently done manually by developers, but it can be automated by Dependabot. By dropping the dependabot.yml config file in the .github/ directory the bot should get enabled. See-also: https://docs.github.com/en/code-security/supply-chain-security/keeping-your-dependencies-updated-automatically/enabling-and-disabling-version-updates Signed-off-by: Niels de Vos --- .github/dependabot.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..d9a772676 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +--- +version: 2 +updates: + - package-ecosystem: "gomod" + directory: "/" + schedule: + interval: "weekly" + labels: + - rebase + commit-message: + prefix: "rebase" From b2e5e0574bbf0ef2f12e4dd1489cb5f6fd6ec3b9 Mon Sep 17 00:00:00 2001 From: Niels de Vos Date: Mon, 30 Aug 2021 12:25:55 +0200 Subject: [PATCH 09/17] build: get gomodules.xyz/jsonpatch/v2 from github The `make containerized-test TARGET=mod-check` jobs fail because the web service at gomodules.xyz is not behaving correctly at the moment: running: go mod verify go: sigs.k8s.io/controller-runtime@v0.9.2 requires gomodules.xyz/jsonpatch/v2@v2.2.0: unrecognized import path "gomodules.xyz/jsonpatch/v2": reading https://gomodules.xyz/jsonpatch/v2?go-get=1: 404 Not Found Signed-off-by: Niels de Vos --- go.mod | 1 + go.sum | 5 ++--- vendor/modules.txt | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 39552dd3f..5c163d04e 100644 --- a/go.mod +++ b/go.mod @@ -39,6 +39,7 @@ replace ( github.com/hashicorp/vault/api => github.com/hashicorp/vault/api v1.0.5-0.20200902155336-f9d5ce5a171a github.com/hashicorp/vault/sdk => github.com/hashicorp/vault/sdk v0.1.14-0.20201116234512-b4d4137dfe8b github.com/portworx/sched-ops => github.com/portworx/sched-ops v0.20.4-openstorage-rc3 + gomodules.xyz/jsonpatch/v2 => github.com/gomodules/jsonpatch/v2 v2.2.0 google.golang.org/grpc => google.golang.org/grpc v1.35.0 k8s.io/api => k8s.io/api v0.22.0 k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.22.0 diff --git a/go.sum b/go.sum index bbf5fa9d3..15cb71062 100644 --- a/go.sum +++ b/go.sum @@ -380,6 +380,8 @@ github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8l github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk= +github.com/gomodules/jsonpatch/v2 v2.2.0 h1:QBjDK/nX43P4z/Os3gnk8VeFdLDgBuMns1Wljyo607U= +github.com/gomodules/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= @@ -1329,9 +1331,6 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gomodules.xyz/jsonpatch/v2 v2.0.1/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= -gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY= -gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= gonum.org/v1/gonum v0.6.2/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= diff --git a/vendor/modules.txt b/vendor/modules.txt index f0c1a2265..4e7e6a8df 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -412,7 +412,7 @@ golang.org/x/text/unicode/bidi golang.org/x/text/unicode/norm # golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac golang.org/x/time/rate -# gomodules.xyz/jsonpatch/v2 v2.2.0 +# gomodules.xyz/jsonpatch/v2 v2.2.0 => github.com/gomodules/jsonpatch/v2 v2.2.0 gomodules.xyz/jsonpatch/v2 # google.golang.org/appengine v1.6.7 google.golang.org/appengine/internal @@ -1059,6 +1059,7 @@ sigs.k8s.io/yaml # github.com/hashicorp/vault/api => github.com/hashicorp/vault/api v1.0.5-0.20200902155336-f9d5ce5a171a # github.com/hashicorp/vault/sdk => github.com/hashicorp/vault/sdk v0.1.14-0.20201116234512-b4d4137dfe8b # github.com/portworx/sched-ops => github.com/portworx/sched-ops v0.20.4-openstorage-rc3 +# gomodules.xyz/jsonpatch/v2 => github.com/gomodules/jsonpatch/v2 v2.2.0 # google.golang.org/grpc => google.golang.org/grpc v1.35.0 # k8s.io/api => k8s.io/api v0.22.0 # k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.22.0 From 2cc96dc539ace42d2d55f323cf4d8c4836adb891 Mon Sep 17 00:00:00 2001 From: Niels de Vos Date: Mon, 30 Aug 2021 13:33:09 +0200 Subject: [PATCH 10/17] build: vendor code.cloudfoundry.org/gofileutils from GitHub There is a problem accessing the code.cloudfoundry.org web service iver TLS. It seems to redirect to GitHub, so use the package from there: running: go mod verify go: github.com/libopenstorage/secrets@v0.0.0-20210709082113-dde442ea20ec requires github.com/hashicorp/vault@v1.4.2 requires github.com/hashicorp/vault-plugin-auth-cf@v0.5.4 requires github.com/cloudfoundry-community/go-cfclient@v0.0.0-20190201205600-f136f9222381 requires code.cloudfoundry.org/gofileutils@v0.0.0-20170111115228-4d0c80011a0f: unrecognized import path "code.cloudfoundry.org/gofileutils": https fetch: Get "https://code.cloudfoundry.org/gofileutils?go-get=1": x509: certificate signed by unknown authority Signed-off-by: Niels de Vos --- go.mod | 1 + go.sum | 2 +- vendor/modules.txt | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 5c163d04e..516081f77 100644 --- a/go.mod +++ b/go.mod @@ -35,6 +35,7 @@ require ( ) replace ( + code.cloudfoundry.org/gofileutils => github.com/cloudfoundry/gofileutils v0.0.0-20170111115228-4d0c80011a0f github.com/golang/protobuf => github.com/golang/protobuf v1.4.3 github.com/hashicorp/vault/api => github.com/hashicorp/vault/api v1.0.5-0.20200902155336-f9d5ce5a171a github.com/hashicorp/vault/sdk => github.com/hashicorp/vault/sdk v0.1.14-0.20201116234512-b4d4137dfe8b diff --git a/go.sum b/go.sum index 15cb71062..8f48998fa 100644 --- a/go.sum +++ b/go.sum @@ -36,7 +36,6 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -code.cloudfoundry.org/gofileutils v0.0.0-20170111115228-4d0c80011a0f/go.mod h1:sk5LnIjB/nIEU7yP5sDQExVm62wu0pBh3yrElngUisI= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= dmitri.shuralyov.com/gpu/mtl v0.0.0-20201218220906-28db891af037/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= git.apache.org/thrift.git v0.12.0/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= @@ -178,6 +177,7 @@ github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJ github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/cloudfoundry-community/go-cfclient v0.0.0-20190201205600-f136f9222381/go.mod h1:e5+USP2j8Le2M0Jo3qKPFnNhuo1wueU4nWHCXBOfQ14= +github.com/cloudfoundry/gofileutils v0.0.0-20170111115228-4d0c80011a0f/go.mod h1:Zv7xtAh/T/tmfZlxpESaWWiWOdiJz2GfbBYxImuI6T4= github.com/clusterhq/flocker-go v0.0.0-20160920122132-2b8b7259d313/go.mod h1:P1wt9Z3DP8O6W3rvwCt0REIlshg1InHImaLW0t3ObY0= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= diff --git a/vendor/modules.txt b/vendor/modules.txt index 4e7e6a8df..24feb43a8 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1055,6 +1055,7 @@ sigs.k8s.io/structured-merge-diff/v4/typed sigs.k8s.io/structured-merge-diff/v4/value # sigs.k8s.io/yaml v1.2.0 sigs.k8s.io/yaml +# code.cloudfoundry.org/gofileutils => github.com/cloudfoundry/gofileutils v0.0.0-20170111115228-4d0c80011a0f # github.com/golang/protobuf => github.com/golang/protobuf v1.4.3 # github.com/hashicorp/vault/api => github.com/hashicorp/vault/api v1.0.5-0.20200902155336-f9d5ce5a171a # github.com/hashicorp/vault/sdk => github.com/hashicorp/vault/sdk v0.1.14-0.20201116234512-b4d4137dfe8b From 778b5e86dea04eefaf9d8e06cd50c7705e1055ef Mon Sep 17 00:00:00 2001 From: Niels de Vos Date: Thu, 26 Aug 2021 13:15:47 +0200 Subject: [PATCH 11/17] cleanup: move k8s functions to the util/k8s package By placing the NewK8sClient() function in its own package, the KMS API can be split from the "internal/util" package. Some of the KMS providers use the NewK8sClient() function, and this causes circular dependencies between "internal/utils" -> "internal/kms" -> "internal/utils", which are not alowed in Go. Updates: #852 Signed-off-by: Niels de Vos --- internal/rbd/rbd_healer.go | 3 ++- internal/rbd/rbd_util.go | 3 ++- internal/util/aws_metadata.go | 4 +++- internal/util/{k8s.go => k8s/client.go} | 8 ++++---- internal/util/kms.go | 4 +++- internal/util/secretskms.go | 4 +++- internal/util/topology.go | 3 ++- internal/util/vault_tokens.go | 4 +++- 8 files changed, 22 insertions(+), 11 deletions(-) rename internal/util/{k8s.go => k8s/client.go} (90%) diff --git a/internal/rbd/rbd_healer.go b/internal/rbd/rbd_healer.go index 9a893764f..c785d4cf5 100644 --- a/internal/rbd/rbd_healer.go +++ b/internal/rbd/rbd_healer.go @@ -21,6 +21,7 @@ import ( "sync" "github.com/ceph/ceph-csi/internal/util" + kubeclient "github.com/ceph/ceph-csi/internal/util/k8s" "github.com/ceph/ceph-csi/internal/util/log" "github.com/container-storage-interface/spec/lib/go/csi" @@ -128,7 +129,7 @@ func callNodeStageVolume(ns *NodeServer, c *k8s.Clientset, pv *v1.PersistentVolu // runVolumeHealer heal the volumes attached on a node. func runVolumeHealer(ns *NodeServer, conf *util.Config) error { - c := util.NewK8sClient() + c := kubeclient.NewK8sClient() val, err := c.StorageV1().VolumeAttachments().List(context.TODO(), metav1.ListOptions{}) if err != nil { log.ErrorLogMsg("list volumeAttachments failed, err: %v", err) diff --git a/internal/rbd/rbd_util.go b/internal/rbd/rbd_util.go index e02af3446..731066f2e 100644 --- a/internal/rbd/rbd_util.go +++ b/internal/rbd/rbd_util.go @@ -29,6 +29,7 @@ import ( "time" "github.com/ceph/ceph-csi/internal/util" + "github.com/ceph/ceph-csi/internal/util/k8s" "github.com/ceph/ceph-csi/internal/util/log" "github.com/ceph/go-ceph/rados" @@ -1063,7 +1064,7 @@ func genVolFromVolID( // be the same in the PV.Spec.CSI.VolumeHandle. Check the PV annotation for // the new volumeHandle. If the new volumeHandle is found, generate the RBD // volume structure from the new volumeHandle. - c := util.NewK8sClient() + c := k8s.NewK8sClient() listOpt := metav1.ListOptions{ LabelSelector: PVReplicatedLabelKey, } diff --git a/internal/util/aws_metadata.go b/internal/util/aws_metadata.go index 3aadca739..dd4ba1e6d 100644 --- a/internal/util/aws_metadata.go +++ b/internal/util/aws_metadata.go @@ -22,6 +22,8 @@ import ( "errors" "fmt" + "github.com/ceph/ceph-csi/internal/util/k8s" + "github.com/aws/aws-sdk-go/aws" awsCreds "github.com/aws/aws-sdk-go/aws/credentials" awsSession "github.com/aws/aws-sdk-go/aws/session" @@ -123,7 +125,7 @@ func initAWSMetadataKMS(args KMSInitializerArgs) (EncryptionKMS, error) { } func (kms *AWSMetadataKMS) getSecrets() (map[string]interface{}, error) { - c := NewK8sClient() + c := k8s.NewK8sClient() secret, err := c.CoreV1().Secrets(kms.namespace).Get(context.TODO(), kms.secretName, metav1.GetOptions{}) if err != nil { diff --git a/internal/util/k8s.go b/internal/util/k8s/client.go similarity index 90% rename from internal/util/k8s.go rename to internal/util/k8s/client.go index 4eca31be5..d17c19fad 100644 --- a/internal/util/k8s.go +++ b/internal/util/k8s/client.go @@ -14,20 +14,20 @@ See the License for the specific language governing permissions and limitations under the License. */ -package util +package k8s import ( "os" "github.com/ceph/ceph-csi/internal/util/log" - k8s "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" ) // NewK8sClient create kubernetes client. -func NewK8sClient() *k8s.Clientset { +func NewK8sClient() *kubernetes.Clientset { var cfg *rest.Config var err error cPath := os.Getenv("KUBERNETES_CONFIG_PATH") @@ -42,7 +42,7 @@ func NewK8sClient() *k8s.Clientset { log.FatalLogMsg("Failed to get cluster config with error: %v\n", err) } } - client, err := k8s.NewForConfig(cfg) + client, err := kubernetes.NewForConfig(cfg) if err != nil { log.FatalLogMsg("Failed to create client with error: %v\n", err) } diff --git a/internal/util/kms.go b/internal/util/kms.go index c5b57d5b3..f2af1fe0b 100644 --- a/internal/util/kms.go +++ b/internal/util/kms.go @@ -23,6 +23,8 @@ import ( "io/ioutil" "os" + "github.com/ceph/ceph-csi/internal/util/k8s" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -146,7 +148,7 @@ func getKMSConfigMap() (map[string]interface{}, error) { } cmName := getKMSConfigMapName() - c := NewK8sClient() + c := k8s.NewK8sClient() cm, err := c.CoreV1().ConfigMaps(ns).Get(context.Background(), cmName, metav1.GetOptions{}) if err != nil { diff --git a/internal/util/secretskms.go b/internal/util/secretskms.go index 879327983..1436c9017 100644 --- a/internal/util/secretskms.go +++ b/internal/util/secretskms.go @@ -26,6 +26,8 @@ import ( "fmt" "io" + "github.com/ceph/ceph-csi/internal/util/k8s" + "golang.org/x/crypto/scrypt" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -155,7 +157,7 @@ func (kms SecretsMetadataKMS) fetchEncryptionPassphrase( secretNamespace = defaultNamespace } - c := NewK8sClient() + c := k8s.NewK8sClient() secret, err := c.CoreV1().Secrets(secretNamespace).Get(context.TODO(), secretName, metav1.GetOptions{}) if err != nil { diff --git a/internal/util/topology.go b/internal/util/topology.go index 3e782d0c1..6a2b6c2d5 100644 --- a/internal/util/topology.go +++ b/internal/util/topology.go @@ -22,6 +22,7 @@ import ( "fmt" "strings" + "github.com/ceph/ceph-csi/internal/util/k8s" "github.com/ceph/ceph-csi/internal/util/log" "github.com/container-storage-interface/spec/lib/go/csi" @@ -34,7 +35,7 @@ const ( ) func k8sGetNodeLabels(nodeName string) (map[string]string, error) { - client := NewK8sClient() + client := k8s.NewK8sClient() node, err := client.CoreV1().Nodes().Get(context.TODO(), nodeName, metav1.GetOptions{}) if err != nil { return nil, fmt.Errorf("failed to get node %q information: %w", nodeName, err) diff --git a/internal/util/vault_tokens.go b/internal/util/vault_tokens.go index 73eb7a19f..57f8e286d 100644 --- a/internal/util/vault_tokens.go +++ b/internal/util/vault_tokens.go @@ -24,6 +24,8 @@ import ( "os" "strconv" + "github.com/ceph/ceph-csi/internal/util/k8s" + "github.com/hashicorp/vault/api" apierrs "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -438,7 +440,7 @@ func (vtc *vaultTenantConnection) initCertificates(config map[string]interface{} func (vtc *vaultTenantConnection) getK8sClient() *kubernetes.Clientset { if vtc.client == nil { - vtc.client = NewK8sClient() + vtc.client = k8s.NewK8sClient() } return vtc.client From 4a3b1181ce64df86e4e620a45e167b1c9e328248 Mon Sep 17 00:00:00 2001 From: Niels de Vos Date: Thu, 26 Aug 2021 13:43:20 +0200 Subject: [PATCH 12/17] cleanup: move KMS functionality into its own package A new "internal/kms" package is introduced, it holds the API that can be consumed by the RBD components. The KMS providers are currently in the same package as the API. With later follow-up changes the providers will be placed in their own sub-package. Because of the name of the package "kms", the types, functions and structs inside the package should not be prefixed with KMS anymore: internal/kms/kms.go:213:6: type name will be used as kms.KMSInitializerArgs by other packages, and that stutters; consider calling this InitializerArgs (golint) Updates: #852 Signed-off-by: Niels de Vos --- internal/{util => kms}/aws_metadata.go | 10 +- internal/{util => kms}/aws_metadata_test.go | 2 +- internal/{util => kms}/kms.go | 153 ++++++++++++++++---- internal/{util => kms}/kms_test.go | 16 +- internal/{util => kms}/secretskms.go | 24 +-- internal/{util => kms}/secretskms_test.go | 32 +++- internal/{util => kms}/vault.go | 8 +- internal/{util => kms}/vault_sa.go | 6 +- internal/{util => kms}/vault_sa_test.go | 2 +- internal/{util => kms}/vault_test.go | 2 +- internal/{util => kms}/vault_tokens.go | 8 +- internal/{util => kms}/vault_tokens_test.go | 4 +- internal/rbd/encryption.go | 3 +- internal/util/crypto.go | 90 ++---------- internal/util/crypto_test.go | 31 ++-- internal/util/util.go | 14 -- 16 files changed, 215 insertions(+), 190 deletions(-) rename internal/{util => kms}/aws_metadata.go (95%) rename internal/{util => kms}/aws_metadata_test.go (98%) rename internal/{util => kms}/kms.go (60%) rename internal/{util => kms}/kms_test.go (76%) rename internal/{util => kms}/secretskms.go (94%) rename internal/{util => kms}/secretskms_test.go (81%) rename internal/{util => kms}/vault.go (99%) rename internal/{util => kms}/vault_sa.go (98%) rename internal/{util => kms}/vault_sa_test.go (99%) rename internal/{util => kms}/vault_test.go (99%) rename internal/{util => kms}/vault_tokens.go (99%) rename internal/{util => kms}/vault_tokens_test.go (99%) diff --git a/internal/util/aws_metadata.go b/internal/kms/aws_metadata.go similarity index 95% rename from internal/util/aws_metadata.go rename to internal/kms/aws_metadata.go index dd4ba1e6d..a281b219f 100644 --- a/internal/util/aws_metadata.go +++ b/internal/kms/aws_metadata.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package util +package kms import ( "context" @@ -60,7 +60,7 @@ const ( awsCMK = "AWS_CMK_ARN" ) -var _ = RegisterKMSProvider(KMSProvider{ +var _ = RegisterProvider(Provider{ UniqueID: kmsTypeAWSMetadata, Initializer: initAWSMetadataKMS, }) @@ -78,7 +78,7 @@ type AWSMetadataKMS struct { cmk string } -func initAWSMetadataKMS(args KMSInitializerArgs) (EncryptionKMS, error) { +func initAWSMetadataKMS(args ProviderInitArgs) (EncryptionKMS, error) { kms := &AWSMetadataKMS{ namespace: args.Namespace, } @@ -152,10 +152,10 @@ func (kms *AWSMetadataKMS) Destroy() { // Nothing to do. } -// requiresDEKStore indicates that the DEKs should get stored in the metadata +// RequiresDEKStore indicates that the DEKs should get stored in the metadata // of the volumes. This Amazon KMS provider does not support storing DEKs in // AWS as that adds additional costs. -func (kms *AWSMetadataKMS) requiresDEKStore() DEKStoreType { +func (kms *AWSMetadataKMS) RequiresDEKStore() DEKStoreType { return DEKStoreMetadata } diff --git a/internal/util/aws_metadata_test.go b/internal/kms/aws_metadata_test.go similarity index 98% rename from internal/util/aws_metadata_test.go rename to internal/kms/aws_metadata_test.go index 5f82db763..4b7078817 100644 --- a/internal/util/aws_metadata_test.go +++ b/internal/kms/aws_metadata_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package util +package kms import ( "testing" diff --git a/internal/util/kms.go b/internal/kms/kms.go similarity index 60% rename from internal/util/kms.go rename to internal/kms/kms.go index f2af1fe0b..5fd7d3dcb 100644 --- a/internal/util/kms.go +++ b/internal/kms/kms.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package util +package kms import ( "context" @@ -46,6 +46,12 @@ const ( // defaultKMSConfigMapName default ConfigMap name to fetch kms // connection details. defaultKMSConfigMapName = "csi-kms-connection-details" + + // kmsConfigPath is the location of the vault config file. + kmsConfigPath = "/etc/ceph-csi-encryption-kms-config/config.json" + + // Default KMS type. + DefaultKMSType = "default" ) // GetKMS returns an instance of Key Management System. @@ -56,8 +62,8 @@ const ( // - secrets contain additional details, like TLS certificates to connect to // the KMS func GetKMS(tenant, kmsID string, secrets map[string]string) (EncryptionKMS, error) { - if kmsID == "" || kmsID == defaultKMSType { - return initSecretsKMS(secrets) + if kmsID == "" || kmsID == DefaultKMSType { + return GetDefaultKMS(secrets) } config, err := getKMSConfiguration() @@ -170,10 +176,10 @@ func getKMSConfigMap() (map[string]interface{}, error) { return kmsConfig, nil } -// getKMSProvider inspects the configuration and tries to identify what -// KMSProvider is expected to be used with it. This returns the -// KMSProvider.UniqueID. -func getKMSProvider(config map[string]interface{}) (string, error) { +// getProvider inspects the configuration and tries to identify what +// Provider is expected to be used with it. This returns the +// Provider.UniqueID. +func getProvider(config map[string]interface{}) (string, error) { var name string providerName, ok := config[kmsTypeKey] @@ -202,45 +208,45 @@ func getKMSProvider(config map[string]interface{}) (string, error) { "configuration option %q or %q", kmsTypeKey, kmsProviderKey) } -// KMSInitializerArgs get passed to KMSInitializerFunc when a new instance of a -// KMSProvider is initialized. -type KMSInitializerArgs struct { +// ProviderInitArgs get passed to ProviderInitFunc when a new instance of a +// Provider is initialized. +type ProviderInitArgs struct { Tenant string Config map[string]interface{} Secrets map[string]string // Namespace contains the Kubernetes Namespace where the Ceph-CSI Pods // are running. This is an optional option, and might be unset when the - // KMSProvider.Initializer is called. + // Provider.Initializer is called. Namespace string } -// KMSInitializerFunc gets called when the KMSProvider needs to be +// ProviderInitFunc gets called when the Provider needs to be // instantiated. -type KMSInitializerFunc func(args KMSInitializerArgs) (EncryptionKMS, error) +type ProviderInitFunc func(args ProviderInitArgs) (EncryptionKMS, error) -type KMSProvider struct { +type Provider struct { UniqueID string - Initializer KMSInitializerFunc + Initializer ProviderInitFunc } type kmsProviderList struct { - providers map[string]KMSProvider + providers map[string]Provider } // kmsManager is used to create instances for a KMS provider. -var kmsManager = kmsProviderList{providers: map[string]KMSProvider{}} +var kmsManager = kmsProviderList{providers: map[string]Provider{}} -// RegisterKMSProvider uses kmsManager to register the given KMSProvider. The -// KMSProvider.Initializer function will get called when a new instance of the +// RegisterProvider uses kmsManager to register the given Provider. The +// Provider.Initializer function will get called when a new instance of the // KMS is required. -func RegisterKMSProvider(provider KMSProvider) bool { +func RegisterProvider(provider Provider) bool { // validate uniqueness of the UniqueID if provider.UniqueID == "" { panic("a provider MUST set a UniqueID") } _, ok := kmsManager.providers[provider.UniqueID] if ok { - panic("duplicate registration of KMSProvider.UniqueID: " + provider.UniqueID) + panic("duplicate registration of Provider.UniqueID: " + provider.UniqueID) } // validate the Initializer @@ -253,14 +259,14 @@ func RegisterKMSProvider(provider KMSProvider) bool { return true } -// buildKMS creates a new KMSProvider instance, based on the configuration that -// was passed. This uses getKMSProvider() internally to identify the -// KMSProvider to instantiate. +// buildKMS creates a new Provider instance, based on the configuration that +// was passed. This uses getProvider() internally to identify the +// Provider to instantiate. func (kf *kmsProviderList) buildKMS( tenant string, config map[string]interface{}, secrets map[string]string) (EncryptionKMS, error) { - providerName, err := getKMSProvider(config) + providerName, err := getProvider(config) if err != nil { return nil, err } @@ -271,14 +277,14 @@ func (kf *kmsProviderList) buildKMS( providerName) } - kmsInitArgs := KMSInitializerArgs{ + kmsInitArgs := ProviderInitArgs{ Tenant: tenant, Config: config, Secrets: secrets, } // Namespace is an optional parameter, it may not be set and is not - // required for all KMSProviders + // required for all Providers ns, err := getPodNamespace() if err == nil { kmsInitArgs.Namespace = ns @@ -286,3 +292,96 @@ func (kf *kmsProviderList) buildKMS( return provider.Initializer(kmsInitArgs) } + +func GetDefaultKMS(secrets map[string]string) (EncryptionKMS, error) { + provider, ok := kmsManager.providers[DefaultKMSType] + if !ok { + return nil, fmt.Errorf("could not find KMS provider %q", DefaultKMSType) + } + + kmsInitArgs := ProviderInitArgs{ + Secrets: secrets, + } + + return provider.Initializer(kmsInitArgs) +} + +// EncryptionKMS provides external Key Management System for encryption +// passphrases storage. +type EncryptionKMS interface { + Destroy() + + // RequiresDEKStore returns the DEKStoreType that is needed to be + // configure for the KMS. Nothing needs to be done when this function + // returns DEKStoreIntegrated, otherwise you will need to configure an + // alternative storage for the DEKs. + RequiresDEKStore() DEKStoreType + + // EncryptDEK provides a way for a KMS to encrypt a DEK. In case the + // encryption is done transparently inside the KMS service, the + // function can return an unencrypted value. + EncryptDEK(volumeID, plainDEK string) (string, error) + + // DecryptDEK provides a way for a KMS to decrypt a DEK. In case the + // encryption is done transparently inside the KMS service, the + // function does not need to do anything except return the encyptedDEK + // as it was received. + DecryptDEK(volumeID, encyptedDEK string) (string, error) +} + +// DEKStoreType describes what DEKStore needs to be configured when using a +// particular KMS. A KMS might support different DEKStores depending on its +// configuration. +type DEKStoreType string + +const ( + // DEKStoreIntegrated indicates that the KMS itself supports storing + // DEKs. + DEKStoreIntegrated = DEKStoreType("") + // DEKStoreMetadata indicates that the KMS should be configured to + // store the DEK in the metadata of the volume. + DEKStoreMetadata = DEKStoreType("metadata") +) + +// DEKStore allows KMS instances to implement a modular backend for DEK +// storage. This can be used to store the DEK in a different location, in case +// the KMS can not store passphrases for volumes. +type DEKStore interface { + // StoreDEK saves the DEK in the configured store. + StoreDEK(volumeID string, dek string) error + // FetchDEK reads the DEK from the configured store and returns it. + FetchDEK(volumeID string) (string, error) + // RemoveDEK deletes the DEK from the configured store. + RemoveDEK(volumeID string) error +} + +// IntegratedDEK is a DEKStore that can not be configured. Either the KMS does +// not use a DEK, or the DEK is stored in the KMS without additional +// configuration options. +type IntegratedDEK struct{} + +func (i IntegratedDEK) RequiresDEKStore() DEKStoreType { + return DEKStoreIntegrated +} + +func (i IntegratedDEK) EncryptDEK(volumeID, plainDEK string) (string, error) { + return plainDEK, nil +} + +func (i IntegratedDEK) DecryptDEK(volumeID, encyptedDEK string) (string, error) { + return encyptedDEK, nil +} + +// getKeys takes a map that uses strings for keys and returns a slice with the +// keys. +func getKeys(m map[string]interface{}) []string { + keys := make([]string, len(m)) + + i := 0 + for k := range m { + keys[i] = k + i++ + } + + return keys +} diff --git a/internal/util/kms_test.go b/internal/kms/kms_test.go similarity index 76% rename from internal/util/kms_test.go rename to internal/kms/kms_test.go index 0d5f10b20..ec3452897 100644 --- a/internal/util/kms_test.go +++ b/internal/kms/kms_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package util +package kms import ( "testing" @@ -22,22 +22,22 @@ import ( "github.com/stretchr/testify/assert" ) -func noinitKMS(args KMSInitializerArgs) (EncryptionKMS, error) { +func noinitKMS(args ProviderInitArgs) (EncryptionKMS, error) { return nil, nil } -func TestRegisterKMSProvider(t *testing.T) { +func TestRegisterProvider(t *testing.T) { t.Parallel() tests := []struct { - provider KMSProvider + provider Provider panics bool }{{ - KMSProvider{ + Provider{ UniqueID: "incomplete-provider", }, true, }, { - KMSProvider{ + Provider{ UniqueID: "initializer-only", Initializer: noinitKMS, }, @@ -47,9 +47,9 @@ func TestRegisterKMSProvider(t *testing.T) { for _, test := range tests { provider := test.provider if test.panics { - assert.Panics(t, func() { RegisterKMSProvider(provider) }) + assert.Panics(t, func() { RegisterProvider(provider) }) } else { - assert.True(t, RegisterKMSProvider(provider)) + assert.True(t, RegisterProvider(provider)) } } } diff --git a/internal/util/secretskms.go b/internal/kms/secretskms.go similarity index 94% rename from internal/util/secretskms.go rename to internal/kms/secretskms.go index 1436c9017..cac5838fb 100644 --- a/internal/util/secretskms.go +++ b/internal/kms/secretskms.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package util +package kms import ( "context" @@ -36,9 +36,6 @@ const ( // Encryption passphrase location in K8s secrets. encryptionPassphraseKey = "encryptionPassphrase" - // Default KMS type. - defaultKMSType = "default" - // kmsTypeSecretsMetadata is the SecretsKMS with per-volume encryption, // where the DEK is stored in the metadata of the volume itself. kmsTypeSecretsMetadata = "metadata" @@ -53,16 +50,21 @@ const ( // SecretsKMS is default KMS implementation that means no KMS is in use. type SecretsKMS struct { - integratedDEK + IntegratedDEK passphrase string } -// initSecretsKMS initializes a SecretsKMS that uses the passphrase from the +var _ = RegisterProvider(Provider{ + UniqueID: DefaultKMSType, + Initializer: newSecretsKMS, +}) + +// newSecretsKMS initializes a SecretsKMS that uses the passphrase from the // secret that is configured for the StorageClass. This KMS provider uses a // single (LUKS) passhprase for all volumes. -func initSecretsKMS(secrets map[string]string) (EncryptionKMS, error) { - passphraseValue, ok := secrets[encryptionPassphraseKey] +func newSecretsKMS(args ProviderInitArgs) (EncryptionKMS, error) { + passphraseValue, ok := args.Secrets[encryptionPassphraseKey] if !ok { return nil, errors.New("missing encryption passphrase in secrets") } @@ -98,7 +100,7 @@ type SecretsMetadataKMS struct { SecretsKMS } -var _ = RegisterKMSProvider(KMSProvider{ +var _ = RegisterProvider(Provider{ UniqueID: kmsTypeSecretsMetadata, Initializer: initSecretsMetadataKMS, }) @@ -106,7 +108,7 @@ var _ = RegisterKMSProvider(KMSProvider{ // initSecretsMetadataKMS initializes a SecretsMetadataKMS that wraps a SecretsKMS, // so that the passphrase from the user provided or StorageClass secrets can be used // for encrypting/decrypting DEKs that are stored in a detached DEKStore. -func initSecretsMetadataKMS(args KMSInitializerArgs) (EncryptionKMS, error) { +func initSecretsMetadataKMS(args ProviderInitArgs) (EncryptionKMS, error) { var ( smKMS SecretsMetadataKMS encryptionPassphrase string @@ -179,7 +181,7 @@ func (kms SecretsMetadataKMS) Destroy() { kms.SecretsKMS.Destroy() } -func (kms SecretsMetadataKMS) requiresDEKStore() DEKStoreType { +func (kms SecretsMetadataKMS) RequiresDEKStore() DEKStoreType { return DEKStoreMetadata } diff --git a/internal/util/secretskms_test.go b/internal/kms/secretskms_test.go similarity index 81% rename from internal/util/secretskms_test.go rename to internal/kms/secretskms_test.go index f27dba7fd..e8a9c326f 100644 --- a/internal/util/secretskms_test.go +++ b/internal/kms/secretskms_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package util +package kms import ( "testing" @@ -23,6 +23,26 @@ import ( "github.com/stretchr/testify/require" ) +func TestNewSecretsKMS(t *testing.T) { + t.Parallel() + secrets := map[string]string{} + + // no passphrase in the secrets, should fail + kms, err := newSecretsKMS(ProviderInitArgs{ + Secrets: secrets, + }) + assert.Error(t, err) + assert.Nil(t, kms) + + // set a passphrase and it should pass + secrets[encryptionPassphraseKey] = "plaintext encryption key" + kms, err = newSecretsKMS(ProviderInitArgs{ + Secrets: secrets, + }) + assert.NotNil(t, kms) + assert.NoError(t, err) +} + func TestGenerateNonce(t *testing.T) { t.Parallel() size := 64 @@ -44,7 +64,7 @@ func TestGenerateCipher(t *testing.T) { func TestInitSecretsMetadataKMS(t *testing.T) { t.Parallel() - args := KMSInitializerArgs{ + args := ProviderInitArgs{ Tenant: "tenant", Config: nil, Secrets: map[string]string{}, @@ -61,7 +81,7 @@ func TestInitSecretsMetadataKMS(t *testing.T) { kms, err = initSecretsMetadataKMS(args) assert.NoError(t, err) require.NotNil(t, kms) - assert.Equal(t, DEKStoreMetadata, kms.requiresDEKStore()) + assert.Equal(t, DEKStoreMetadata, kms.RequiresDEKStore()) } func TestWorkflowSecretsMetadataKMS(t *testing.T) { @@ -69,7 +89,7 @@ func TestWorkflowSecretsMetadataKMS(t *testing.T) { secrets := map[string]string{ encryptionPassphraseKey: "my-passphrase-from-kubernetes", } - args := KMSInitializerArgs{ + args := ProviderInitArgs{ Tenant: "tenant", Config: nil, Secrets: secrets, @@ -81,9 +101,7 @@ func TestWorkflowSecretsMetadataKMS(t *testing.T) { require.NotNil(t, kms) // plainDEK is the (LUKS) passphrase for the volume - plainDEK, err := generateNewEncryptionPassphrase() - assert.NoError(t, err) - assert.NotEqual(t, "", plainDEK) + plainDEK := "usually created with generateNewEncryptionPassphrase()" encryptedDEK, err := kms.EncryptDEK(volumeID, plainDEK) assert.NoError(t, err) diff --git a/internal/util/vault.go b/internal/kms/vault.go similarity index 99% rename from internal/util/vault.go rename to internal/kms/vault.go index bb095d362..6a7fc001c 100644 --- a/internal/util/vault.go +++ b/internal/kms/vault.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package util +package kms import ( "errors" @@ -89,7 +89,7 @@ type vaultConnection struct { type VaultKMS struct { vaultConnection - integratedDEK + IntegratedDEK // vaultPassphrasePath (VPP) used to be added before the "key" of the // secret (like /v1/secret/data//key) @@ -317,13 +317,13 @@ func (vc *vaultConnection) getDeleteKeyContext() map[string]string { return keyContext } -var _ = RegisterKMSProvider(KMSProvider{ +var _ = RegisterProvider(Provider{ UniqueID: kmsTypeVault, Initializer: initVaultKMS, }) // InitVaultKMS returns an interface to HashiCorp Vault KMS. -func initVaultKMS(args KMSInitializerArgs) (EncryptionKMS, error) { +func initVaultKMS(args ProviderInitArgs) (EncryptionKMS, error) { kms := &VaultKMS{} err := kms.initConnection(args.Config) if err != nil { diff --git a/internal/util/vault_sa.go b/internal/kms/vault_sa.go similarity index 98% rename from internal/util/vault_sa.go rename to internal/kms/vault_sa.go index c8940ea03..11ec2a3b5 100644 --- a/internal/util/vault_sa.go +++ b/internal/kms/vault_sa.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package util +package kms import ( "context" @@ -77,14 +77,14 @@ type VaultTenantSA struct { saTokenDir string } -var _ = RegisterKMSProvider(KMSProvider{ +var _ = RegisterProvider(Provider{ UniqueID: kmsTypeVaultTenantSA, Initializer: initVaultTenantSA, }) // initVaultTenantSA returns an interface to HashiCorp Vault KMS where Tenants // use their ServiceAccount to access the service. -func initVaultTenantSA(args KMSInitializerArgs) (EncryptionKMS, error) { +func initVaultTenantSA(args ProviderInitArgs) (EncryptionKMS, error) { var err error config := args.Config diff --git a/internal/util/vault_sa_test.go b/internal/kms/vault_sa_test.go similarity index 99% rename from internal/util/vault_sa_test.go rename to internal/kms/vault_sa_test.go index 789d42b3f..5778f2aba 100644 --- a/internal/util/vault_sa_test.go +++ b/internal/kms/vault_sa_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package util +package kms import ( "errors" diff --git a/internal/util/vault_test.go b/internal/kms/vault_test.go similarity index 99% rename from internal/util/vault_test.go rename to internal/kms/vault_test.go index 93caf707b..70abbbb2c 100644 --- a/internal/util/vault_test.go +++ b/internal/kms/vault_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package util +package kms import ( "errors" diff --git a/internal/util/vault_tokens.go b/internal/kms/vault_tokens.go similarity index 99% rename from internal/util/vault_tokens.go rename to internal/kms/vault_tokens.go index 57f8e286d..bae918aa9 100644 --- a/internal/util/vault_tokens.go +++ b/internal/kms/vault_tokens.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package util +package kms import ( "context" @@ -181,7 +181,7 @@ Example JSON structure in the KMS config is, */ type vaultTenantConnection struct { vaultConnection - integratedDEK + IntegratedDEK client *kubernetes.Clientset @@ -204,13 +204,13 @@ type VaultTokensKMS struct { TokenName string } -var _ = RegisterKMSProvider(KMSProvider{ +var _ = RegisterProvider(Provider{ UniqueID: kmsTypeVaultTokens, Initializer: initVaultTokensKMS, }) // InitVaultTokensKMS returns an interface to HashiCorp Vault KMS. -func initVaultTokensKMS(args KMSInitializerArgs) (EncryptionKMS, error) { +func initVaultTokensKMS(args ProviderInitArgs) (EncryptionKMS, error) { var err error config := args.Config diff --git a/internal/util/vault_tokens_test.go b/internal/kms/vault_tokens_test.go similarity index 99% rename from internal/util/vault_tokens_test.go rename to internal/kms/vault_tokens_test.go index cddd8dd0a..df1e105f0 100644 --- a/internal/util/vault_tokens_test.go +++ b/internal/kms/vault_tokens_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package util +package kms import ( "encoding/json" @@ -76,7 +76,7 @@ func TestInitVaultTokensKMS(t *testing.T) { return } - args := KMSInitializerArgs{ + args := ProviderInitArgs{ Tenant: "bob", Config: make(map[string]interface{}), Secrets: nil, diff --git a/internal/rbd/encryption.go b/internal/rbd/encryption.go index 99d35469f..c28aa532c 100644 --- a/internal/rbd/encryption.go +++ b/internal/rbd/encryption.go @@ -22,6 +22,7 @@ import ( "fmt" "strings" + kmsapi "github.com/ceph/ceph-csi/internal/kms" "github.com/ceph/ceph-csi/internal/util" "github.com/ceph/ceph-csi/internal/util/log" @@ -289,7 +290,7 @@ func (ri *rbdImage) ParseEncryptionOpts(ctx context.Context, volOptions map[stri // configureEncryption sets up the VolumeEncryption for this rbdImage. Once // configured, use isEncrypted() to see if the volume supports encryption. func (ri *rbdImage) configureEncryption(kmsID string, credentials map[string]string) error { - kms, err := util.GetKMS(ri.Owner, kmsID, credentials) + kms, err := kmsapi.GetKMS(ri.Owner, kmsID, credentials) if err != nil { return err } diff --git a/internal/util/crypto.go b/internal/util/crypto.go index 79821e328..d04ab2cf7 100644 --- a/internal/util/crypto.go +++ b/internal/util/crypto.go @@ -26,6 +26,7 @@ import ( "strconv" "strings" + "github.com/ceph/ceph-csi/internal/kms" "github.com/ceph/ceph-csi/internal/util/log" ) @@ -33,9 +34,6 @@ const ( mapperFilePrefix = "luks-rbd-" mapperFilePathPrefix = "/dev/mapper" - // kmsConfigPath is the location of the vault config file. - kmsConfigPath = "/etc/ceph-csi-encryption-kms-config/config.json" - // Passphrase size - 20 bytes is 160 bits to satisfy: // https://tools.ietf.org/html/rfc6749#section-10.10 encryptionPassphraseSize = 20 @@ -54,11 +52,11 @@ var ( ) type VolumeEncryption struct { - KMS EncryptionKMS + KMS kms.EncryptionKMS // dekStore that will be used, this can be the EncryptionKMS or a // different object implementing the DEKStore interface. - dekStore DEKStore + dekStore kms.DEKStore id string } @@ -76,7 +74,7 @@ func FetchEncryptionKMSID(encrypted, kmsID string) (string, error) { } if kmsID == "" { - kmsID = defaultKMSType + kmsID = kms.DefaultKMSType } return kmsID, nil @@ -88,24 +86,24 @@ func FetchEncryptionKMSID(encrypted, kmsID string) (string, error) { // Callers that receive a ErrDEKStoreNeeded error, should use // VolumeEncryption.SetDEKStore() to configure an alternative storage for the // DEKs. -func NewVolumeEncryption(id string, kms EncryptionKMS) (*VolumeEncryption, error) { +func NewVolumeEncryption(id string, ekms kms.EncryptionKMS) (*VolumeEncryption, error) { kmsID := id if kmsID == "" { // if kmsID is not set, encryption is enabled, and the type is // SecretsKMS - kmsID = defaultKMSType + kmsID = kms.DefaultKMSType } ve := &VolumeEncryption{ id: kmsID, - KMS: kms, + KMS: ekms, } - if kms.requiresDEKStore() == DEKStoreIntegrated { - dekStore, ok := kms.(DEKStore) + if ekms.RequiresDEKStore() == kms.DEKStoreIntegrated { + dekStore, ok := ekms.(kms.DEKStore) if !ok { return nil, fmt.Errorf("KMS %T does not implement the "+ - "DEKStore interface", kms) + "DEKStore interface", ekms) } ve.dekStore = dekStore @@ -118,7 +116,7 @@ func NewVolumeEncryption(id string, kms EncryptionKMS) (*VolumeEncryption, error // SetDEKStore sets the DEKStore for this VolumeEncryption instance. It will be // used when StoreNewCryptoPassphrase() or RemoveDEK() is called. -func (ve *VolumeEncryption) SetDEKStore(dekStore DEKStore) { +func (ve *VolumeEncryption) SetDEKStore(dekStore kms.DEKStore) { ve.dekStore = dekStore } @@ -141,72 +139,6 @@ func (ve *VolumeEncryption) GetID() string { return ve.id } -// EncryptionKMS provides external Key Management System for encryption -// passphrases storage. -type EncryptionKMS interface { - Destroy() - - // requiresDEKStore returns the DEKStoreType that is needed to be - // configure for the KMS. Nothing needs to be done when this function - // returns DEKStoreIntegrated, otherwise you will need to configure an - // alternative storage for the DEKs. - requiresDEKStore() DEKStoreType - - // EncryptDEK provides a way for a KMS to encrypt a DEK. In case the - // encryption is done transparently inside the KMS service, the - // function can return an unencrypted value. - EncryptDEK(volumeID, plainDEK string) (string, error) - - // DecryptDEK provides a way for a KMS to decrypt a DEK. In case the - // encryption is done transparently inside the KMS service, the - // function does not need to do anything except return the encyptedDEK - // as it was received. - DecryptDEK(volumeID, encyptedDEK string) (string, error) -} - -// DEKStoreType describes what DEKStore needs to be configured when using a -// particular KMS. A KMS might support different DEKStores depending on its -// configuration. -type DEKStoreType string - -const ( - // DEKStoreIntegrated indicates that the KMS itself supports storing - // DEKs. - DEKStoreIntegrated = DEKStoreType("") - // DEKStoreMetadata indicates that the KMS should be configured to - // store the DEK in the metadata of the volume. - DEKStoreMetadata = DEKStoreType("metadata") -) - -// DEKStore allows KMS instances to implement a modular backend for DEK -// storage. This can be used to store the DEK in a different location, in case -// the KMS can not store passphrases for volumes. -type DEKStore interface { - // StoreDEK saves the DEK in the configured store. - StoreDEK(volumeID string, dek string) error - // FetchDEK reads the DEK from the configured store and returns it. - FetchDEK(volumeID string) (string, error) - // RemoveDEK deletes the DEK from the configured store. - RemoveDEK(volumeID string) error -} - -// integratedDEK is a DEKStore that can not be configured. Either the KMS does -// not use a DEK, or the DEK is stored in the KMS without additional -// configuration options. -type integratedDEK struct{} - -func (i integratedDEK) requiresDEKStore() DEKStoreType { - return DEKStoreIntegrated -} - -func (i integratedDEK) EncryptDEK(volumeID, plainDEK string) (string, error) { - return plainDEK, nil -} - -func (i integratedDEK) DecryptDEK(volumeID, encyptedDEK string) (string, error) { - return encyptedDEK, nil -} - // StoreCryptoPassphrase takes an unencrypted passphrase, encrypts it and saves // it in the DEKStore. func (ve *VolumeEncryption) StoreCryptoPassphrase(volumeID, passphrase string) error { diff --git a/internal/util/crypto_test.go b/internal/util/crypto_test.go index d063c8d5a..28b8fefea 100644 --- a/internal/util/crypto_test.go +++ b/internal/util/crypto_test.go @@ -20,26 +20,12 @@ import ( "encoding/base64" "testing" + "github.com/ceph/ceph-csi/internal/kms" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestInitSecretsKMS(t *testing.T) { - t.Parallel() - secrets := map[string]string{} - - // no passphrase in the secrets, should fail - kms, err := initSecretsKMS(secrets) - assert.Error(t, err) - assert.Nil(t, kms) - - // set a passphrase and it should pass - secrets[encryptionPassphraseKey] = "plaintext encryption key" - kms, err = initSecretsKMS(secrets) - assert.NotNil(t, kms) - assert.NoError(t, err) -} - func TestGenerateNewEncryptionPassphrase(t *testing.T) { t.Parallel() b64Passphrase, err := generateNewEncryptionPassphrase() @@ -55,17 +41,18 @@ func TestGenerateNewEncryptionPassphrase(t *testing.T) { func TestKMSWorkflow(t *testing.T) { t.Parallel() secrets := map[string]string{ - encryptionPassphraseKey: "workflow test", + // FIXME: use encryptionPassphraseKey from SecretsKMS + "encryptionPassphrase": "workflow test", } - kms, err := GetKMS("tenant", defaultKMSType, secrets) + kmsProvider, err := kms.GetDefaultKMS(secrets) assert.NoError(t, err) - require.NotNil(t, kms) + require.NotNil(t, kmsProvider) - ve, err := NewVolumeEncryption("", kms) + ve, err := NewVolumeEncryption("", kmsProvider) assert.NoError(t, err) require.NotNil(t, ve) - assert.Equal(t, defaultKMSType, ve.GetID()) + assert.Equal(t, kms.DefaultKMSType, ve.GetID()) volumeID := "volume-id" @@ -74,5 +61,5 @@ func TestKMSWorkflow(t *testing.T) { passphrase, err := ve.GetCryptoPassphrase(volumeID) assert.NoError(t, err) - assert.Equal(t, secrets[encryptionPassphraseKey], passphrase) + assert.Equal(t, secrets["encryptionPassphrase"], passphrase) } diff --git a/internal/util/util.go b/internal/util/util.go index 62dcf2b2f..17a2836d1 100644 --- a/internal/util/util.go +++ b/internal/util/util.go @@ -344,20 +344,6 @@ func contains(s []string, key string) bool { return false } -// getKeys takes a map that uses strings for keys and returns a slice with the -// keys. -func getKeys(m map[string]interface{}) []string { - keys := make([]string, len(m)) - - i := 0 - for k := range m { - keys[i] = k - i++ - } - - return keys -} - // CallStack returns the stack of the calls in the current goroutine. Useful // for debugging or reporting errors. This is a friendly alternative to // assert() or panic(). From 92e7f16654c9e9128c48939ad30ad888dfc56443 Mon Sep 17 00:00:00 2001 From: Humble Chirammal Date: Mon, 30 Aug 2021 16:14:30 +0530 Subject: [PATCH 13/17] ci: remove the rules for backport on unsupported versions CSI version v3.3.0 and above are the supported versions now and this patch also adjust the mergify rules according to that, ie the backport rules are removed for all unsupported versions here. Signed-off-by: Humble Chirammal --- .mergify.yml | 158 --------------------------------------------------- 1 file changed, 158 deletions(-) diff --git a/.mergify.yml b/.mergify.yml index 83fbb14d7..63e4e23b8 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -89,164 +89,6 @@ pull_request_rules: merge: {} dismiss_reviews: {} delete_head_branch: {} - - name: backport patches to release v1.2.0 branch - conditions: - - base=devel - - label=backport-to-release-v1.2.0 - actions: - backport: - branches: - - release-v1.2.0 - # automerge backports if CI successfully ran - - name: automerge backport release-v1.2.0 - conditions: - - author=mergify[bot] - - base=release-v1.2.0 - - label!=DNM - - "#changes-requested-reviews-by=0" - - "#approved-reviews-by>=1" - actions: - merge: {} - dismiss_reviews: {} - delete_head_branch: {} - - name: backport patches to release-v2.0 branch - conditions: - - base=devel - - label=backport-to-release-v2.0 - actions: - backport: - branches: - - release-v2.0 - # automerge backports if CI successfully ran - - name: automerge backport release-v2.0 - conditions: - - author=mergify[bot] - - base=release-v2.0 - - label!=DNM - - "#changes-requested-reviews-by=0" - - "#approved-reviews-by>=2" - - "approved-reviews-by=@ceph/ceph-csi-contributors" - - "approved-reviews-by=@ceph/ceph-csi-maintainers" - actions: - merge: {} - dismiss_reviews: {} - delete_head_branch: {} - - name: backport patches to release-v2.1 branch - conditions: - - base=devel - - label=backport-to-release-v2.1 - actions: - backport: - branches: - - release-v2.1 - # automerge backports if CI successfully ran - - name: automerge backport release-v2.1 - conditions: - - author=mergify[bot] - - base=release-v2.1 - - label!=DNM - - "#changes-requested-reviews-by=0" - - "#approved-reviews-by>=2" - - "approved-reviews-by=@ceph/ceph-csi-contributors" - - "approved-reviews-by=@ceph/ceph-csi-maintainers" - actions: - merge: {} - dismiss_reviews: {} - delete_head_branch: {} - - name: backport patches to release-v3.0 branch - conditions: - - base=devel - - label=backport-to-release-v3.0 - actions: - backport: - branches: - - release-v3.0 - # automerge backports if CI successfully ran - - name: automerge backport release-v3.0 - conditions: - - author=mergify[bot] - - base=release-v3.0 - - label!=DNM - - "#changes-requested-reviews-by=0" - - "#approved-reviews-by>=2" - - "approved-reviews-by=@ceph/ceph-csi-contributors" - - "approved-reviews-by=@ceph/ceph-csi-maintainers" - actions: - merge: {} - dismiss_reviews: {} - delete_head_branch: {} - - name: backport patches to release-v3.1 branch - conditions: - - base=devel - - label=backport-to-release-v3.1 - actions: - backport: - branches: - - release-v3.1 - # automerge backports if CI successfully ran - - name: automerge backport release-v3.1 - conditions: - - author=mergify[bot] - - base=release-v3.1 - - label!=DNM - - "#changes-requested-reviews-by=0" - - "#approved-reviews-by>=2" - - "approved-reviews-by=@ceph/ceph-csi-contributors" - - "approved-reviews-by=@ceph/ceph-csi-maintainers" - - "status-success=multi-arch-build" - - "status-success=commitlint" - - "status-success=ci/centos/mini-e2e-helm/k8s-1.20" - - "status-success=ci/centos/mini-e2e-helm/k8s-1.21" - - "status-success=ci/centos/mini-e2e-helm/k8s-1.22" - - "status-success=ci/centos/mini-e2e/k8s-1.20" - - "status-success=ci/centos/mini-e2e/k8s-1.21" - - "status-success=ci/centos/mini-e2e/k8s-1.22" - - "status-success=ci/centos/upgrade-tests-cephfs" - - "status-success=ci/centos/upgrade-tests-rbd" - - "status-success=DCO" - actions: - merge: {} - dismiss_reviews: {} - delete_head_branch: {} - - name: backport patches to release-v3.2 branch - conditions: - - base=devel - - label=backport-to-release-v3.2 - actions: - backport: - branches: - - release-v3.2 - # automerge backports if CI successfully ran - - name: automerge backport release-v3.2 - conditions: - - author=mergify[bot] - - base=release-v3.2 - - label!=DNM - - "#approved-reviews-by>=2" - - "approved-reviews-by=@ceph/ceph-csi-contributors" - - "approved-reviews-by=@ceph/ceph-csi-maintainers" - - "status-success=codespell" - - "status-success=multi-arch-build" - - "status-success=go-test" - - "status-success=golangci-lint" - - "status-success=gosec" - - "status-success=commitlint" - - "status-success=mod-check" - - "status-success=lint-extras" - - "#changes-requested-reviews-by=0" - - "status-success=ci/centos/mini-e2e-helm/k8s-1.20" - - "status-success=ci/centos/mini-e2e-helm/k8s-1.21" - - "status-success=ci/centos/mini-e2e-helm/k8s-1.22" - - "status-success=ci/centos/mini-e2e/k8s-1.20" - - "status-success=ci/centos/mini-e2e/k8s-1.21" - - "status-success=ci/centos/mini-e2e/k8s-1.22" - - "status-success=ci/centos/upgrade-tests-cephfs" - - "status-success=ci/centos/upgrade-tests-rbd" - - "status-success=DCO" - actions: - merge: {} - dismiss_reviews: {} - delete_head_branch: {} - name: backport patches to release-v3.3 branch conditions: - base=devel From 1b30a58e5345c6948871472a062724b6773bf636 Mon Sep 17 00:00:00 2001 From: Niels de Vos Date: Mon, 30 Aug 2021 17:08:19 +0200 Subject: [PATCH 14/17] ci: skip commitlint for dependabot PRs Dependabot creates commit messages that commitlint fails to parse correctly. The messages contain simple references to which dependencies are updated, and do not need full validation like manual written messages do. Signed-off-by: Niels de Vos --- .mergify.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/.mergify.yml b/.mergify.yml index 63e4e23b8..1eb195bec 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -32,6 +32,35 @@ pull_request_rules: message: "This pull request now has conflicts with the target branch. Could you please resolve conflicts and force push the corrected changes? 🙏" + - name: update dependencies by dependabot (skip commitlint) + conditions: + - author=dependabot[bot] + - label!=DNM + - base~=^(devel)|(release-.+)$ + - "#approved-reviews-by>=2" + - "#changes-requested-reviews-by=0" + - "approved-reviews-by=@ceph/ceph-csi-contributors" + - "approved-reviews-by=@ceph/ceph-csi-maintainers" + - "status-success=codespell" + - "status-success=multi-arch-build" + - "status-success=go-test" + - "status-success=golangci-lint" + - "status-success=gosec" + - "status-success=mod-check" + - "status-success=lint-extras" + - "status-success=ci/centos/mini-e2e-helm/k8s-1.20" + - "status-success=ci/centos/mini-e2e-helm/k8s-1.21" + - "status-success=ci/centos/mini-e2e-helm/k8s-1.22" + - "status-success=ci/centos/mini-e2e/k8s-1.20" + - "status-success=ci/centos/mini-e2e/k8s-1.21" + - "status-success=ci/centos/mini-e2e/k8s-1.22" + - "status-success=ci/centos/upgrade-tests-cephfs" + - "status-success=ci/centos/upgrade-tests-rbd" + - "status-success=DCO" + actions: + merge: {} + dismiss_reviews: {} + delete_head_branch: {} - name: automatic merge conditions: - label!=DNM From e08d184984f8c77c9293808513fd176bca173b1a Mon Sep 17 00:00:00 2001 From: Niels de Vos Date: Tue, 31 Aug 2021 09:28:02 +0200 Subject: [PATCH 15/17] ci: ignore k8s.io/kubernetes dependencies These dependencies are pulled in by k8s.io/kubernetes with version v0.0.0. It is therefore required to use 'replace' in go.mod to select a compatible version of the additional k8s.io packages. Dependabot does not seem to update packages listed in 'replace', only under 'require'. That means, the version updates done by Dependabot do not have any effect, as the contents is replaced with a different version anyway. Ignoring these packages prevents the creation of non-functional PRs. Signed-off-by: Niels de Vos --- .github/dependabot.yml | 27 +++++++++++++++++++++++++++ go.mod | 3 +++ 2 files changed, 30 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index d9a772676..577cc6182 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -9,3 +9,30 @@ updates: - rebase commit-message: prefix: "rebase" + ignore: + # k8s.io/kubernetes prevents auto-updating these + - dependency-name: "k8s.io/api" + - dependency-name: "k8s.io/apiextensions-apiserver" + - dependency-name: "k8s.io/apimachinery" + - dependency-name: "k8s.io/apiserver" + - dependency-name: "k8s.io/cli-runtime" + - dependency-name: "k8s.io/client-go" + - dependency-name: "k8s.io/cloud-provider" + - dependency-name: "k8s.io/cluster-bootstrap" + - dependency-name: "k8s.io/code-generator" + - dependency-name: "k8s.io/component-base" + - dependency-name: "k8s.io/component-helpers" + - dependency-name: "k8s.io/controller-manager" + - dependency-name: "k8s.io/cri-api" + - dependency-name: "k8s.io/csi-translation-lib" + - dependency-name: "k8s.io/kube-aggregator" + - dependency-name: "k8s.io/kube-controller-manager" + - dependency-name: "k8s.io/kube-proxy" + - dependency-name: "k8s.io/kube-scheduler" + - dependency-name: "k8s.io/kubectl" + - dependency-name: "k8s.io/kubelet" + - dependency-name: "k8s.io/legacy-cloud-providers" + - dependency-name: "k8s.io/metrics" + - dependency-name: "k8s.io/mount-utils" + - dependency-name: "k8s.io/pod-security-admission" + - dependency-name: "k8s.io/sample-apiserver" diff --git a/go.mod b/go.mod index 516081f77..0f873f295 100644 --- a/go.mod +++ b/go.mod @@ -42,6 +42,9 @@ replace ( github.com/portworx/sched-ops => github.com/portworx/sched-ops v0.20.4-openstorage-rc3 gomodules.xyz/jsonpatch/v2 => github.com/gomodules/jsonpatch/v2 v2.2.0 google.golang.org/grpc => google.golang.org/grpc v1.35.0 + // + // k8s.io/kubernetes depends on these k8s.io packages, but unversioned + // k8s.io/api => k8s.io/api v0.22.0 k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.22.0 k8s.io/apimachinery => k8s.io/apimachinery v0.22.0 From 496bcba85c8fc1b4a00b71872cb0f560036180fe Mon Sep 17 00:00:00 2001 From: Rakshith R Date: Mon, 30 Aug 2021 10:17:45 +0530 Subject: [PATCH 16/17] rbd: move GetMappedID() to util package This commit moves getMappedID() from rbd to util package since it is not rbd specific and exports it from there. Signed-off-by: Rakshith R --- internal/rbd/rbd_util.go | 18 ++-------- internal/rbd/rbd_util_test.go | 52 --------------------------- internal/util/cluster_mapping.go | 14 ++++++++ internal/util/cluster_mapping_test.go | 52 +++++++++++++++++++++++++++ 4 files changed, 68 insertions(+), 68 deletions(-) diff --git a/internal/rbd/rbd_util.go b/internal/rbd/rbd_util.go index 731066f2e..d11169fcc 100644 --- a/internal/rbd/rbd_util.go +++ b/internal/rbd/rbd_util.go @@ -1105,7 +1105,7 @@ func generateVolumeFromMapping( // extract clusterID mapping for _, cm := range *mapping { for key, val := range cm.ClusterIDMapping { - mappedClusterID := getMappedID(key, val, vi.ClusterID) + mappedClusterID := util.GetMappedID(key, val, vi.ClusterID) if mappedClusterID == "" { continue } @@ -1119,7 +1119,7 @@ func generateVolumeFromMapping( poolID := fmt.Sprintf("%d", (vi.LocationID)) for _, pools := range cm.RBDpoolIDMappingInfo { for key, val := range pools { - mappedPoolID := getMappedID(key, val, poolID) + mappedPoolID := util.GetMappedID(key, val, poolID) if mappedPoolID == "" { continue } @@ -1146,20 +1146,6 @@ func generateVolumeFromMapping( return vol, util.ErrPoolNotFound } -// getMappedID check the input id is matching key or value. -// If key==id the value will be returned. -// If value==id the key will be returned. -func getMappedID(key, value, id string) string { - if key == id { - return value - } - if value == id { - return key - } - - return "" -} - func genVolFromVolumeOptions( ctx context.Context, volOptions, credentials map[string]string, diff --git a/internal/rbd/rbd_util_test.go b/internal/rbd/rbd_util_test.go index 2a44682db..a121daaf8 100644 --- a/internal/rbd/rbd_util_test.go +++ b/internal/rbd/rbd_util_test.go @@ -138,58 +138,6 @@ func TestValidateImageFeatures(t *testing.T) { } } -func TestGetMappedID(t *testing.T) { - t.Parallel() - type args struct { - key string - value string - id string - } - tests := []struct { - name string - args args - expected string - }{ - { - name: "test for matching key", - args: args{ - key: "cluster1", - value: "cluster2", - id: "cluster1", - }, - expected: "cluster2", - }, - { - name: "test for matching value", - args: args{ - key: "cluster1", - value: "cluster2", - id: "cluster2", - }, - expected: "cluster1", - }, - { - name: "test for invalid match", - args: args{ - key: "cluster1", - value: "cluster2", - id: "cluster3", - }, - expected: "", - }, - } - for _, tt := range tests { - tt := tt - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - val := getMappedID(tt.args.key, tt.args.value, tt.args.id) - if val != tt.expected { - t.Errorf("getMappedID() got = %v, expected %v", val, tt.expected) - } - }) - } -} - func TestGetCephClientLogFileName(t *testing.T) { t.Parallel() type args struct { diff --git a/internal/util/cluster_mapping.go b/internal/util/cluster_mapping.go index 173f5126e..9d3b76b13 100644 --- a/internal/util/cluster_mapping.go +++ b/internal/util/cluster_mapping.go @@ -120,3 +120,17 @@ func getClusterMappingInfo(clusterID, filename string) (*[]ClusterMappingInfo, e func GetClusterMappingInfo(clusterID string) (*[]ClusterMappingInfo, error) { return getClusterMappingInfo(clusterID, clusterMappingConfigFile) } + +// GetMappedID check the input id is matching key or value. +// If key==id the value will be returned. +// If value==id the key will be returned. +func GetMappedID(key, value, id string) string { + if key == id { + return value + } + if value == id { + return key + } + + return "" +} diff --git a/internal/util/cluster_mapping_test.go b/internal/util/cluster_mapping_test.go index 2a85ed9ef..d783d9e47 100644 --- a/internal/util/cluster_mapping_test.go +++ b/internal/util/cluster_mapping_test.go @@ -239,3 +239,55 @@ func validateMapping(t *testing.T, clusterID, rbdPoolID, cephFSPoolID string, ma return nil } + +func TestGetMappedID(t *testing.T) { + t.Parallel() + type args struct { + key string + value string + id string + } + tests := []struct { + name string + args args + expected string + }{ + { + name: "test for matching key", + args: args{ + key: "cluster1", + value: "cluster2", + id: "cluster1", + }, + expected: "cluster2", + }, + { + name: "test for matching value", + args: args{ + key: "cluster1", + value: "cluster2", + id: "cluster2", + }, + expected: "cluster1", + }, + { + name: "test for invalid match", + args: args{ + key: "cluster1", + value: "cluster2", + id: "cluster3", + }, + expected: "", + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + val := GetMappedID(tt.args.key, tt.args.value, tt.args.id) + if val != tt.expected { + t.Errorf("getMappedID() got = %v, expected %v", val, tt.expected) + } + }) + } +} From 99168dc822a4c81f4a12dcf00a7165e7426594ce Mon Sep 17 00:00:00 2001 From: Rakshith R Date: Thu, 26 Aug 2021 15:51:29 +0530 Subject: [PATCH 17/17] rbd: check for clusterid mapping in RegenerateJournal() This commit adds fetchMappedClusterIDAndMons() which returns monitors and clusterID info after checking cluster mapping info. This is required for regenerating omap entries in mirrored cluster with different clusterID. Signed-off-by: Rakshith R --- internal/rbd/rbd_journal.go | 21 ++-- internal/util/cluster_mapping.go | 53 +++++++++++ internal/util/cluster_mapping_test.go | 132 ++++++++++++++++++++++++++ 3 files changed, 192 insertions(+), 14 deletions(-) diff --git a/internal/rbd/rbd_journal.go b/internal/rbd/rbd_journal.go index 7408b7bdf..a61257d77 100644 --- a/internal/rbd/rbd_journal.go +++ b/internal/rbd/rbd_journal.go @@ -523,6 +523,7 @@ func undoVolReservation(ctx context.Context, rbdVol *rbdVolume, cr *util.Credent // complete omap mapping between imageName and volumeID. // RegenerateJournal performs below operations +// Extract clusterID, Mons after checkig clusterID mapping // Extract parameters journalPool, pool from volumeAttributes // Extract optional parameters volumeNamePrefix, kmsID, owner from volumeAttributes // Extract information from volumeID @@ -538,15 +539,13 @@ func RegenerateJournal( cr *util.Credentials) (string, error) { ctx := context.Background() var ( - options map[string]string - vi util.CSIIdentifier - rbdVol *rbdVolume - kmsID string - err error - ok bool + vi util.CSIIdentifier + rbdVol *rbdVolume + kmsID string + err error + ok bool ) - options = make(map[string]string) rbdVol = &rbdVolume{} rbdVol.VolID = volumeID @@ -561,14 +560,8 @@ func RegenerateJournal( return "", err } - // TODO check clusterID mapping exists - rbdVol.ClusterID = vi.ClusterID - options["clusterID"] = rbdVol.ClusterID - - rbdVol.Monitors, _, err = util.GetMonsAndClusterID(options) + rbdVol.Monitors, rbdVol.ClusterID, err = util.FetchMappedClusterIDAndMons(ctx, vi.ClusterID) if err != nil { - log.ErrorLog(ctx, "failed getting mons (%s)", err) - return "", err } diff --git a/internal/util/cluster_mapping.go b/internal/util/cluster_mapping.go index 9d3b76b13..37155f623 100644 --- a/internal/util/cluster_mapping.go +++ b/internal/util/cluster_mapping.go @@ -17,11 +17,14 @@ limitations under the License. package util import ( + "context" "encoding/json" "errors" "fmt" "io/ioutil" "os" + + "github.com/ceph/ceph-csi/internal/util/log" ) // clusterMappingConfigFile is the location of the cluster mapping config file. @@ -134,3 +137,53 @@ func GetMappedID(key, value, id string) string { return "" } + +// fetchMappedClusterIDAndMons returns monitors and clusterID info after checking cluster mapping. +func fetchMappedClusterIDAndMons(ctx context.Context, + clusterID, clusterMappingConfigFile, csiConfigFile string) (string, string, error) { + var mons string + clusterMappingInfo, err := getClusterMappingInfo(clusterID, clusterMappingConfigFile) + if err != nil { + return "", "", err + } + + if clusterMappingInfo != nil { + for _, cm := range *clusterMappingInfo { + for key, val := range cm.ClusterIDMapping { + mappedClusterID := GetMappedID(key, val, clusterID) + if mappedClusterID == "" { + continue + } + log.DebugLog(ctx, + "found new clusterID mapping %q for existing clusterID %q", + mappedClusterID, + clusterID) + + mons, err = Mons(csiConfigFile, mappedClusterID) + if err != nil { + log.DebugLog(ctx, "failed getting mons with mapped cluster id %q: %v", + mappedClusterID, err) + + continue + } + + return mons, mappedClusterID, nil + } + } + } + + // check original clusterID for backward compatibility when cluster ids were expected to be same. + mons, err = Mons(csiConfigFile, clusterID) + if err != nil { + log.ErrorLog(ctx, "failed getting mons with cluster id %q: %v", clusterID, err) + + return "", "", err + } + + return mons, clusterID, err +} + +// FetchMappedClusterIDAndMons returns monitors and clusterID info after checking cluster mapping. +func FetchMappedClusterIDAndMons(ctx context.Context, clusterID string) (string, string, error) { + return fetchMappedClusterIDAndMons(ctx, clusterID, clusterMappingConfigFile, CsiConfigFile) +} diff --git a/internal/util/cluster_mapping_test.go b/internal/util/cluster_mapping_test.go index d783d9e47..ded989e35 100644 --- a/internal/util/cluster_mapping_test.go +++ b/internal/util/cluster_mapping_test.go @@ -17,10 +17,12 @@ limitations under the License. package util import ( + "context" "encoding/json" "fmt" "io/ioutil" "reflect" + "strings" "testing" ) @@ -291,3 +293,133 @@ func TestGetMappedID(t *testing.T) { }) } } + +func TestFetchMappedClusterIDAndMons(t *testing.T) { + t.Parallel() + ctx := context.TODO() + type args struct { + ctx context.Context + clusterID string + } + mappingBasePath := t.TempDir() + csiConfigFile := mappingBasePath + "/config.json" + clusterMappingConfigFile := mappingBasePath + "/cluster-mapping.json" + csiConfig := []ClusterInfo{ + { + ClusterID: "cluster-1", + Monitors: []string{"ip-1", "ip-2"}, + }, + { + ClusterID: "cluster-2", + Monitors: []string{"ip-3", "ip-4"}, + }, + } + csiConfigFileContent, err := json.Marshal(csiConfig) + if err != nil { + t.Errorf("failed to marshal csi config info %v", err) + } + err = ioutil.WriteFile(csiConfigFile, csiConfigFileContent, 0o600) + if err != nil { + t.Errorf("failed to write %s file content: %v", CsiConfigFile, err) + } + + t.Run("cluster-mapping.json does not exist", func(t *testing.T) { + _, _, err = fetchMappedClusterIDAndMons(ctx, "cluster-2", clusterMappingConfigFile, csiConfigFile) + if err != nil { + t.Errorf("FetchMappedClusterIDAndMons() error = %v, wantErr %v", err, nil) + } + }) + + clusterMapping := []ClusterMappingInfo{ + { + ClusterIDMapping: map[string]string{ + "cluster-1": "cluster-3", + }, + }, + { + ClusterIDMapping: map[string]string{ + "cluster-1": "cluster-4", + }, + }, + { + ClusterIDMapping: map[string]string{ + "cluster-4": "cluster-3", + }, + }, + } + clusterMappingFileContent, err := json.Marshal(clusterMapping) + if err != nil { + t.Errorf("failed to marshal mapping info %v", err) + } + err = ioutil.WriteFile(clusterMappingConfigFile, clusterMappingFileContent, 0o600) + if err != nil { + t.Errorf("failed to write %s file content: %v", clusterMappingFileContent, err) + } + + tests := []struct { + name string + args args + want string + want1 string + wantErr bool + }{ + { + name: "test cluster id=cluster-1", + args: args{ + ctx: ctx, + clusterID: "cluster-1", + }, + want: strings.Join(csiConfig[0].Monitors, ","), + want1: "cluster-1", + wantErr: false, + }, + { + name: "test cluster id=cluster-3", + args: args{ + ctx: ctx, + clusterID: "cluster-3", + }, + want: strings.Join(csiConfig[0].Monitors, ","), + want1: "cluster-1", + wantErr: false, + }, + { + name: "test cluster id=cluster-4", + args: args{ + ctx: ctx, + clusterID: "cluster-4", + }, + want: strings.Join(csiConfig[0].Monitors, ","), + want1: "cluster-1", + wantErr: false, + }, + { + name: "test missing cluster id=cluster-6", + args: args{ + ctx: ctx, + clusterID: "cluster-6", + }, + want: "", + want1: "", + wantErr: true, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got, got1, err := fetchMappedClusterIDAndMons(ctx, tt.args.clusterID, clusterMappingConfigFile, csiConfigFile) + if (err != nil) != tt.wantErr { + t.Errorf("FetchMappedClusterIDAndMons() error = %v, wantErr %v", err, tt.wantErr) + + return + } + if got != tt.want { + t.Errorf("FetchMappedClusterIDAndMons() got = %v, want %v", got, tt.want) + } + if got1 != tt.want1 { + t.Errorf("FetchMappedClusterIDAndMons() got1 = %v, want %v", got1, tt.want1) + } + }) + } +}