From c9cc36d8dbed422ebde4231927f47d468bd2f3e3 Mon Sep 17 00:00:00 2001 From: Prasanna Kumar Kalever Date: Wed, 1 Sep 2021 17:23:43 +0530 Subject: [PATCH] rbd: provide alternatives to preserve the ceph log files Currently, we delete the ceph client log file on unmap/detach. This patch provides additional alternatives for users who would like to persist the log files. Strategies: ----------- `remove`: delete log file on unmap/detach `compress`: compress the log file to gzip on unmap/detach `preserve`: preserve the log file in text format Note that the default strategy will be remove on unmap, and these options can be tweaked from the storage class Compression size details example: On Map: (with debug-rbd=20) --------- $ ls -lh -rw-r--r-- 1 root root 526K Sep 1 18:15 rbd-nbd-0001-0024-fed5480a-f00f-417a-a51d-31d8a8144c03-0000000000000003-d2e89c87-0b4d-11ec-8ea6-160f128e682d.log On unmap: --------- $ ls -lh -rw-r--r-- 1 root root 33K Sep 1 18:15 rbd-nbd-0001-0024-fed5480a-f00f-417a-a51d-31d8a8144c03-0000000000000003-d2e89c87-0b4d-11ec-8ea6-160f128e682d.gz Signed-off-by: Prasanna Kumar Kalever --- charts/ceph-csi-rbd/README.md | 1 + .../ceph-csi-rbd/templates/storageclass.yaml | 3 + charts/ceph-csi-rbd/values.yaml | 14 +++- examples/rbd/storageclass.yaml | 13 +++- internal/rbd/nodeserver.go | 5 ++ internal/rbd/rbd_attach.go | 7 +- internal/rbd/rbd_util.go | 28 ++++++- internal/rbd/rbd_util_test.go | 75 +++++++++++++++++++ 8 files changed, 134 insertions(+), 12 deletions(-) diff --git a/charts/ceph-csi-rbd/README.md b/charts/ceph-csi-rbd/README.md index 72d1d44a5..46223c54d 100644 --- a/charts/ceph-csi-rbd/README.md +++ b/charts/ceph-csi-rbd/README.md @@ -152,6 +152,7 @@ charts and their default values. | `storageclass.imageFeatures` | Specifies RBD image features | `layering` | | `storageClass.mounter` | Specifies RBD mounter | `""` | | `storageClass.cephLogDir` | ceph client log location, it is the target bindmount path used inside container | `"/var/log/ceph"` | +| `storageClass.cephLogStrategy` | ceph client log strategy, available options `remove` or `compress` or `preserve` | `"remove"` | | `storageClass.volumeNamePrefix` | Prefix to use for naming RBD images | `""` | | `storageClass.encrypted` | Specifies whether volume should be encrypted. Set it to true if you want to enable encryption | `""` | | `storageClass.encryptionKMSID` | Specifies the encryption kms id | `""` | diff --git a/charts/ceph-csi-rbd/templates/storageclass.yaml b/charts/ceph-csi-rbd/templates/storageclass.yaml index cccd56a29..88d12c1a4 100644 --- a/charts/ceph-csi-rbd/templates/storageclass.yaml +++ b/charts/ceph-csi-rbd/templates/storageclass.yaml @@ -21,6 +21,9 @@ parameters: {{- if .Values.storageClass.cephLogDir }} cephLogDir: {{ .Values.storageClass.cephLogDir }} {{- end }} +{{- if .Values.storageClass.cephLogStrategy }} + cephLogStrategy: {{ .Values.storageClass.cephLogStrategy }} +{{- end }} {{- if .Values.storageClass.dataPool }} dataPool: {{ .Values.storageClass.dataPool }} {{- end }} diff --git a/charts/ceph-csi-rbd/values.yaml b/charts/ceph-csi-rbd/values.yaml index 54b13acfb..655fce1aa 100644 --- a/charts/ceph-csi-rbd/values.yaml +++ b/charts/ceph-csi-rbd/values.yaml @@ -289,12 +289,20 @@ storageClass: # (optional) ceph client log location, eg: rbd-nbd # By default host-path /var/log/ceph of node is bind-mounted into - # csi-rbdplugin pod at /var/log/ceph mount path. See docs/rbd-nbd.md - # for available configuration options. - # This is to configure target bindmount path used inside container. + # csi-rbdplugin pod at /var/log/ceph mount path. This is to configure + # target bindmount path used inside container for ceph clients logging. + # See docs/rbd-nbd.md for available configuration options. # cephLogDir: /var/log/ceph cephLogDir: "" + # (optional) ceph client log strategy + # By default, log file belonging to a particular volume will be deleted + # on unmap, but you can choose to just compress instead of deleting it + # or even preserve the log file in text format as it is. + # Available options `remove` or `compress` or `preserve` + # cephLogStrategy: remove + cephLogStrategy: "" + # (optional) Prefix to use for naming RBD images. # If omitted, defaults to "csi-vol-". # volumeNamePrefix: "foo-bar-" diff --git a/examples/rbd/storageclass.yaml b/examples/rbd/storageclass.yaml index 6cc329403..61182eb10 100644 --- a/examples/rbd/storageclass.yaml +++ b/examples/rbd/storageclass.yaml @@ -71,11 +71,18 @@ parameters: # (optional) ceph client log location, eg: rbd-nbd # By default host-path /var/log/ceph of node is bind-mounted into - # csi-rbdplugin pod at /var/log/ceph mount path. See docs/rbd-nbd.md - # for available configuration options. - # This is to configure target bindmount path used inside container. + # csi-rbdplugin pod at /var/log/ceph mount path. This is to configure + # target bindmount path used inside container for ceph clients logging. + # See docs/rbd-nbd.md for available configuration options. # cephLogDir: /var/log/ceph + # (optional) ceph client log strategy + # By default, log file belonging to a particular volume will be deleted + # on unmap, but you can choose to just compress instead of deleting it + # or even preserve the log file in text format as it is. + # Available options `remove` or `compress` or `preserve` + # cephLogStrategy: remove + # (optional) Prefix to use for naming RBD images. # If omitted, defaults to "csi-vol-". # volumeNamePrefix: "foo-bar-" diff --git a/internal/rbd/nodeserver.go b/internal/rbd/nodeserver.go index b6bcbda74..c8a0e114e 100644 --- a/internal/rbd/nodeserver.go +++ b/internal/rbd/nodeserver.go @@ -225,6 +225,10 @@ func populateRbdVol( if rv.LogDir == "" { rv.LogDir = defaultLogDir } + rv.LogStrategy = req.GetVolumeContext()["cephLogStrategy"] + if rv.LogStrategy == "" { + rv.LogStrategy = defaultLogStrategy + } return rv, err } @@ -846,6 +850,7 @@ func (ns *NodeServer) NodeUnstageVolume( volumeID: req.GetVolumeId(), unmapOptions: imgInfo.UnmapOptions, logDir: imgInfo.LogDir, + logStrategy: imgInfo.LogStrategy, } if err = detachRBDImageOrDeviceSpec(ctx, &dArgs); err != nil { log.ErrorLog( diff --git a/internal/rbd/rbd_attach.go b/internal/rbd/rbd_attach.go index 8f2f36789..789bf097e 100644 --- a/internal/rbd/rbd_attach.go +++ b/internal/rbd/rbd_attach.go @@ -102,6 +102,7 @@ type detachRBDImageArgs struct { volumeID string unmapOptions string logDir string + logStrategy string } // rbdGetDeviceList queries rbd about mapped devices and returns a list of rbdDeviceInfo @@ -383,6 +384,7 @@ func createPath(ctx context.Context, volOpt *rbdVolume, device string, cr *util. volumeID: volOpt.VolID, unmapOptions: volOpt.UnmapOptions, logDir: volOpt.LogDir, + logStrategy: volOpt.LogStrategy, } detErr := detachRBDImageOrDeviceSpec(ctx, &dArgs) if detErr != nil { @@ -490,10 +492,7 @@ func detachRBDImageOrDeviceSpec( } if dArgs.isNbd && dArgs.logDir != "" { logFile := getCephClientLogFileName(dArgs.volumeID, dArgs.logDir, "rbd-nbd") - if err = os.Remove(logFile); err != nil { - log.WarningLog(ctx, "failed to remove logfile: %s, error: %v", - logFile, err) - } + go strategicActionOnLogFile(ctx, dArgs.logStrategy, logFile) } return nil diff --git a/internal/rbd/rbd_util.go b/internal/rbd/rbd_util.go index df5323da4..aeb63ab6a 100644 --- a/internal/rbd/rbd_util.go +++ b/internal/rbd/rbd_util.go @@ -53,6 +53,7 @@ const ( rbdDefaultMounter = "rbd" rbdNbdMounter = "rbd-nbd" defaultLogDir = "/var/log/ceph" + defaultLogStrategy = "remove" // supports remove, compress and preserve // Output strings returned during invocation of "ceph rbd task add remove " when // command is not supported by ceph manager. Used to check errors and recover when the command @@ -140,6 +141,7 @@ type rbdVolume struct { MapOptions string UnmapOptions string LogDir string + LogStrategy string VolName string `json:"volName"` MonValueFromSecret string `json:"monValueFromSecret"` VolSize int64 `json:"volSize"` @@ -1515,8 +1517,9 @@ type rbdImageMetadataStash struct { UnmapOptions string `json:"unmapOptions"` NbdAccess bool `json:"accessType"` Encrypted bool `json:"encrypted"` - DevicePath string `json:"device"` // holds NBD device path for now - LogDir string `json:"logDir"` // holds the client log path + DevicePath string `json:"device"` // holds NBD device path for now + LogDir string `json:"logDir"` // holds the client log path + LogStrategy string `json:"logFileStrategy"` // ceph client log strategy } // file name in which image metadata is stashed. @@ -1548,6 +1551,7 @@ func stashRBDImageMetadata(volOptions *rbdVolume, metaDataPath string) error { if volOptions.Mounter == rbdTonbd && hasNBD { imgMeta.NbdAccess = true imgMeta.LogDir = volOptions.LogDir + imgMeta.LogStrategy = volOptions.LogStrategy } encodedBytes, err := json.Marshal(imgMeta) @@ -2021,3 +2025,23 @@ func CheckSliceContains(options []string, opt string) bool { return false } + +// strategicActionOnLogFile act on log file based on cephLogStrategy. +func strategicActionOnLogFile(ctx context.Context, logStrategy, logFile string) { + var err error + + switch strings.ToLower(logStrategy) { + case "compress": + if err = log.GzipLogFile(logFile); err != nil { + log.ErrorLog(ctx, "failed to compress logfile %q: %v", logFile, err) + } + case "remove": + if err = os.Remove(logFile); err != nil { + log.ErrorLog(ctx, "failed to remove logfile %q: %v", logFile, err) + } + case "preserve": + // do nothing + default: + log.ErrorLog(ctx, "unknown cephLogStrategy option %q: hint: 'remove'|'compress'|'preserve'", logStrategy) + } +} diff --git a/internal/rbd/rbd_util_test.go b/internal/rbd/rbd_util_test.go index a121daaf8..eeb94c69e 100644 --- a/internal/rbd/rbd_util_test.go +++ b/internal/rbd/rbd_util_test.go @@ -17,6 +17,9 @@ limitations under the License. package rbd import ( + "context" + "io/ioutil" + "os" "strings" "testing" @@ -208,3 +211,75 @@ func TestGetCephClientLogFileName(t *testing.T) { }) } } + +func TestStrategicActionOnLogFile(t *testing.T) { + t.Parallel() + ctx := context.TODO() + tmpDir := t.TempDir() + + var logFile [3]string + for i := 0; i < 3; i++ { + f, err := ioutil.TempFile(tmpDir, "rbd-*.log") + if err != nil { + t.Errorf("creating tempfile failed: %v", err) + } + logFile[i] = f.Name() + } + + type args struct { + logStrategy string + logFile string + } + tests := []struct { + name string + args args + }{ + { + name: "test for compress", + args: args{ + logStrategy: "compress", + logFile: logFile[0], + }, + }, + { + name: "test for remove", + args: args{ + logStrategy: "remove", + logFile: logFile[1], + }, + }, + { + name: "test for preserve", + args: args{ + logStrategy: "preserve", + logFile: logFile[2], + }, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + strategicActionOnLogFile(ctx, tt.args.logStrategy, tt.args.logFile) + + var err error + switch tt.args.logStrategy { + case "compress": + newExt := strings.Replace(tt.args.logFile, ".log", ".gz", -1) + if _, err = os.Stat(newExt); os.IsNotExist(err) { + t.Errorf("compressed logFile (%s) not found: %v", newExt, err) + } + os.Remove(newExt) + case "remove": + if _, err = os.Stat(tt.args.logFile); !os.IsNotExist(err) { + t.Errorf("logFile (%s) not removed: %v", tt.args.logFile, err) + } + case "preserve": + if _, err = os.Stat(tt.args.logFile); os.IsNotExist(err) { + t.Errorf("logFile (%s) not preserved: %v", tt.args.logFile, err) + } + os.Remove(tt.args.logFile) + } + }) + } +}