util: enable golang profiling

Add support for golang profiling.
Standard tools like go tool pprof and curl
work. example:
$ go tool pprof http://localhost:8080/debug/pprof/profile
$ go tool pprof http://localhost:8080/debug/pprof/heap
$ curl http://localhost:8080/debug/pprof/heap?debug=1

https://golang.org/pkg/net/http/pprof/ contains
more details about the pprof interface.

Fixes: #1699

Signed-off-by: Yati Padia <ypadia@redhat.com>

charts/ceph-csi-cephfs/templates/nodeplugin-daemonset.yaml

@@ -75,6 +75,9 @@ spec:
             - "--drivername=$(DRIVER_NAME)"
 {{- if .Values.topology.enabled }}
             - "--domainlabels={{ .Values.topology.domainLabels | join "," }}"
+{{- end }}
+{{- if .Values.nodeplugin.profiling.enabled }}
+            - "--enableprofiling={{ .Values.nodeplugin.profiling.enabled }}"
 {{- end }}
           env:
             - name: POD_IP

charts/ceph-csi-cephfs/templates/provisioner-deployment.yaml

@@ -136,6 +136,9 @@ spec:
             - "--endpoint=$(CSI_ENDPOINT)"
             - "--v={{ .Values.logLevel }}"
             - "--drivername=$(DRIVER_NAME)"
+{{- if .Values.provisioner.profiling.enabled }}
+            - "--enableprofiling={{ .Values.provisioner.profiling.enabled }}"
+{{- end }}
           env:
             - name: POD_IP
               valueFrom:

charts/ceph-csi-cephfs/values.yaml

@@ -74,6 +74,9 @@ nodeplugin:
       loadBalancerIP: ""
       loadBalancerSourceRanges: []
 
+  profiling:
+    enabled: false
+
   registrar:
     image:
       repository: k8s.gcr.io/sig-storage/csi-node-driver-registrar
@@ -144,6 +147,9 @@ provisioner:
       loadBalancerIP: ""
       loadBalancerSourceRanges: []
 
+  profiling:
+    enabled: false
+
   provisioner:
     image:
       repository: k8s.gcr.io/sig-storage/csi-provisioner

charts/ceph-csi-rbd/templates/nodeplugin-daemonset.yaml

@@ -73,6 +73,9 @@ spec:
             - "--drivername=$(DRIVER_NAME)"
 {{- if .Values.topology.enabled }}
             - "--domainlabels={{ .Values.topology.domainLabels | join "," }}"
+{{- end }}
+{{- if .Values.nodeplugin.profiling.enabled }}
+            - "--enableprofiling={{ .Values.nodeplugin.profiling.enabled }}"
 {{- end }}
           env:
             - name: POD_IP

charts/ceph-csi-rbd/templates/provisioner-deployment.yaml

@@ -142,6 +142,9 @@ spec:
             {{- if .Values.provisioner.skipForceFlatten }}
             - "--skipforceflatten={{ .Values.provisioner.skipForceFlatten }}"
             {{- end }}
+            {{- if .Values.provisioner.profiling.enabled }}
+            - "--enableprofiling={{ .Values.provisioner.profiling.enabled }}"
+            {{- end }}
           env:
             - name: POD_IP
               valueFrom:

charts/ceph-csi-rbd/values.yaml

@@ -85,6 +85,10 @@ nodeplugin:
       loadBalancerIP: ""
       loadBalancerSourceRanges: []
 
+  profiling:
+    # enable profiling to check for memory leaks
+    enabled: false
+
   registrar:
     image:
       repository: k8s.gcr.io/sig-storage/csi-node-driver-registrar
@@ -169,6 +173,10 @@ provisioner:
       loadBalancerIP: ""
       loadBalancerSourceRanges: []
 
+  profiling:
+    # enable profiling to check for memory leaks
+    enabled: false
+
   provisioner:
     image:
       repository: k8s.gcr.io/sig-storage/csi-provisioner

cmd/cephcsi.go

@@ -90,6 +90,7 @@ func init() {
 		"skip image flattening if kernel support mapping of rbd images which has the deep-flatten feature")
 
 	flag.BoolVar(&conf.Version, "version", false, "Print cephcsi version information")
+	flag.BoolVar(&conf.EnableProfiling, "enableprofiling", false, "enable go profiling")
 
 	klog.InitFlags(nil)
 	if err := flag.Set("logtostderr", "true"); err != nil {

deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml

@@ -120,6 +120,7 @@ spec:
             - "--v=5"
             - "--drivername=cephfs.csi.ceph.com"
             - "--pidlimit=-1"
+            - "--enableprofiling=false"
           env:
             - name: POD_IP
               valueFrom:

deploy/cephfs/kubernetes/csi-cephfsplugin.yaml

@@ -55,6 +55,7 @@ spec:
             - "--endpoint=$(CSI_ENDPOINT)"
             - "--v=5"
             - "--drivername=cephfs.csi.ceph.com"
+            - "--enableprofiling=false"
             # If topology based provisioning is desired, configure required
             # node labels representing the nodes topology domain
             # and pass the label names below, for CSI to consume and advertise

deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml

@@ -125,6 +125,7 @@ spec:
             - "--pidlimit=-1"
             - "--rbdhardmaxclonedepth=8"
             - "--rbdsoftmaxclonedepth=4"
+            - "--enableprofiling=false"
           env:
             - name: POD_IP
               valueFrom:

deploy/rbd/kubernetes/csi-rbdplugin.yaml

@@ -56,6 +56,7 @@ spec:
             - "--endpoint=$(CSI_ENDPOINT)"
             - "--v=5"
             - "--drivername=rbd.csi.ceph.com"
+            - "--enableprofiling=false"
             # If topology based provisioning is desired, configure required
             # node labels representing the nodes topology domain
             # and pass the label names below, for CSI to consume and advertise

internal/cephfs/driver.go

@@ -164,5 +164,12 @@ func (fs *Driver) Run(conf *util.Config) {
 		util.WarningLogMsg("EnableGRPCMetrics is deprecated")
 		go util.StartMetricsServer(conf)
 	}
+	if conf.EnableProfiling {
+		if !conf.EnableGRPCMetrics {
+			go util.StartMetricsServer(conf)
+		}
+		util.DebugLogMsg("Registering profiling handler")
+		go util.EnableProfiling()
+	}
 	server.Wait()
 }

internal/rbd/driver.go

@@ -187,5 +187,12 @@ func (r *Driver) Run(conf *util.Config) {
 		util.WarningLogMsg("EnableGRPCMetrics is deprecated")
 		go util.StartMetricsServer(conf)
 	}
+	if conf.EnableProfiling {
+		if !conf.EnableGRPCMetrics {
+			go util.StartMetricsServer(conf)
+		}
+		util.DebugLogMsg("Registering profiling handler")
+		go util.EnableProfiling()
+	}
 	s.Wait()
 }

internal/util/httpserver.go

@@ -3,7 +3,9 @@ package util
 import (
 	"net"
 	"net/http"
+	"net/http/pprof"
 	"net/url"
+	runtime_pprof "runtime/pprof"
 	"strconv"
 
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -24,3 +26,22 @@ func StartMetricsServer(c *Config) {
 		FatalLogMsg("failed to listen on address %v: %s", addr, err)
 	}
 }
+func addPath(name string, handler http.Handler) {
+	http.Handle(name, handler)
+	DebugLogMsg("DEBUG: registered profiling handler on /debug/pprof/%s\n", name)
+}
+
+// EnableProfiling enables golang profiling.
+func EnableProfiling() {
+	for _, profile := range runtime_pprof.Profiles() {
+		name := profile.Name()
+		handler := pprof.Handler(name)
+		addPath(name, handler)
+	}
+
+	// static profiles as listed in net/http/pprof/pprof.go:init()
+	addPath("cmdline", http.HandlerFunc(pprof.Cmdline))
+	addPath("profile", http.HandlerFunc(pprof.Profile))
+	addPath("symbol", http.HandlerFunc(pprof.Symbol))
+	addPath("trace", http.HandlerFunc(pprof.Trace))
+}

internal/util/util.go

@@ -88,6 +88,7 @@ type Config struct {
 	PoolTimeout       time.Duration // probe timeout in seconds
 	EnableGRPCMetrics bool          // option to enable grpc metrics
 
+	EnableProfiling    bool // flag to enable profiling
 	IsControllerServer bool // if set to true start provisoner server
 	IsNodeServer       bool // if set to true start node server
 	Version            bool // cephcsi version