From a81a3bf96bcfe3ce9b26adc8d376ff65bdbd071b Mon Sep 17 00:00:00 2001 From: Madhu Rajanna Date: Wed, 21 Aug 2019 14:58:02 +0530 Subject: [PATCH] implement grpc metrics for ceph-csi Signed-off-by: Madhu Rajanna --- Gopkg.lock | 9 + Gopkg.toml | 8 + cmd/cephcsi.go | 31 ++- .../v1.13/csi-cephfsplugin-provisioner.yaml | 17 +- .../kubernetes/v1.13/csi-cephfsplugin.yaml | 21 +- .../templates/nodeplugin-daemonset.yaml | 11 +- .../templates/nodeplugin-service.yaml | 25 +++ .../templates/provisioner-service.yaml | 6 +- .../templates/provisioner-statefulset.yaml | 11 +- .../v1.14+/csi-cephfsplugin-provisioner.yaml | 17 +- .../kubernetes/v1.14+/csi-cephfsplugin.yaml | 21 +- .../templates/nodeplugin-daemonset.yaml | 11 +- .../templates/nodeplugin-service.yaml | 25 +++ .../templates/provisioner-deployment.yaml | 11 +- .../templates/provisioner-service.yaml | 25 +++ .../v1.13/csi-rbdplugin-provisioner.yaml | 17 +- .../rbd/kubernetes/v1.13/csi-rbdplugin.yaml | 21 +- .../templates/nodeplugin-daemonset.yaml | 11 +- .../templates/nodeplugin-service.yaml | 25 +++ .../templates/provisioner-service.yaml | 6 +- .../templates/provisioner-statefulset.yaml | 15 +- .../v1.14+/csi-rbdplugin-provisioner.yaml | 18 +- .../rbd/kubernetes/v1.14+/csi-rbdplugin.yaml | 21 +- .../templates/nodeplugin-daemonset.yaml | 11 +- .../templates/nodeplugin-service.yaml | 25 +++ .../templates/provisioner-deployment.yaml | 7 + docs/deploy-cephfs.md | 36 ++-- docs/deploy-rbd.md | 44 ++-- docs/metrics.md | 25 ++- examples/service-monitor.yaml | 7 +- pkg/cephfs/driver.go | 5 +- pkg/csi-common/server.go | 49 +++-- pkg/csi-common/utils.go | 12 +- pkg/liveness/liveness.go | 19 +- pkg/rbd/driver.go | 5 +- pkg/util/httpserver.go | 27 +++ pkg/util/util.go | 40 ++-- .../grpc-ecosystem/go-grpc-prometheus/LICENSE | 201 ++++++++++++++++++ .../go-grpc-prometheus/client.go | 39 ++++ .../go-grpc-prometheus/client_metrics.go | 170 +++++++++++++++ .../go-grpc-prometheus/client_reporter.go | 46 ++++ .../go-grpc-prometheus/metric_options.go | 41 ++++ .../go-grpc-prometheus/server.go | 48 +++++ .../go-grpc-prometheus/server_metrics.go | 185 ++++++++++++++++ .../go-grpc-prometheus/server_reporter.go | 46 ++++ .../grpc-ecosystem/go-grpc-prometheus/util.go | 50 +++++ 46 files changed, 1363 insertions(+), 158 deletions(-) create mode 100644 deploy/cephfs/kubernetes/v1.13/helm/ceph-csi-cephfs/templates/nodeplugin-service.yaml create mode 100644 deploy/cephfs/kubernetes/v1.14+/helm/ceph-csi-cephfs/templates/nodeplugin-service.yaml create mode 100644 deploy/cephfs/kubernetes/v1.14+/helm/ceph-csi-cephfs/templates/provisioner-service.yaml create mode 100644 deploy/rbd/kubernetes/v1.13/helm/ceph-csi-rbd/templates/nodeplugin-service.yaml create mode 100644 deploy/rbd/kubernetes/v1.14+/helm/ceph-csi-rbd/templates/nodeplugin-service.yaml create mode 100644 pkg/util/httpserver.go create mode 100644 vendor/github.com/grpc-ecosystem/go-grpc-prometheus/LICENSE create mode 100644 vendor/github.com/grpc-ecosystem/go-grpc-prometheus/client.go create mode 100644 vendor/github.com/grpc-ecosystem/go-grpc-prometheus/client_metrics.go create mode 100644 vendor/github.com/grpc-ecosystem/go-grpc-prometheus/client_reporter.go create mode 100644 vendor/github.com/grpc-ecosystem/go-grpc-prometheus/metric_options.go create mode 100644 vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server.go create mode 100644 vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go create mode 100644 vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_reporter.go create mode 100644 vendor/github.com/grpc-ecosystem/go-grpc-prometheus/util.go diff --git a/Gopkg.lock b/Gopkg.lock index 7991c1cd3..d9f079fc7 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -255,6 +255,14 @@ revision = "c250d6563d4d4c20252cd865923440e829844f4e" version = "v1.0.0" +[[projects]] + digest = "1:5872c7f130f62fc34bfda20babad36be6309c00b5c9207717f7cd2a51536fff4" + name = "github.com/grpc-ecosystem/go-grpc-prometheus" + packages = ["."] + pruneopts = "NUT" + revision = "c225b8c3b01faf2899099b768856a9e916e5087b" + version = "v1.2.0" + [[projects]] digest = "1:ed860d2b2c1d066d36a89c982eefc7d019badd534f60e87ab65d3d94f0797ef0" name = "github.com/hashicorp/golang-lru" @@ -1346,6 +1354,7 @@ "github.com/golang/protobuf/ptypes", "github.com/golang/protobuf/ptypes/timestamp", "github.com/grpc-ecosystem/go-grpc-middleware", + "github.com/grpc-ecosystem/go-grpc-prometheus", "github.com/kubernetes-csi/csi-lib-utils/connection", "github.com/kubernetes-csi/csi-lib-utils/protosanitizer", "github.com/kubernetes-csi/csi-lib-utils/rpc", diff --git a/Gopkg.toml b/Gopkg.toml index 244d2e103..f93042bdf 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -59,6 +59,14 @@ name = "k8s.io/kube-aggregator" version = "kubernetes-1.15.2" +[[constraint]] + name = "github.com/grpc-ecosystem/go-grpc-prometheus" + version = "1.2.0" + +[[constraint]] + name = "github.com/prometheus/client_golang" + version = "1.1.0" + [prune] go-tests = true non-go = true diff --git a/cmd/cephcsi.go b/cmd/cephcsi.go index d65114894..c2c25a89f 100644 --- a/cmd/cephcsi.go +++ b/cmd/cephcsi.go @@ -65,11 +65,16 @@ func init() { flag.StringVar(&conf.VolumeMounter, "volumemounter", "", "default volume mounter (possible options are 'kernel', 'fuse')") flag.StringVar(&conf.MountCacheDir, "mountcachedir", "", "mount info cache save dir") - // livenes related flags - flag.IntVar(&conf.LivenessPort, "livenessport", 8080, "TCP port for liveness requests") - flag.StringVar(&conf.LivenessPath, "livenesspath", "/metrics", "path of prometheus endpoint where metrics will be available") + // liveness/grpc metrics related flags + flag.IntVar(&conf.MetricsPort, "metricsport", 8080, "TCP port for liveness/grpc metrics requests") + flag.StringVar(&conf.MetricsPath, "metricspath", "/metrics", "path of prometheus endpoint where metrics will be available") flag.DurationVar(&conf.PollTime, "polltime", time.Second*60, "time interval in seconds between each poll") flag.DurationVar(&conf.PoolTimeout, "timeout", time.Second*3, "probe timeout in seconds") + + flag.BoolVar(&conf.EnableGRPCMetrics, "enablegrpcmetrics", false, "enable grpc metrics") + flag.StringVar(&conf.HistogramOption, "histogramoption", "0.5,2,6", + "Histogram option for grpc metrics, should be comma separated value, ex:= 0.5,2,6 where start=0.5 factor=2, count=6") + klog.InitFlags(nil) if err := flag.Set("logtostderr", "true"); err != nil { klog.Exitf("failed to set logtostderr flag: %v", err) @@ -119,9 +124,9 @@ func main() { // the driver may need a higher PID limit for handling all concurrent requests if conf.PidLimit != 0 { - currentLimit, err := util.GetPIDLimit() - if err != nil { - klog.Errorf("Failed to get the PID limit, can not reconfigure: %v", err) + currentLimit, pidErr := util.GetPIDLimit() + if pidErr != nil { + klog.Errorf("Failed to get the PID limit, can not reconfigure: %v", pidErr) } else { klog.Infof("Initial PID limit is set to %d", currentLimit) err = util.SetPIDLimit(conf.PidLimit) @@ -137,6 +142,20 @@ func main() { } } + if conf.EnableGRPCMetrics || conf.Vtype == livenessType { + // validate metrics endpoint + conf.MetricsIP = os.Getenv("POD_IP") + + if conf.MetricsIP == "" { + klog.Warning("missing POD_IP env var defaulting to 0.0.0.0") + conf.MetricsIP = "0.0.0.0" + } + err = util.ValidateURL(&conf) + if err != nil { + klog.Fatalln(err) + } + } + klog.Infof("Starting driver type: %v with name: %v", conf.Vtype, dname) switch conf.Vtype { case rbdType: diff --git a/deploy/cephfs/kubernetes/v1.13/csi-cephfsplugin-provisioner.yaml b/deploy/cephfs/kubernetes/v1.13/csi-cephfsplugin-provisioner.yaml index 1a7bf7797..f52bddf0f 100644 --- a/deploy/cephfs/kubernetes/v1.13/csi-cephfsplugin-provisioner.yaml +++ b/deploy/cephfs/kubernetes/v1.13/csi-cephfsplugin-provisioner.yaml @@ -4,7 +4,7 @@ apiVersion: v1 metadata: name: csi-cephfsplugin-provisioner labels: - app: csi-liveness + app: csi-metrics spec: selector: app: csi-cephfsplugin-provisioner @@ -13,6 +13,10 @@ spec: port: 8080 protocol: TCP targetPort: 8081 + - name: grpc-metrics + port: 8090 + protocol: TCP + targetPort: 8091 --- kind: StatefulSet @@ -74,7 +78,14 @@ spec: - "--drivername=cephfs.csi.ceph.com" - "--metadatastorage=k8s_configmap" - "--pidlimit=-1" + - "--metricsport=8091" + - "--metricspath=/metrics" + - "--enablegrpcmetrics=false" env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP - name: NODE_ID valueFrom: fieldRef: @@ -105,8 +116,8 @@ spec: args: - "--type=liveness" - "--endpoint=$(CSI_ENDPOINT)" - - "--livenessport=8081" - - "--livenesspath=/metrics" + - "--metricsport=8081" + - "--metricspath=/metrics" - "--polltime=60s" - "--timeout=3s" env: diff --git a/deploy/cephfs/kubernetes/v1.13/csi-cephfsplugin.yaml b/deploy/cephfs/kubernetes/v1.13/csi-cephfsplugin.yaml index c6e716161..c20c649ba 100644 --- a/deploy/cephfs/kubernetes/v1.13/csi-cephfsplugin.yaml +++ b/deploy/cephfs/kubernetes/v1.13/csi-cephfsplugin.yaml @@ -59,7 +59,14 @@ spec: - "--drivername=cephfs.csi.ceph.com" - "--metadatastorage=k8s_configmap" - "--mountcachedir=/mount-cache-dir" + - "--metricsport=8091" + - "--metricspath=/metrics" + - "--enablegrpcmetrics=false" env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP - name: NODE_ID valueFrom: fieldRef: @@ -99,8 +106,8 @@ spec: args: - "--type=liveness" - "--endpoint=$(CSI_ENDPOINT)" - - "--livenessport=8081" - - "--livenesspath=/metrics" + - "--metricsport=8081" + - "--metricspath=/metrics" - "--polltime=60s" - "--timeout=3s" env: @@ -150,18 +157,22 @@ spec: medium: "Memory" } --- -# This is a service to expose the liveness side car +# This is a service to expose the liveness and grpc metrics apiVersion: v1 kind: Service metadata: - name: csi-liveness-cephfsplugin + name: csi-metrics-cephfsplugin labels: - app: csi-liveness + app: csi-metrics spec: ports: - name: http-metrics port: 8080 protocol: TCP targetPort: 8081 + - name: grpc-metrics + port: 8090 + protocol: TCP + targetPort: 8091 selector: app: csi-cephfsplugin diff --git a/deploy/cephfs/kubernetes/v1.13/helm/ceph-csi-cephfs/templates/nodeplugin-daemonset.yaml b/deploy/cephfs/kubernetes/v1.13/helm/ceph-csi-cephfs/templates/nodeplugin-daemonset.yaml index 4587b4c27..d61b9cc20 100644 --- a/deploy/cephfs/kubernetes/v1.13/helm/ceph-csi-cephfs/templates/nodeplugin-daemonset.yaml +++ b/deploy/cephfs/kubernetes/v1.13/helm/ceph-csi-cephfs/templates/nodeplugin-daemonset.yaml @@ -74,7 +74,14 @@ spec: - "--drivername=$(DRIVER_NAME)" - "--metadatastorage=k8s_configmap" - "--mountcachedir=/mount-cache-dir" + - "--metricsport=8091" + - "--metricspath=/metrics" + - "--enablegrpcmetrics=false" env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP - name: DRIVER_NAME value: {{ .Values.driverName }} - name: NODE_ID @@ -115,8 +122,8 @@ spec: args: - "--type=liveness" - "--endpoint=$(CSI_ENDPOINT)" - - "--livenessport=8081" - - "--livenesspath=/metrics" + - "--metricsport=8081" + - "--metricspath=/metrics" - "--polltime=60s" - "--timeout=3s" imagePullPolicy: {{ .Values.nodeplugin.plugin.image.pullPolicy }} diff --git a/deploy/cephfs/kubernetes/v1.13/helm/ceph-csi-cephfs/templates/nodeplugin-service.yaml b/deploy/cephfs/kubernetes/v1.13/helm/ceph-csi-cephfs/templates/nodeplugin-service.yaml new file mode 100644 index 000000000..00500c053 --- /dev/null +++ b/deploy/cephfs/kubernetes/v1.13/helm/ceph-csi-cephfs/templates/nodeplugin-service.yaml @@ -0,0 +1,25 @@ +kind: Service +apiVersion: v1 +metadata: + name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: csi-metrics + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + selector: + app: {{ include "ceph-csi-cephfs.name" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + ports: + - name: http-metrics + port: 8080 + protocol: TCP + targetPort: 8081 + - name: grpc-metrics + port: 8090 + protocol: TCP + targetPort: 8091 diff --git a/deploy/cephfs/kubernetes/v1.13/helm/ceph-csi-cephfs/templates/provisioner-service.yaml b/deploy/cephfs/kubernetes/v1.13/helm/ceph-csi-cephfs/templates/provisioner-service.yaml index 073f09440..65247935d 100644 --- a/deploy/cephfs/kubernetes/v1.13/helm/ceph-csi-cephfs/templates/provisioner-service.yaml +++ b/deploy/cephfs/kubernetes/v1.13/helm/ceph-csi-cephfs/templates/provisioner-service.yaml @@ -4,7 +4,7 @@ metadata: name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }} namespace: {{ .Release.Namespace }} labels: - app: csi-liveness + app: csi-metrics chart: {{ include "ceph-csi-cephfs.chart" . }} component: {{ .Values.provisioner.name }} release: {{ .Release.Name }} @@ -19,3 +19,7 @@ spec: port: 8080 protocol: TCP targetPort: 8081 + - name: grpc-metrics + port: 8090 + protocol: TCP + targetPort: 8091 diff --git a/deploy/cephfs/kubernetes/v1.13/helm/ceph-csi-cephfs/templates/provisioner-statefulset.yaml b/deploy/cephfs/kubernetes/v1.13/helm/ceph-csi-cephfs/templates/provisioner-statefulset.yaml index 90a7c6691..955c16566 100644 --- a/deploy/cephfs/kubernetes/v1.13/helm/ceph-csi-cephfs/templates/provisioner-statefulset.yaml +++ b/deploy/cephfs/kubernetes/v1.13/helm/ceph-csi-cephfs/templates/provisioner-statefulset.yaml @@ -77,7 +77,14 @@ spec: - "--drivername=$(DRIVER_NAME)" - "--metadatastorage=k8s_configmap" - "--pidlimit=-1" + - "--metricsport=8091" + - "--metricspath=/metrics" + - "--enablegrpcmetrics=false" env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP - name: DRIVER_NAME value: {{ .Values.driverName }} - name: NODE_ID @@ -102,8 +109,8 @@ spec: args: - "--type=liveness" - "--endpoint=$(CSI_ENDPOINT)" - - "--livenessport=8081" - - "--livenesspath=/metrics" + - "--metricsport=8081" + - "--metricspath=/metrics" - "--polltime=60s" - "--timeout=3s" imagePullPolicy: {{ .Values.nodeplugin.plugin.image.pullPolicy }} diff --git a/deploy/cephfs/kubernetes/v1.14+/csi-cephfsplugin-provisioner.yaml b/deploy/cephfs/kubernetes/v1.14+/csi-cephfsplugin-provisioner.yaml index 3be2f87d0..270077940 100644 --- a/deploy/cephfs/kubernetes/v1.14+/csi-cephfsplugin-provisioner.yaml +++ b/deploy/cephfs/kubernetes/v1.14+/csi-cephfsplugin-provisioner.yaml @@ -4,7 +4,7 @@ apiVersion: v1 metadata: name: csi-cephfsplugin-provisioner labels: - app: csi-liveness + app: csi-metrics spec: selector: app: csi-cephfsplugin-provisioner @@ -13,6 +13,10 @@ spec: port: 8080 protocol: TCP targetPort: 8081 + - name: grpc-metrics + port: 8090 + protocol: TCP + targetPort: 8091 --- kind: Deployment @@ -77,7 +81,14 @@ spec: - "--drivername=cephfs.csi.ceph.com" - "--metadatastorage=k8s_configmap" - "--pidlimit=-1" + - "--metricsport=8091" + - "--metricspath=/metrics" + - "--enablegrpcmetrics=false" env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP - name: NODE_ID valueFrom: fieldRef: @@ -108,8 +119,8 @@ spec: args: - "--type=liveness" - "--endpoint=$(CSI_ENDPOINT)" - - "--livenessport=8081" - - "--livenesspath=/metrics" + - "--metricsport=8081" + - "--metricspath=/metrics" - "--polltime=60s" - "--timeout=3s" env: diff --git a/deploy/cephfs/kubernetes/v1.14+/csi-cephfsplugin.yaml b/deploy/cephfs/kubernetes/v1.14+/csi-cephfsplugin.yaml index d3d5adfb6..fa022ef90 100644 --- a/deploy/cephfs/kubernetes/v1.14+/csi-cephfsplugin.yaml +++ b/deploy/cephfs/kubernetes/v1.14+/csi-cephfsplugin.yaml @@ -59,7 +59,14 @@ spec: - "--drivername=cephfs.csi.ceph.com" - "--metadatastorage=k8s_configmap" - "--mountcachedir=/mount-cache-dir" + - "--metricsport=8090" + - "--metricspath=/metrics" + - "--enablegrpcmetrics=false" env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP - name: NODE_ID valueFrom: fieldRef: @@ -98,8 +105,8 @@ spec: args: - "--type=liveness" - "--endpoint=$(CSI_ENDPOINT)" - - "--livenessport=8081" - - "--livenesspath=/metrics" + - "--metricsport=8081" + - "--metricspath=/metrics" - "--polltime=60s" - "--timeout=3s" env: @@ -149,18 +156,22 @@ spec: medium: "Memory" } --- -# This is a service to expose the liveness side car +# This is a service to expose the liveness and grpc metrics apiVersion: v1 kind: Service metadata: - name: csi-liveness-cephfsplugin + name: csi-metrics-cephfsplugin labels: - app: csi-liveness + app: csi-metrics spec: ports: - name: http-metrics port: 8080 protocol: TCP targetPort: 8081 + - name: grpc-metrics + port: 8090 + protocol: TCP + targetPort: 8091 selector: app: csi-cephfsplugin diff --git a/deploy/cephfs/kubernetes/v1.14+/helm/ceph-csi-cephfs/templates/nodeplugin-daemonset.yaml b/deploy/cephfs/kubernetes/v1.14+/helm/ceph-csi-cephfs/templates/nodeplugin-daemonset.yaml index 31099e685..da474afe6 100644 --- a/deploy/cephfs/kubernetes/v1.14+/helm/ceph-csi-cephfs/templates/nodeplugin-daemonset.yaml +++ b/deploy/cephfs/kubernetes/v1.14+/helm/ceph-csi-cephfs/templates/nodeplugin-daemonset.yaml @@ -74,7 +74,14 @@ spec: - "--drivername=$(DRIVER_NAME)" - "--metadatastorage=k8s_configmap" - "--mountcachedir=/mount-cache-dir" + - "--metricsport=8091" + - "--metricspath=/metrics" + - "--enablegrpcmetrics=false" env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP - name: DRIVER_NAME value: {{ .Values.driverName }} - name: NODE_ID @@ -115,8 +122,8 @@ spec: args: - "--type=liveness" - "--endpoint=$(CSI_ENDPOINT)" - - "--livenessport=8081" - - "--livenesspath=/metrics" + - "--metricsport=8081" + - "--metricspath=/metrics" - "--polltime=60s" - "--timeout=3s" imagePullPolicy: {{ .Values.nodeplugin.plugin.image.pullPolicy }} diff --git a/deploy/cephfs/kubernetes/v1.14+/helm/ceph-csi-cephfs/templates/nodeplugin-service.yaml b/deploy/cephfs/kubernetes/v1.14+/helm/ceph-csi-cephfs/templates/nodeplugin-service.yaml new file mode 100644 index 000000000..00500c053 --- /dev/null +++ b/deploy/cephfs/kubernetes/v1.14+/helm/ceph-csi-cephfs/templates/nodeplugin-service.yaml @@ -0,0 +1,25 @@ +kind: Service +apiVersion: v1 +metadata: + name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: csi-metrics + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + selector: + app: {{ include "ceph-csi-cephfs.name" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + ports: + - name: http-metrics + port: 8080 + protocol: TCP + targetPort: 8081 + - name: grpc-metrics + port: 8090 + protocol: TCP + targetPort: 8091 diff --git a/deploy/cephfs/kubernetes/v1.14+/helm/ceph-csi-cephfs/templates/provisioner-deployment.yaml b/deploy/cephfs/kubernetes/v1.14+/helm/ceph-csi-cephfs/templates/provisioner-deployment.yaml index 4be3a7cee..8f9988d5d 100644 --- a/deploy/cephfs/kubernetes/v1.14+/helm/ceph-csi-cephfs/templates/provisioner-deployment.yaml +++ b/deploy/cephfs/kubernetes/v1.14+/helm/ceph-csi-cephfs/templates/provisioner-deployment.yaml @@ -79,7 +79,14 @@ spec: - "--drivername=$(DRIVER_NAME)" - "--metadatastorage=k8s_configmap" - "--pidlimit=-1" + - "--metricsport=8091" + - "--metricspath=/metrics" + - "--enablegrpcmetrics=false" env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP - name: DRIVER_NAME value: {{ .Values.driverName }} - name: NODE_ID @@ -105,8 +112,8 @@ spec: args: - "--type=liveness" - "--endpoint=$(CSI_ENDPOINT)" - - "--livenessport=8081" - - "--livenesspath=/metrics" + - "--metricsport=8081" + - "--metricspath=/metrics" - "--polltime=60s" - "--timeout=3s" imagePullPolicy: {{ .Values.nodeplugin.plugin.image.pullPolicy }} diff --git a/deploy/cephfs/kubernetes/v1.14+/helm/ceph-csi-cephfs/templates/provisioner-service.yaml b/deploy/cephfs/kubernetes/v1.14+/helm/ceph-csi-cephfs/templates/provisioner-service.yaml new file mode 100644 index 000000000..65247935d --- /dev/null +++ b/deploy/cephfs/kubernetes/v1.14+/helm/ceph-csi-cephfs/templates/provisioner-service.yaml @@ -0,0 +1,25 @@ +kind: Service +apiVersion: v1 +metadata: + name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: csi-metrics + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + selector: + app: {{ include "ceph-csi-cephfs.name" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + ports: + - name: http-metrics + port: 8080 + protocol: TCP + targetPort: 8081 + - name: grpc-metrics + port: 8090 + protocol: TCP + targetPort: 8091 diff --git a/deploy/rbd/kubernetes/v1.13/csi-rbdplugin-provisioner.yaml b/deploy/rbd/kubernetes/v1.13/csi-rbdplugin-provisioner.yaml index 22d5217b2..ff0908fc2 100644 --- a/deploy/rbd/kubernetes/v1.13/csi-rbdplugin-provisioner.yaml +++ b/deploy/rbd/kubernetes/v1.13/csi-rbdplugin-provisioner.yaml @@ -4,7 +4,7 @@ apiVersion: v1 metadata: name: csi-rbdplugin-provisioner labels: - app: csi-liveness + app: csi-metrics spec: selector: app: csi-rbdplugin-provisioner @@ -13,6 +13,10 @@ spec: port: 8080 protocol: TCP targetPort: 8080 + - name: grpc-metrics + port: 8090 + protocol: TCP + targetPort: 8090 --- kind: StatefulSet @@ -89,7 +93,14 @@ spec: - "--drivername=rbd.csi.ceph.com" - "--containerized=true" - "--pidlimit=-1" + - "--metricsport=8090" + - "--metricspath=/metrics" + - "--enablegrpcmetrics=false" env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP - name: NODE_ID valueFrom: fieldRef: @@ -118,8 +129,8 @@ spec: args: - "--type=liveness" - "--endpoint=$(CSI_ENDPOINT)" - - "--livenessport=8080" - - "--livenesspath=/metrics" + - "--metricsport=8080" + - "--metricspath=/metrics" - "--polltime=60s" - "--timeout=3s" env: diff --git a/deploy/rbd/kubernetes/v1.13/csi-rbdplugin.yaml b/deploy/rbd/kubernetes/v1.13/csi-rbdplugin.yaml index de62dfa3a..2897bdae7 100644 --- a/deploy/rbd/kubernetes/v1.13/csi-rbdplugin.yaml +++ b/deploy/rbd/kubernetes/v1.13/csi-rbdplugin.yaml @@ -61,7 +61,14 @@ spec: - "--v=5" - "--drivername=rbd.csi.ceph.com" - "--containerized=true" + - "--metricsport=8090" + - "--metricspath=/metrics" + - "--enablegrpcmetrics=false" env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP - name: NODE_ID valueFrom: fieldRef: @@ -96,8 +103,8 @@ spec: args: - "--type=liveness" - "--endpoint=$(CSI_ENDPOINT)" - - "--livenessport=8080" - - "--livenesspath=/metrics" + - "--metricsport=8080" + - "--metricspath=/metrics" - "--polltime=60s" - "--timeout=3s" env: @@ -148,18 +155,22 @@ spec: medium: "Memory" } --- -# This is a service to expose the liveness side car +# This is a service to expose the liveness and grpc metrics apiVersion: v1 kind: Service metadata: - name: csi-liveness-rbdplugin + name: csi-metrics-rbdplugin labels: - app: csi-liveness + app: csi-metrics spec: ports: - name: http-metrics port: 8080 protocol: TCP targetPort: 8080 + - name: grpc-metrics + port: 8090 + protocol: TCP + targetPort: 8090 selector: app: csi-rbdplugin diff --git a/deploy/rbd/kubernetes/v1.13/helm/ceph-csi-rbd/templates/nodeplugin-daemonset.yaml b/deploy/rbd/kubernetes/v1.13/helm/ceph-csi-rbd/templates/nodeplugin-daemonset.yaml index 07941b255..d2ece9503 100644 --- a/deploy/rbd/kubernetes/v1.13/helm/ceph-csi-rbd/templates/nodeplugin-daemonset.yaml +++ b/deploy/rbd/kubernetes/v1.13/helm/ceph-csi-rbd/templates/nodeplugin-daemonset.yaml @@ -75,7 +75,14 @@ spec: - "--v=5" - "--drivername=$(DRIVER_NAME)" - "--containerized=true" + - "--metricsport=8090" + - "--metricspath=/metrics" + - "--enablegrpcmetrics=false" env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP - name: DRIVER_NAME value: {{ .Values.driverName }} - name: NODE_ID @@ -114,8 +121,8 @@ spec: args: - "--type=liveness" - "--endpoint=$(CSI_ENDPOINT)" - - "--livenessport=8080" - - "--livenesspath=/metrics" + - "--metricsport=8080" + - "--metricspath=/metrics" - "--polltime=60s" - "--timeout=3s" imagePullPolicy: {{ .Values.nodeplugin.plugin.image.pullPolicy }} diff --git a/deploy/rbd/kubernetes/v1.13/helm/ceph-csi-rbd/templates/nodeplugin-service.yaml b/deploy/rbd/kubernetes/v1.13/helm/ceph-csi-rbd/templates/nodeplugin-service.yaml new file mode 100644 index 000000000..da24ecb53 --- /dev/null +++ b/deploy/rbd/kubernetes/v1.13/helm/ceph-csi-rbd/templates/nodeplugin-service.yaml @@ -0,0 +1,25 @@ +kind: Service +apiVersion: v1 +metadata: + name: {{ include "ceph-csi-rbd.nodeplugin.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: csi-metrics + chart: {{ include "ceph-csi-rbd.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + selector: + app: {{ include "ceph-csi-rbd.name" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + ports: + - name: http-metrics + port: 8080 + protocol: TCP + targetPort: 8080 + - name: grpc-metrics + port: 8090 + protocol: TCP + targetPort: 8090 diff --git a/deploy/rbd/kubernetes/v1.13/helm/ceph-csi-rbd/templates/provisioner-service.yaml b/deploy/rbd/kubernetes/v1.13/helm/ceph-csi-rbd/templates/provisioner-service.yaml index 73e8798c5..8042f8b01 100644 --- a/deploy/rbd/kubernetes/v1.13/helm/ceph-csi-rbd/templates/provisioner-service.yaml +++ b/deploy/rbd/kubernetes/v1.13/helm/ceph-csi-rbd/templates/provisioner-service.yaml @@ -4,7 +4,7 @@ metadata: name: {{ include "ceph-csi-rbd.provisioner.fullname" . }} namespace: {{ .Release.Namespace }} labels: - app: csi-liveness + app: csi-metrics chart: {{ include "ceph-csi-rbd.chart" . }} component: {{ .Values.provisioner.name }} release: {{ .Release.Name }} @@ -19,3 +19,7 @@ spec: port: 8080 protocol: TCP targetPort: 8080 + - name: grpc-metrics + port: 8090 + protocol: TCP + targetPort: 8090 diff --git a/deploy/rbd/kubernetes/v1.13/helm/ceph-csi-rbd/templates/provisioner-statefulset.yaml b/deploy/rbd/kubernetes/v1.13/helm/ceph-csi-rbd/templates/provisioner-statefulset.yaml index 1c67d2f29..a9ddbdf47 100644 --- a/deploy/rbd/kubernetes/v1.13/helm/ceph-csi-rbd/templates/provisioner-statefulset.yaml +++ b/deploy/rbd/kubernetes/v1.13/helm/ceph-csi-rbd/templates/provisioner-statefulset.yaml @@ -95,7 +95,14 @@ spec: - "--drivername=$(DRIVER_NAME)" - "--containerized=true" - "--pidlimit=-1" + - "--metricsport=8090" + - "--metricspath=/metrics" + - "--enablegrpcmetrics=false" env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP - name: DRIVER_NAME value: {{ .Values.driverName }} - name: NODE_ID @@ -120,8 +127,8 @@ spec: args: - "--type=liveness" - "--endpoint=$(CSI_ENDPOINT)" - - "--livenessport=8080" - - "--livenesspath=/metrics" + - "--metricsport=8080" + - "--metricspath=/metrics" - "--polltime=60s" - "--timeout=3s" imagePullPolicy: {{ .Values.nodeplugin.plugin.image.pullPolicy }} @@ -142,8 +149,8 @@ spec: args: - "--type=liveness" - "--endpoint=$(CSI_ENDPOINT)" - - "--livenessport=8081" - - "--livenesspath=/metrics" + - "--metricsport=8081" + - "--metricspath=/metrics" - "--polltime=60s" - "--timeout=3s" imagePullPolicy: {{ .Values.nodeplugin.plugin.image.pullPolicy }} diff --git a/deploy/rbd/kubernetes/v1.14+/csi-rbdplugin-provisioner.yaml b/deploy/rbd/kubernetes/v1.14+/csi-rbdplugin-provisioner.yaml index 16f7ad174..70c2494b3 100644 --- a/deploy/rbd/kubernetes/v1.14+/csi-rbdplugin-provisioner.yaml +++ b/deploy/rbd/kubernetes/v1.14+/csi-rbdplugin-provisioner.yaml @@ -4,7 +4,7 @@ apiVersion: v1 metadata: name: csi-rbdplugin-provisioner labels: - app: csi-liveness + app: csi-metrics spec: selector: app: csi-rbdplugin-provisioner @@ -13,7 +13,10 @@ spec: port: 8080 protocol: TCP targetPort: 8080 - + - name: grpc-metrics + port: 8090 + protocol: TCP + targetPort: 8090 --- kind: Deployment apiVersion: apps/v1 @@ -93,7 +96,14 @@ spec: - "--drivername=rbd.csi.ceph.com" - "--containerized=true" - "--pidlimit=-1" + - "--metricsport=8090" + - "--metricspath=/metrics" + - "--enablegrpcmetrics=false" env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP - name: NODE_ID valueFrom: fieldRef: @@ -122,8 +132,8 @@ spec: args: - "--type=liveness" - "--endpoint=$(CSI_ENDPOINT)" - - "--livenessport=8080" - - "--livenesspath=/metrics" + - "--metricsport=8080" + - "--metricspath=/metrics" - "--polltime=60s" - "--timeout=3s" env: diff --git a/deploy/rbd/kubernetes/v1.14+/csi-rbdplugin.yaml b/deploy/rbd/kubernetes/v1.14+/csi-rbdplugin.yaml index de62dfa3a..2897bdae7 100644 --- a/deploy/rbd/kubernetes/v1.14+/csi-rbdplugin.yaml +++ b/deploy/rbd/kubernetes/v1.14+/csi-rbdplugin.yaml @@ -61,7 +61,14 @@ spec: - "--v=5" - "--drivername=rbd.csi.ceph.com" - "--containerized=true" + - "--metricsport=8090" + - "--metricspath=/metrics" + - "--enablegrpcmetrics=false" env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP - name: NODE_ID valueFrom: fieldRef: @@ -96,8 +103,8 @@ spec: args: - "--type=liveness" - "--endpoint=$(CSI_ENDPOINT)" - - "--livenessport=8080" - - "--livenesspath=/metrics" + - "--metricsport=8080" + - "--metricspath=/metrics" - "--polltime=60s" - "--timeout=3s" env: @@ -148,18 +155,22 @@ spec: medium: "Memory" } --- -# This is a service to expose the liveness side car +# This is a service to expose the liveness and grpc metrics apiVersion: v1 kind: Service metadata: - name: csi-liveness-rbdplugin + name: csi-metrics-rbdplugin labels: - app: csi-liveness + app: csi-metrics spec: ports: - name: http-metrics port: 8080 protocol: TCP targetPort: 8080 + - name: grpc-metrics + port: 8090 + protocol: TCP + targetPort: 8090 selector: app: csi-rbdplugin diff --git a/deploy/rbd/kubernetes/v1.14+/helm/ceph-csi-rbd/templates/nodeplugin-daemonset.yaml b/deploy/rbd/kubernetes/v1.14+/helm/ceph-csi-rbd/templates/nodeplugin-daemonset.yaml index 9b2790104..538ffd609 100644 --- a/deploy/rbd/kubernetes/v1.14+/helm/ceph-csi-rbd/templates/nodeplugin-daemonset.yaml +++ b/deploy/rbd/kubernetes/v1.14+/helm/ceph-csi-rbd/templates/nodeplugin-daemonset.yaml @@ -75,7 +75,14 @@ spec: - "--v=5" - "--drivername=$(DRIVER_NAME)" - "--containerized=true" + - "--metricsport=8090" + - "--metricspath=/metrics" + - "--enablegrpcmetrics=false" env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP - name: DRIVER_NAME value: {{ .Values.driverName }} - name: NODE_ID @@ -114,8 +121,8 @@ spec: args: - "--type=liveness" - "--endpoint=$(CSI_ENDPOINT)" - - "--livenessport=8081" - - "--livenesspath=/metrics" + - "--metricsport=8081" + - "--metricspath=/metrics" - "--polltime=60s" - "--timeout=3s" imagePullPolicy: {{ .Values.nodeplugin.plugin.image.pullPolicy }} diff --git a/deploy/rbd/kubernetes/v1.14+/helm/ceph-csi-rbd/templates/nodeplugin-service.yaml b/deploy/rbd/kubernetes/v1.14+/helm/ceph-csi-rbd/templates/nodeplugin-service.yaml new file mode 100644 index 000000000..da24ecb53 --- /dev/null +++ b/deploy/rbd/kubernetes/v1.14+/helm/ceph-csi-rbd/templates/nodeplugin-service.yaml @@ -0,0 +1,25 @@ +kind: Service +apiVersion: v1 +metadata: + name: {{ include "ceph-csi-rbd.nodeplugin.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: csi-metrics + chart: {{ include "ceph-csi-rbd.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + selector: + app: {{ include "ceph-csi-rbd.name" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + ports: + - name: http-metrics + port: 8080 + protocol: TCP + targetPort: 8080 + - name: grpc-metrics + port: 8090 + protocol: TCP + targetPort: 8090 diff --git a/deploy/rbd/kubernetes/v1.14+/helm/ceph-csi-rbd/templates/provisioner-deployment.yaml b/deploy/rbd/kubernetes/v1.14+/helm/ceph-csi-rbd/templates/provisioner-deployment.yaml index 270a551b6..d655e3039 100644 --- a/deploy/rbd/kubernetes/v1.14+/helm/ceph-csi-rbd/templates/provisioner-deployment.yaml +++ b/deploy/rbd/kubernetes/v1.14+/helm/ceph-csi-rbd/templates/provisioner-deployment.yaml @@ -98,7 +98,14 @@ spec: - "--drivername=$(DRIVER_NAME)" - "--containerized=true" - "--pidlimit=-1" + - "--metricsport=8090" + - "--metricspath=/metrics" + - "--enablegrpcmetrics=false" env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP - name: DRIVER_NAME value: {{ .Values.driverName }} - name: NODE_ID diff --git a/docs/deploy-cephfs.md b/docs/deploy-cephfs.md index 1b78e2d86..b5825ee02 100644 --- a/docs/deploy-cephfs.md +++ b/docs/deploy-cephfs.md @@ -43,22 +43,24 @@ that should be resolved in v14.2.3. **Available command line arguments:** -| Option | Default value | Description | -| ------------------- | --------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `--endpoint` | `unix://tmp/csi.sock` | CSI endpoint, must be a UNIX socket | -| `--drivername` | `cephfs.csi.ceph.com` | Name of the driver (Kubernetes: `provisioner` field in StorageClass must correspond to this value) | -| `--nodeid` | _empty_ | This node's ID | -| `--type` | _empty_ | Driver type `[rbd | cephfs]` If the driver type is set to `rbd` it will act as a `rbd plugin` or if it's set to `cephfs` will act as a `cephfs plugin` | -| `--volumemounter` | _empty_ | Default volume mounter. Available options are `kernel` and `fuse`. This is the mount method used if volume parameters don't specify otherwise. If left unspecified, the driver will first probe for `ceph-fuse` in system's path and will choose Ceph kernel client if probing failed. | -| `--mountcachedir` | _empty_ | Volume mount cache info save dir. If left unspecified, the dirver will not record mount info, or it will save mount info and when driver restart it will remount volume it cached. | -| `--instanceid` | "default" | Unique ID distinguishing this instance of Ceph CSI among other instances, when sharing Ceph clusters across CSI instances for provisioning | -| `--pluginpath` | "/var/lib/kubelet/plugins/" | The location of cephcsi plugin on host | -| `--metadatastorage` | _empty_ | Points to where older (1.0.0 or older plugin versions) metadata about provisioned volumes are kept, as file or in as k8s configmap (`node` or `k8s_configmap` respectively) | -| `--pidlimit` | _0_ | Configure the PID limit in cgroups. The container runtime can restrict the number of processes/tasks which can cause problems while provisioning (or deleting) a large number of volumes. A value of `-1` configures the limit to the maximum, `0` does not configure limits at all. | -| `--livenessport` | `8080` | TCP port for liveness requests | -| `--livenesspath` | `/metrics` | Path of prometheus endpoint where metrics will be available | -| `--polltime` | `60s` | Time interval in between each poll | -| `--timeout` | `3s` | Probe timeout in seconds | +| Option | Default value | Description | +| --------------------- | --------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `--endpoint` | `unix://tmp/csi.sock` | CSI endpoint, must be a UNIX socket | +| `--drivername` | `cephfs.csi.ceph.com` | Name of the driver (Kubernetes: `provisioner` field in StorageClass must correspond to this value) | +| `--nodeid` | _empty_ | This node's ID | +| `--type` | _empty_ | Driver type `[rbd | cephfs]` If the driver type is set to `rbd` it will act as a `rbd plugin` or if it's set to `cephfs` will act as a `cephfs plugin` | +| `--volumemounter` | _empty_ | Default volume mounter. Available options are `kernel` and `fuse`. This is the mount method used if volume parameters don't specify otherwise. If left unspecified, the driver will first probe for `ceph-fuse` in system's path and will choose Ceph kernel client if probing failed. | +| `--mountcachedir` | _empty_ | Volume mount cache info save dir. If left unspecified, the dirver will not record mount info, or it will save mount info and when driver restart it will remount volume it cached. | +| `--instanceid` | "default" | Unique ID distinguishing this instance of Ceph CSI among other instances, when sharing Ceph clusters across CSI instances for provisioning | +| `--pluginpath` | "/var/lib/kubelet/plugins/" | The location of cephcsi plugin on host | +| `--metadatastorage` | _empty_ | Points to where older (1.0.0 or older plugin versions) metadata about provisioned volumes are kept, as file or in as k8s configmap (`node` or `k8s_configmap` respectively) | +| `--pidlimit` | _0_ | Configure the PID limit in cgroups. The container runtime can restrict the number of processes/tasks which can cause problems while provisioning (or deleting) a large number of volumes. A value of `-1` configures the limit to the maximum, `0` does not configure limits at all. | +| `--metricsport` | `8080` | TCP port for /grpc metrics requests | +| `--metricspath` | `/metrics` | Path of prometheus endpoint where metrics will be available | +| `--enablegrpcmetrics` | `false` | Enable grpc metrics collection and start prometheus server | +| `--polltime` | `60s` | Time interval in between each poll | +| `--timeout` | `3s` | Probe timeout in seconds | +| `--histogramoption` | `0.5,2,6` | Histogram option for grpc metrics, should be comma separated value (ex:= "0.5,2,6" where start=0.5 factor=2, count=6) | **Available environmental variables:** @@ -76,7 +78,7 @@ is used to define in which namespace you want the configmaps to be stored | `clusterID` | yes | String representing a Ceph cluster, must be unique across all Ceph clusters in use for provisioning, cannot be greater than 36 bytes in length, and should remain immutable for the lifetime of the Ceph cluster in use | | `fsName` | yes | CephFS filesystem name into which the volume shall be created | | `mounter` | no | Mount method to be used for this volume. Available options are `kernel` for Ceph kernel client and `fuse` for Ceph FUSE driver. Defaults to "default mounter", see command line arguments. | -| `pool` | no | Ceph pool into which volume data shall be stored | +| `pool` | no | Ceph pool into which volume data shall be stored | | `csi.storage.k8s.io/provisioner-secret-name`, `csi.storage.k8s.io/node-stage-secret-name` | for Kubernetes | Name of the Kubernetes Secret object containing Ceph client credentials. Both parameters should have the same value | | `csi.storage.k8s.io/provisioner-secret-namespace`, `csi.storage.k8s.io/node-stage-secret-namespace` | for Kubernetes | Namespaces of the above Secret objects | diff --git a/docs/deploy-rbd.md b/docs/deploy-rbd.md index ae35b606c..29ea815c9 100644 --- a/docs/deploy-rbd.md +++ b/docs/deploy-rbd.md @@ -27,32 +27,34 @@ make image-cephcsi **Available command line arguments:** -| Option | Default value | Description | -| ------------------- | --------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `--endpoint` | `unix://tmp/csi.sock` | CSI endpoint, must be a UNIX socket | -| `--drivername` | `rbd.csi.ceph.com` | Name of the driver (Kubernetes: `provisioner` field in StorageClass must correspond to this value) | -| `--nodeid` | _empty_ | This node's ID | -| `--type` | _empty_ | Driver type `[rbd | cephfs]` If the driver type is set to `rbd` it will act as a `rbd plugin` or if it's set to `cephfs` will act as a `cephfs plugin` | -| `--containerized` | true | Whether running in containerized mode | -| `--instanceid` | "default" | Unique ID distinguishing this instance of Ceph CSI among other instances, when sharing Ceph clusters across CSI instances for provisioning | -| `--metadatastorage` | _empty_ | Points to where legacy (1.0.0 or older plugin versions) metadata about provisioned volumes are kept, as file or in as k8s configmap (`node` or `k8s_configmap` respectively) | -| `--pidlimit` | _0_ | Configure the PID limit in cgroups. The container runtime can restrict the number of processes/tasks which can cause problems while provisioning (or deleting) a large number of volumes. A value of `-1` configures the limit to the maximum, `0` does not configure limits at all. | -| `--livenessport` | `8080` | TCP port for liveness requests | -| `--livenesspath` | `"/metrics"` | Path of prometheus endpoint where metrics will be available | -| `--polltime` | `"60s"` | Time interval in between each poll | -| `--timeout` | `"3s"` | Probe timeout in seconds | +| Option | Default value | Description | +| --------------------- | --------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `--endpoint` | `unix://tmp/csi.sock` | CSI endpoint, must be a UNIX socket | +| `--drivername` | `rbd.csi.ceph.com` | Name of the driver (Kubernetes: `provisioner` field in StorageClass must correspond to this value) | +| `--nodeid` | _empty_ | This node's ID | +| `--type` | _empty_ | Driver type `[rbd | cephfs]` If the driver type is set to `rbd` it will act as a `rbd plugin` or if it's set to `cephfs` will act as a `cephfs plugin` | +| `--containerized` | true | Whether running in containerized mode | +| `--instanceid` | "default" | Unique ID distinguishing this instance of Ceph CSI among other instances, when sharing Ceph clusters across CSI instances for provisioning | +| `--metadatastorage` | _empty_ | Points to where legacy (1.0.0 or older plugin versions) metadata about provisioned volumes are kept, as file or in as k8s configmap (`node` or `k8s_configmap` respectively) | +| `--pidlimit` | _0_ | Configure the PID limit in cgroups. The container runtime can restrict the number of processes/tasks which can cause problems while provisioning (or deleting) a large number of volumes. A value of `-1` configures the limit to the maximum, `0` does not configure limits at all. | +| `--metricsport` | `8080` | TCP port for liveness/grpc metrics requests | +| `--metricspath` | `"/metrics"` | Path of prometheus endpoint where metrics will be available | +| `--enablegrpcmetrics` | `false` | Enable grpc metrics collection and start prometheus server | +| `--polltime` | `"60s"` | Time interval in between each poll | +| `--timeout` | `"3s"` | Probe timeout in seconds | +| `--histogramoption` | `0.5,2,6` | Histogram option for grpc metrics, should be comma separated value (ex:= "0.5,2,6" where start=0.5 factor=2, count=6) | **Available volume parameters:** -| Parameter | Required | Description | -| ----------------------------------------------------------------------------------------------------- | -------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `clusterID` | yes | String representing a Ceph cluster, must be unique across all Ceph clusters in use for provisioning, cannot be greater than 36 bytes in length, and should remain immutable for the lifetime of the Ceph cluster in use | -| `pool` | yes | Ceph pool into which the RBD image shall be created | -| `imageFormat` | no | RBD image format. Defaults to `2`. See [man pages](http://docs.ceph.com/docs/mimic/man/8/rbd/#cmdoption-rbd-image-format) | -| `imageFeatures` | no | RBD image features. Available for `imageFormat=2`. CSI RBD currently supports only `layering` feature. See [man pages](http://docs.ceph.com/docs/mimic/man/8/rbd/#cmdoption-rbd-image-feature) | +| Parameter | Required | Description | +| --------------------------------------------------------------------------------------------------- | -------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `clusterID` | yes | String representing a Ceph cluster, must be unique across all Ceph clusters in use for provisioning, cannot be greater than 36 bytes in length, and should remain immutable for the lifetime of the Ceph cluster in use | +| `pool` | yes | Ceph pool into which the RBD image shall be created | +| `imageFormat` | no | RBD image format. Defaults to `2`. See [man pages](http://docs.ceph.com/docs/mimic/man/8/rbd/#cmdoption-rbd-image-format) | +| `imageFeatures` | no | RBD image features. Available for `imageFormat=2`. CSI RBD currently supports only `layering` feature. See [man pages](http://docs.ceph.com/docs/mimic/man/8/rbd/#cmdoption-rbd-image-feature) | | `csi.storage.k8s.io/provisioner-secret-name`, `csi.storage.k8s.io/node-stage-secret-name` | yes (for Kubernetes) | name of the Kubernetes Secret object containing Ceph client credentials. Both parameters should have the same value | | `csi.storage.k8s.io/provisioner-secret-namespace`, `csi.storage.k8s.io/node-stage-secret-namespace` | yes (for Kubernetes) | namespaces of the above Secret objects | -| `mounter` | no | if set to `rbd-nbd`, use `rbd-nbd` on nodes that have `rbd-nbd` and `nbd` kernel modules to map rbd images | +| `mounter` | no | if set to `rbd-nbd`, use `rbd-nbd` on nodes that have `rbd-nbd` and `nbd` kernel modules to map rbd images | **NOTE:** An accompanying CSI configuration file, needs to be provided to the running pods. Refer to [Creating CSI configuration](../examples/README.md#creating-csi-configuration) diff --git a/docs/metrics.md b/docs/metrics.md index 7f4a9ef62..a71244beb 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -1,9 +1,13 @@ # Metrics -CSI deploys a sidecar container that is responsible for collecting metrics. +- [Metrics](#metrics) + - [Liveness](#liveness) + - [GRPC metrics](#grpc-metrics) ## Liveness +CSI deploys a sidecar container that is responsible for collecting metrics. + Liveness metrics are intended to be collected by prometheus but can be accesesed through a GET request to a specific pod ip. @@ -13,7 +17,7 @@ for example the expected output should be ```bash -[root@worker2 /]# curl -X GET http://10.109.65.142:8080/metrics 2>/dev/null | grep csi +curl -X GET http://10.109.65.142:8080/metrics 2>/dev/null | grep csi # HELP csi_liveness Liveness Probe # TYPE csi_liveness gauge csi_liveness 1 @@ -23,10 +27,23 @@ Promethues can be deployed through the promethues operator described [here](http The [service-monitor](../examples/service-monitor.yaml) will tell promethues how to pull metrics out of CSI. -Each CSI pod has a service to expose the end point to prometheus. By default rbd +Each CSI pod has a service to expose the endpoint to prometheus. By default rbd pods run on port 8080 and cephfs 8081. These can be changed if desired or if multiple ceph clusters are deployed more ports will be used for additional CSI pods. -You may need to open the ports used in your firewall depending on how you +Note: You may need to open the ports used in your firewall depending on how you +cluster is setup. + +## GRPC metrics + +grpc metrics are intended to be collected by prometheus but can be accesesed +through a GET request to a specific pod ip. + +Each CSI pod has a service to expose the endpoint to prometheus. By default rbd +pods run on port 8090 and cephfs 8091. +These can be changed if desired or if multiple ceph clusters are deployed more +ports will be used for additional CSI pods. + +Note: You may need to open the ports used in your firewall depending on how you cluster is setup. diff --git a/examples/service-monitor.yaml b/examples/service-monitor.yaml index 29065bc7f..842aa8094 100644 --- a/examples/service-monitor.yaml +++ b/examples/service-monitor.yaml @@ -5,7 +5,7 @@ apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: - name: csi-liveness + name: csi-metrics namespace: rook-ceph labels: team: rook @@ -15,8 +15,11 @@ spec: - default selector: matchLabels: - app: csi-liveness + app: csi-metrics endpoints: - port: http-metrics path: /metrics interval: 5s + - port: grpc-metrics + path: /metrics + interval: 5s diff --git a/pkg/cephfs/driver.go b/pkg/cephfs/driver.go index ffa95b859..c17b7f9cf 100644 --- a/pkg/cephfs/driver.go +++ b/pkg/cephfs/driver.go @@ -173,6 +173,9 @@ func (fs *Driver) Run(conf *util.Config, cachePersister util.CachePersister) { } server := csicommon.NewNonBlockingGRPCServer() - server.Start(conf.Endpoint, fs.is, fs.cs, fs.ns) + server.Start(conf.Endpoint, conf.HistogramOption, fs.is, fs.cs, fs.ns, conf.EnableGRPCMetrics) + if conf.EnableGRPCMetrics { + go util.StartMetricsServer(conf) + } server.Wait() } diff --git a/pkg/csi-common/server.go b/pkg/csi-common/server.go index 53d56c8f6..d6ef6d953 100644 --- a/pkg/csi-common/server.go +++ b/pkg/csi-common/server.go @@ -19,10 +19,14 @@ package csicommon import ( "net" "os" + "strconv" + "strings" "sync" "github.com/container-storage-interface/spec/lib/go/csi" grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" + grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" + "github.com/prometheus/client_golang/prometheus" "google.golang.org/grpc" "k8s.io/klog" ) @@ -30,7 +34,7 @@ import ( // NonBlockingGRPCServer defines Non blocking GRPC server interfaces type NonBlockingGRPCServer interface { // Start services at the endpoint - Start(endpoint string, ids csi.IdentityServer, cs csi.ControllerServer, ns csi.NodeServer) + Start(endpoint, hstOptions string, ids csi.IdentityServer, cs csi.ControllerServer, ns csi.NodeServer, metrics bool) // Waits for the service to stop Wait() // Stops the service gracefully @@ -51,10 +55,10 @@ type nonBlockingGRPCServer struct { } // Start start service on endpoint -func (s *nonBlockingGRPCServer) Start(endpoint string, ids csi.IdentityServer, cs csi.ControllerServer, ns csi.NodeServer) { +func (s *nonBlockingGRPCServer) Start(endpoint, hstOptions string, ids csi.IdentityServer, cs csi.ControllerServer, ns csi.NodeServer, metrics bool) { s.wg.Add(1) - go s.serve(endpoint, ids, cs, ns) + go s.serve(endpoint, hstOptions, ids, cs, ns, metrics) } // Wait blocks until the WaitGroup counter @@ -72,7 +76,7 @@ func (s *nonBlockingGRPCServer) ForceStop() { s.server.Stop() } -func (s *nonBlockingGRPCServer) serve(endpoint string, ids csi.IdentityServer, cs csi.ControllerServer, ns csi.NodeServer) { +func (s *nonBlockingGRPCServer) serve(endpoint, hstOptions string, ids csi.IdentityServer, cs csi.ControllerServer, ns csi.NodeServer, metrics bool) { proto, addr, err := parseEndpoint(endpoint) if err != nil { @@ -91,13 +95,14 @@ func (s *nonBlockingGRPCServer) serve(endpoint string, ids csi.IdentityServer, c klog.Fatalf("Failed to listen: %v", err) } - opts := []grpc.ServerOption{ - grpc_middleware.WithUnaryServerChain( - contextIDInjector, - logGRPC, - panicHandler, - ), + middleWare := []grpc.UnaryServerInterceptor{contextIDInjector, logGRPC, panicHandler} + if metrics { + middleWare = append(middleWare, grpc_prometheus.UnaryServerInterceptor) } + opts := []grpc.ServerOption{ + grpc_middleware.WithUnaryServerChain(middleWare...), + } + server := grpc.NewServer(opts...) s.server = server @@ -110,9 +115,29 @@ func (s *nonBlockingGRPCServer) serve(endpoint string, ids csi.IdentityServer, c if ns != nil { csi.RegisterNodeServer(server, ns) } - klog.Infof("Listening for connections on address: %#v", listener.Addr()) - + if metrics { + ho := strings.Split(hstOptions, ",") + if len(ho) != 3 { + klog.Fatalf("invalid histogram options provided: %v", hstOptions) + } + start, e := strconv.ParseFloat(ho[0], 32) + if e != nil { + klog.Fatalf("failed to parse histogram start value: %v", e) + } + factor, e := strconv.ParseFloat(ho[1], 32) + if err != nil { + klog.Fatalf("failed to parse histogram factor value: %v", e) + } + count, e := strconv.Atoi(ho[2]) + if err != nil { + klog.Fatalf("failed to parse histogram count value: %v", e) + } + buckets := prometheus.ExponentialBuckets(start, factor, count) + bktOptios := grpc_prometheus.WithHistogramBuckets(buckets) + grpc_prometheus.EnableHandlingTimeHistogram(bktOptios) + grpc_prometheus.Register(server) + } err = server.Serve(listener) if err != nil { klog.Fatalf("Failed to server: %v", err) diff --git a/pkg/csi-common/utils.go b/pkg/csi-common/utils.go index 285c0dba3..7f1d02a76 100644 --- a/pkg/csi-common/utils.go +++ b/pkg/csi-common/utils.go @@ -81,29 +81,29 @@ func NewControllerServiceCapability(ctrlCap csi.ControllerServiceCapability_RPC_ } // RunNodePublishServer starts node server -func RunNodePublishServer(endpoint string, d *CSIDriver, ns csi.NodeServer) { +func RunNodePublishServer(endpoint, hstOption string, d *CSIDriver, ns csi.NodeServer, m bool) { ids := NewDefaultIdentityServer(d) s := NewNonBlockingGRPCServer() - s.Start(endpoint, ids, nil, ns) + s.Start(endpoint, hstOption, ids, nil, ns, m) s.Wait() } // RunControllerPublishServer starts controller server -func RunControllerPublishServer(endpoint string, d *CSIDriver, cs csi.ControllerServer) { +func RunControllerPublishServer(endpoint, hstOption string, d *CSIDriver, cs csi.ControllerServer, m bool) { ids := NewDefaultIdentityServer(d) s := NewNonBlockingGRPCServer() - s.Start(endpoint, ids, cs, nil) + s.Start(endpoint, hstOption, ids, cs, nil, m) s.Wait() } // RunControllerandNodePublishServer starts both controller and node server -func RunControllerandNodePublishServer(endpoint string, d *CSIDriver, cs csi.ControllerServer, ns csi.NodeServer) { +func RunControllerandNodePublishServer(endpoint, hstOption string, d *CSIDriver, cs csi.ControllerServer, ns csi.NodeServer, m bool) { ids := NewDefaultIdentityServer(d) s := NewNonBlockingGRPCServer() - s.Start(endpoint, ids, cs, ns) + s.Start(endpoint, hstOption, ids, cs, ns, m) s.Wait() } diff --git a/pkg/liveness/liveness.go b/pkg/liveness/liveness.go index 5ec873bdc..8c0275dd5 100644 --- a/pkg/liveness/liveness.go +++ b/pkg/liveness/liveness.go @@ -18,10 +18,6 @@ package liveness import ( "context" - "net" - "net/http" - "os" - "strconv" "time" "github.com/ceph/ceph-csi/pkg/util" @@ -29,7 +25,6 @@ import ( connlib "github.com/kubernetes-csi/csi-lib-utils/connection" "github.com/kubernetes-csi/csi-lib-utils/rpc" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" "k8s.io/klog" ) @@ -88,21 +83,9 @@ func recordLiveness(endpoint string, pollTime, timeout time.Duration) { func Run(conf *util.Config) { klog.Infof("Liveness Running") - ip := os.Getenv("POD_IP") - - if ip == "" { - klog.Warning("missing POD_IP env var defaulting to 0.0.0.0") - ip = "0.0.0.0" - } - // start liveness collection go recordLiveness(conf.Endpoint, conf.PollTime, conf.PoolTimeout) // start up prometheus endpoint - addr := net.JoinHostPort(ip, strconv.Itoa(conf.LivenessPort)) - http.Handle(conf.LivenessPath, promhttp.Handler()) - err := http.ListenAndServe(addr, nil) - if err != nil { - klog.Fatalln(err) - } + util.StartMetricsServer(conf) } diff --git a/pkg/rbd/driver.go b/pkg/rbd/driver.go index 10da8efc2..a1bdbc913 100644 --- a/pkg/rbd/driver.go +++ b/pkg/rbd/driver.go @@ -158,6 +158,9 @@ func (r *Driver) Run(conf *util.Config, cachePersister util.CachePersister) { } s := csicommon.NewNonBlockingGRPCServer() - s.Start(conf.Endpoint, r.ids, r.cs, r.ns) + s.Start(conf.Endpoint, conf.HistogramOption, r.ids, r.cs, r.ns, conf.EnableGRPCMetrics) + if conf.EnableGRPCMetrics { + go util.StartMetricsServer(conf) + } s.Wait() } diff --git a/pkg/util/httpserver.go b/pkg/util/httpserver.go new file mode 100644 index 000000000..3f406b5b5 --- /dev/null +++ b/pkg/util/httpserver.go @@ -0,0 +1,27 @@ +package util + +import ( + "net" + "net/http" + "net/url" + "strconv" + + "github.com/prometheus/client_golang/prometheus/promhttp" + "k8s.io/klog" +) + +// ValidateURL validates the url +func ValidateURL(c *Config) error { + _, err := url.Parse(c.MetricsPath) + return err +} + +// StartMetricsServer starts http server +func StartMetricsServer(c *Config) { + addr := net.JoinHostPort(c.MetricsIP, strconv.Itoa(c.MetricsPort)) + http.Handle(c.MetricsPath, promhttp.Handler()) + err := http.ListenAndServe(addr, nil) + if err != nil { + klog.Fatalln(err) + } +} diff --git a/pkg/util/util.go b/pkg/util/util.go index 0ab6e7521..54e6cdd92 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -54,30 +54,32 @@ var ( // Config holds the parameters list which can be configured type Config struct { - // common flags - Vtype string // driver type [rbd|cephfs|liveness] - Endpoint string // CSI endpoint - DriverName string // name of the driver - NodeID string // node id - InstanceID string // unique ID distinguishing this instance of Ceph CSI - MetadataStorage string // metadata persistence method [node|k8s_configmap] - PluginPath string // location of cephcsi plugin - PidLimit int // PID limit to configure through cgroups") - IsControllerServer bool // if set to true start provisoner server - IsNodeServer bool // if set to true start node server - - // rbd related flags - Containerized bool // whether run as containerized + Vtype string // driver type [rbd|cephfs|liveness] + Endpoint string // CSI endpoint + DriverName string // name of the driver + NodeID string // node id + InstanceID string // unique ID distinguishing this instance of Ceph CSI + MetadataStorage string // metadata persistence method [node|k8s_configmap] + PluginPath string // location of cephcsi plugin // cephfs related flags VolumeMounter string // default volume mounter (possible options are 'kernel', 'fuse') MountCacheDir string // mount info cache save dir - // livenes related flags - LivenessPort int // TCP port for liveness requests" - LivenessPath string // path of prometheus endpoint where metrics will be available - PollTime time.Duration // time interval in seconds between each poll - PoolTimeout time.Duration // probe timeout in seconds + // metrics related flags + MetricsPath string // path of prometheus endpoint where metrics will be available + HistogramOption string // Histogram option for grpc metrics, should be comma separated value, ex:= "0.5,2,6" where start=0.5 factor=2, count=6 + MetricsIP string // TCP port for liveness/ metrics requests + PidLimit int // PID limit to configure through cgroups") + MetricsPort int // TCP port for liveness/grpc metrics requests + PollTime time.Duration // time interval in seconds between each poll + PoolTimeout time.Duration // probe timeout in seconds + EnableGRPCMetrics bool // option to enable grpc metrics + + IsControllerServer bool // if set to true start provisoner server + IsNodeServer bool // if set to true start node server + // rbd related flag + Containerized bool // whether run as containerized } diff --git a/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/LICENSE b/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/LICENSE new file mode 100644 index 000000000..b2b065037 --- /dev/null +++ b/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/client.go b/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/client.go new file mode 100644 index 000000000..751a4c72d --- /dev/null +++ b/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/client.go @@ -0,0 +1,39 @@ +// Copyright 2016 Michal Witkowski. All Rights Reserved. +// See LICENSE for licensing terms. + +// gRPC Prometheus monitoring interceptors for client-side gRPC. + +package grpc_prometheus + +import ( + prom "github.com/prometheus/client_golang/prometheus" +) + +var ( + // DefaultClientMetrics is the default instance of ClientMetrics. It is + // intended to be used in conjunction the default Prometheus metrics + // registry. + DefaultClientMetrics = NewClientMetrics() + + // UnaryClientInterceptor is a gRPC client-side interceptor that provides Prometheus monitoring for Unary RPCs. + UnaryClientInterceptor = DefaultClientMetrics.UnaryClientInterceptor() + + // StreamClientInterceptor is a gRPC client-side interceptor that provides Prometheus monitoring for Streaming RPCs. + StreamClientInterceptor = DefaultClientMetrics.StreamClientInterceptor() +) + +func init() { + prom.MustRegister(DefaultClientMetrics.clientStartedCounter) + prom.MustRegister(DefaultClientMetrics.clientHandledCounter) + prom.MustRegister(DefaultClientMetrics.clientStreamMsgReceived) + prom.MustRegister(DefaultClientMetrics.clientStreamMsgSent) +} + +// EnableClientHandlingTimeHistogram turns on recording of handling time of +// RPCs. Histogram metrics can be very expensive for Prometheus to retain and +// query. This function acts on the DefaultClientMetrics variable and the +// default Prometheus metrics registry. +func EnableClientHandlingTimeHistogram(opts ...HistogramOption) { + DefaultClientMetrics.EnableClientHandlingTimeHistogram(opts...) + prom.Register(DefaultClientMetrics.clientHandledHistogram) +} diff --git a/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/client_metrics.go b/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/client_metrics.go new file mode 100644 index 000000000..9b476f983 --- /dev/null +++ b/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/client_metrics.go @@ -0,0 +1,170 @@ +package grpc_prometheus + +import ( + "io" + + prom "github.com/prometheus/client_golang/prometheus" + "golang.org/x/net/context" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// ClientMetrics represents a collection of metrics to be registered on a +// Prometheus metrics registry for a gRPC client. +type ClientMetrics struct { + clientStartedCounter *prom.CounterVec + clientHandledCounter *prom.CounterVec + clientStreamMsgReceived *prom.CounterVec + clientStreamMsgSent *prom.CounterVec + clientHandledHistogramEnabled bool + clientHandledHistogramOpts prom.HistogramOpts + clientHandledHistogram *prom.HistogramVec +} + +// NewClientMetrics returns a ClientMetrics object. Use a new instance of +// ClientMetrics when not using the default Prometheus metrics registry, for +// example when wanting to control which metrics are added to a registry as +// opposed to automatically adding metrics via init functions. +func NewClientMetrics(counterOpts ...CounterOption) *ClientMetrics { + opts := counterOptions(counterOpts) + return &ClientMetrics{ + clientStartedCounter: prom.NewCounterVec( + opts.apply(prom.CounterOpts{ + Name: "grpc_client_started_total", + Help: "Total number of RPCs started on the client.", + }), []string{"grpc_type", "grpc_service", "grpc_method"}), + + clientHandledCounter: prom.NewCounterVec( + opts.apply(prom.CounterOpts{ + Name: "grpc_client_handled_total", + Help: "Total number of RPCs completed by the client, regardless of success or failure.", + }), []string{"grpc_type", "grpc_service", "grpc_method", "grpc_code"}), + + clientStreamMsgReceived: prom.NewCounterVec( + opts.apply(prom.CounterOpts{ + Name: "grpc_client_msg_received_total", + Help: "Total number of RPC stream messages received by the client.", + }), []string{"grpc_type", "grpc_service", "grpc_method"}), + + clientStreamMsgSent: prom.NewCounterVec( + opts.apply(prom.CounterOpts{ + Name: "grpc_client_msg_sent_total", + Help: "Total number of gRPC stream messages sent by the client.", + }), []string{"grpc_type", "grpc_service", "grpc_method"}), + + clientHandledHistogramEnabled: false, + clientHandledHistogramOpts: prom.HistogramOpts{ + Name: "grpc_client_handling_seconds", + Help: "Histogram of response latency (seconds) of the gRPC until it is finished by the application.", + Buckets: prom.DefBuckets, + }, + clientHandledHistogram: nil, + } +} + +// Describe sends the super-set of all possible descriptors of metrics +// collected by this Collector to the provided channel and returns once +// the last descriptor has been sent. +func (m *ClientMetrics) Describe(ch chan<- *prom.Desc) { + m.clientStartedCounter.Describe(ch) + m.clientHandledCounter.Describe(ch) + m.clientStreamMsgReceived.Describe(ch) + m.clientStreamMsgSent.Describe(ch) + if m.clientHandledHistogramEnabled { + m.clientHandledHistogram.Describe(ch) + } +} + +// Collect is called by the Prometheus registry when collecting +// metrics. The implementation sends each collected metric via the +// provided channel and returns once the last metric has been sent. +func (m *ClientMetrics) Collect(ch chan<- prom.Metric) { + m.clientStartedCounter.Collect(ch) + m.clientHandledCounter.Collect(ch) + m.clientStreamMsgReceived.Collect(ch) + m.clientStreamMsgSent.Collect(ch) + if m.clientHandledHistogramEnabled { + m.clientHandledHistogram.Collect(ch) + } +} + +// EnableClientHandlingTimeHistogram turns on recording of handling time of RPCs. +// Histogram metrics can be very expensive for Prometheus to retain and query. +func (m *ClientMetrics) EnableClientHandlingTimeHistogram(opts ...HistogramOption) { + for _, o := range opts { + o(&m.clientHandledHistogramOpts) + } + if !m.clientHandledHistogramEnabled { + m.clientHandledHistogram = prom.NewHistogramVec( + m.clientHandledHistogramOpts, + []string{"grpc_type", "grpc_service", "grpc_method"}, + ) + } + m.clientHandledHistogramEnabled = true +} + +// UnaryClientInterceptor is a gRPC client-side interceptor that provides Prometheus monitoring for Unary RPCs. +func (m *ClientMetrics) UnaryClientInterceptor() func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + return func(ctx context.Context, method string, req, reply interface{}, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + monitor := newClientReporter(m, Unary, method) + monitor.SentMessage() + err := invoker(ctx, method, req, reply, cc, opts...) + if err != nil { + monitor.ReceivedMessage() + } + st, _ := status.FromError(err) + monitor.Handled(st.Code()) + return err + } +} + +// StreamClientInterceptor is a gRPC client-side interceptor that provides Prometheus monitoring for Streaming RPCs. +func (m *ClientMetrics) StreamClientInterceptor() func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) { + return func(ctx context.Context, desc *grpc.StreamDesc, cc *grpc.ClientConn, method string, streamer grpc.Streamer, opts ...grpc.CallOption) (grpc.ClientStream, error) { + monitor := newClientReporter(m, clientStreamType(desc), method) + clientStream, err := streamer(ctx, desc, cc, method, opts...) + if err != nil { + st, _ := status.FromError(err) + monitor.Handled(st.Code()) + return nil, err + } + return &monitoredClientStream{clientStream, monitor}, nil + } +} + +func clientStreamType(desc *grpc.StreamDesc) grpcType { + if desc.ClientStreams && !desc.ServerStreams { + return ClientStream + } else if !desc.ClientStreams && desc.ServerStreams { + return ServerStream + } + return BidiStream +} + +// monitoredClientStream wraps grpc.ClientStream allowing each Sent/Recv of message to increment counters. +type monitoredClientStream struct { + grpc.ClientStream + monitor *clientReporter +} + +func (s *monitoredClientStream) SendMsg(m interface{}) error { + err := s.ClientStream.SendMsg(m) + if err == nil { + s.monitor.SentMessage() + } + return err +} + +func (s *monitoredClientStream) RecvMsg(m interface{}) error { + err := s.ClientStream.RecvMsg(m) + if err == nil { + s.monitor.ReceivedMessage() + } else if err == io.EOF { + s.monitor.Handled(codes.OK) + } else { + st, _ := status.FromError(err) + s.monitor.Handled(st.Code()) + } + return err +} diff --git a/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/client_reporter.go b/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/client_reporter.go new file mode 100644 index 000000000..cbf153229 --- /dev/null +++ b/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/client_reporter.go @@ -0,0 +1,46 @@ +// Copyright 2016 Michal Witkowski. All Rights Reserved. +// See LICENSE for licensing terms. + +package grpc_prometheus + +import ( + "time" + + "google.golang.org/grpc/codes" +) + +type clientReporter struct { + metrics *ClientMetrics + rpcType grpcType + serviceName string + methodName string + startTime time.Time +} + +func newClientReporter(m *ClientMetrics, rpcType grpcType, fullMethod string) *clientReporter { + r := &clientReporter{ + metrics: m, + rpcType: rpcType, + } + if r.metrics.clientHandledHistogramEnabled { + r.startTime = time.Now() + } + r.serviceName, r.methodName = splitMethodName(fullMethod) + r.metrics.clientStartedCounter.WithLabelValues(string(r.rpcType), r.serviceName, r.methodName).Inc() + return r +} + +func (r *clientReporter) ReceivedMessage() { + r.metrics.clientStreamMsgReceived.WithLabelValues(string(r.rpcType), r.serviceName, r.methodName).Inc() +} + +func (r *clientReporter) SentMessage() { + r.metrics.clientStreamMsgSent.WithLabelValues(string(r.rpcType), r.serviceName, r.methodName).Inc() +} + +func (r *clientReporter) Handled(code codes.Code) { + r.metrics.clientHandledCounter.WithLabelValues(string(r.rpcType), r.serviceName, r.methodName, code.String()).Inc() + if r.metrics.clientHandledHistogramEnabled { + r.metrics.clientHandledHistogram.WithLabelValues(string(r.rpcType), r.serviceName, r.methodName).Observe(time.Since(r.startTime).Seconds()) + } +} diff --git a/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/metric_options.go b/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/metric_options.go new file mode 100644 index 000000000..9d51aec98 --- /dev/null +++ b/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/metric_options.go @@ -0,0 +1,41 @@ +package grpc_prometheus + +import ( + prom "github.com/prometheus/client_golang/prometheus" +) + +// A CounterOption lets you add options to Counter metrics using With* funcs. +type CounterOption func(*prom.CounterOpts) + +type counterOptions []CounterOption + +func (co counterOptions) apply(o prom.CounterOpts) prom.CounterOpts { + for _, f := range co { + f(&o) + } + return o +} + +// WithConstLabels allows you to add ConstLabels to Counter metrics. +func WithConstLabels(labels prom.Labels) CounterOption { + return func(o *prom.CounterOpts) { + o.ConstLabels = labels + } +} + +// A HistogramOption lets you add options to Histogram metrics using With* +// funcs. +type HistogramOption func(*prom.HistogramOpts) + +// WithHistogramBuckets allows you to specify custom bucket ranges for histograms if EnableHandlingTimeHistogram is on. +func WithHistogramBuckets(buckets []float64) HistogramOption { + return func(o *prom.HistogramOpts) { o.Buckets = buckets } +} + +// WithHistogramConstLabels allows you to add custom ConstLabels to +// histograms metrics. +func WithHistogramConstLabels(labels prom.Labels) HistogramOption { + return func(o *prom.HistogramOpts) { + o.ConstLabels = labels + } +} diff --git a/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server.go b/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server.go new file mode 100644 index 000000000..322f99046 --- /dev/null +++ b/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server.go @@ -0,0 +1,48 @@ +// Copyright 2016 Michal Witkowski. All Rights Reserved. +// See LICENSE for licensing terms. + +// gRPC Prometheus monitoring interceptors for server-side gRPC. + +package grpc_prometheus + +import ( + prom "github.com/prometheus/client_golang/prometheus" + "google.golang.org/grpc" +) + +var ( + // DefaultServerMetrics is the default instance of ServerMetrics. It is + // intended to be used in conjunction the default Prometheus metrics + // registry. + DefaultServerMetrics = NewServerMetrics() + + // UnaryServerInterceptor is a gRPC server-side interceptor that provides Prometheus monitoring for Unary RPCs. + UnaryServerInterceptor = DefaultServerMetrics.UnaryServerInterceptor() + + // StreamServerInterceptor is a gRPC server-side interceptor that provides Prometheus monitoring for Streaming RPCs. + StreamServerInterceptor = DefaultServerMetrics.StreamServerInterceptor() +) + +func init() { + prom.MustRegister(DefaultServerMetrics.serverStartedCounter) + prom.MustRegister(DefaultServerMetrics.serverHandledCounter) + prom.MustRegister(DefaultServerMetrics.serverStreamMsgReceived) + prom.MustRegister(DefaultServerMetrics.serverStreamMsgSent) +} + +// Register takes a gRPC server and pre-initializes all counters to 0. This +// allows for easier monitoring in Prometheus (no missing metrics), and should +// be called *after* all services have been registered with the server. This +// function acts on the DefaultServerMetrics variable. +func Register(server *grpc.Server) { + DefaultServerMetrics.InitializeMetrics(server) +} + +// EnableHandlingTimeHistogram turns on recording of handling time +// of RPCs. Histogram metrics can be very expensive for Prometheus +// to retain and query. This function acts on the DefaultServerMetrics +// variable and the default Prometheus metrics registry. +func EnableHandlingTimeHistogram(opts ...HistogramOption) { + DefaultServerMetrics.EnableHandlingTimeHistogram(opts...) + prom.Register(DefaultServerMetrics.serverHandledHistogram) +} diff --git a/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go b/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go new file mode 100644 index 000000000..5b1467e7a --- /dev/null +++ b/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_metrics.go @@ -0,0 +1,185 @@ +package grpc_prometheus + +import ( + prom "github.com/prometheus/client_golang/prometheus" + "golang.org/x/net/context" + "google.golang.org/grpc" + "google.golang.org/grpc/status" +) + +// ServerMetrics represents a collection of metrics to be registered on a +// Prometheus metrics registry for a gRPC server. +type ServerMetrics struct { + serverStartedCounter *prom.CounterVec + serverHandledCounter *prom.CounterVec + serverStreamMsgReceived *prom.CounterVec + serverStreamMsgSent *prom.CounterVec + serverHandledHistogramEnabled bool + serverHandledHistogramOpts prom.HistogramOpts + serverHandledHistogram *prom.HistogramVec +} + +// NewServerMetrics returns a ServerMetrics object. Use a new instance of +// ServerMetrics when not using the default Prometheus metrics registry, for +// example when wanting to control which metrics are added to a registry as +// opposed to automatically adding metrics via init functions. +func NewServerMetrics(counterOpts ...CounterOption) *ServerMetrics { + opts := counterOptions(counterOpts) + return &ServerMetrics{ + serverStartedCounter: prom.NewCounterVec( + opts.apply(prom.CounterOpts{ + Name: "grpc_server_started_total", + Help: "Total number of RPCs started on the server.", + }), []string{"grpc_type", "grpc_service", "grpc_method"}), + serverHandledCounter: prom.NewCounterVec( + opts.apply(prom.CounterOpts{ + Name: "grpc_server_handled_total", + Help: "Total number of RPCs completed on the server, regardless of success or failure.", + }), []string{"grpc_type", "grpc_service", "grpc_method", "grpc_code"}), + serverStreamMsgReceived: prom.NewCounterVec( + opts.apply(prom.CounterOpts{ + Name: "grpc_server_msg_received_total", + Help: "Total number of RPC stream messages received on the server.", + }), []string{"grpc_type", "grpc_service", "grpc_method"}), + serverStreamMsgSent: prom.NewCounterVec( + opts.apply(prom.CounterOpts{ + Name: "grpc_server_msg_sent_total", + Help: "Total number of gRPC stream messages sent by the server.", + }), []string{"grpc_type", "grpc_service", "grpc_method"}), + serverHandledHistogramEnabled: false, + serverHandledHistogramOpts: prom.HistogramOpts{ + Name: "grpc_server_handling_seconds", + Help: "Histogram of response latency (seconds) of gRPC that had been application-level handled by the server.", + Buckets: prom.DefBuckets, + }, + serverHandledHistogram: nil, + } +} + +// EnableHandlingTimeHistogram enables histograms being registered when +// registering the ServerMetrics on a Prometheus registry. Histograms can be +// expensive on Prometheus servers. It takes options to configure histogram +// options such as the defined buckets. +func (m *ServerMetrics) EnableHandlingTimeHistogram(opts ...HistogramOption) { + for _, o := range opts { + o(&m.serverHandledHistogramOpts) + } + if !m.serverHandledHistogramEnabled { + m.serverHandledHistogram = prom.NewHistogramVec( + m.serverHandledHistogramOpts, + []string{"grpc_type", "grpc_service", "grpc_method"}, + ) + } + m.serverHandledHistogramEnabled = true +} + +// Describe sends the super-set of all possible descriptors of metrics +// collected by this Collector to the provided channel and returns once +// the last descriptor has been sent. +func (m *ServerMetrics) Describe(ch chan<- *prom.Desc) { + m.serverStartedCounter.Describe(ch) + m.serverHandledCounter.Describe(ch) + m.serverStreamMsgReceived.Describe(ch) + m.serverStreamMsgSent.Describe(ch) + if m.serverHandledHistogramEnabled { + m.serverHandledHistogram.Describe(ch) + } +} + +// Collect is called by the Prometheus registry when collecting +// metrics. The implementation sends each collected metric via the +// provided channel and returns once the last metric has been sent. +func (m *ServerMetrics) Collect(ch chan<- prom.Metric) { + m.serverStartedCounter.Collect(ch) + m.serverHandledCounter.Collect(ch) + m.serverStreamMsgReceived.Collect(ch) + m.serverStreamMsgSent.Collect(ch) + if m.serverHandledHistogramEnabled { + m.serverHandledHistogram.Collect(ch) + } +} + +// UnaryServerInterceptor is a gRPC server-side interceptor that provides Prometheus monitoring for Unary RPCs. +func (m *ServerMetrics) UnaryServerInterceptor() func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { + return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { + monitor := newServerReporter(m, Unary, info.FullMethod) + monitor.ReceivedMessage() + resp, err := handler(ctx, req) + st, _ := status.FromError(err) + monitor.Handled(st.Code()) + if err == nil { + monitor.SentMessage() + } + return resp, err + } +} + +// StreamServerInterceptor is a gRPC server-side interceptor that provides Prometheus monitoring for Streaming RPCs. +func (m *ServerMetrics) StreamServerInterceptor() func(srv interface{}, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + return func(srv interface{}, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + monitor := newServerReporter(m, streamRPCType(info), info.FullMethod) + err := handler(srv, &monitoredServerStream{ss, monitor}) + st, _ := status.FromError(err) + monitor.Handled(st.Code()) + return err + } +} + +// InitializeMetrics initializes all metrics, with their appropriate null +// value, for all gRPC methods registered on a gRPC server. This is useful, to +// ensure that all metrics exist when collecting and querying. +func (m *ServerMetrics) InitializeMetrics(server *grpc.Server) { + serviceInfo := server.GetServiceInfo() + for serviceName, info := range serviceInfo { + for _, mInfo := range info.Methods { + preRegisterMethod(m, serviceName, &mInfo) + } + } +} + +func streamRPCType(info *grpc.StreamServerInfo) grpcType { + if info.IsClientStream && !info.IsServerStream { + return ClientStream + } else if !info.IsClientStream && info.IsServerStream { + return ServerStream + } + return BidiStream +} + +// monitoredStream wraps grpc.ServerStream allowing each Sent/Recv of message to increment counters. +type monitoredServerStream struct { + grpc.ServerStream + monitor *serverReporter +} + +func (s *monitoredServerStream) SendMsg(m interface{}) error { + err := s.ServerStream.SendMsg(m) + if err == nil { + s.monitor.SentMessage() + } + return err +} + +func (s *monitoredServerStream) RecvMsg(m interface{}) error { + err := s.ServerStream.RecvMsg(m) + if err == nil { + s.monitor.ReceivedMessage() + } + return err +} + +// preRegisterMethod is invoked on Register of a Server, allowing all gRPC services labels to be pre-populated. +func preRegisterMethod(metrics *ServerMetrics, serviceName string, mInfo *grpc.MethodInfo) { + methodName := mInfo.Name + methodType := string(typeFromMethodInfo(mInfo)) + // These are just references (no increments), as just referencing will create the labels but not set values. + metrics.serverStartedCounter.GetMetricWithLabelValues(methodType, serviceName, methodName) + metrics.serverStreamMsgReceived.GetMetricWithLabelValues(methodType, serviceName, methodName) + metrics.serverStreamMsgSent.GetMetricWithLabelValues(methodType, serviceName, methodName) + if metrics.serverHandledHistogramEnabled { + metrics.serverHandledHistogram.GetMetricWithLabelValues(methodType, serviceName, methodName) + } + for _, code := range allCodes { + metrics.serverHandledCounter.GetMetricWithLabelValues(methodType, serviceName, methodName, code.String()) + } +} diff --git a/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_reporter.go b/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_reporter.go new file mode 100644 index 000000000..aa9db5401 --- /dev/null +++ b/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/server_reporter.go @@ -0,0 +1,46 @@ +// Copyright 2016 Michal Witkowski. All Rights Reserved. +// See LICENSE for licensing terms. + +package grpc_prometheus + +import ( + "time" + + "google.golang.org/grpc/codes" +) + +type serverReporter struct { + metrics *ServerMetrics + rpcType grpcType + serviceName string + methodName string + startTime time.Time +} + +func newServerReporter(m *ServerMetrics, rpcType grpcType, fullMethod string) *serverReporter { + r := &serverReporter{ + metrics: m, + rpcType: rpcType, + } + if r.metrics.serverHandledHistogramEnabled { + r.startTime = time.Now() + } + r.serviceName, r.methodName = splitMethodName(fullMethod) + r.metrics.serverStartedCounter.WithLabelValues(string(r.rpcType), r.serviceName, r.methodName).Inc() + return r +} + +func (r *serverReporter) ReceivedMessage() { + r.metrics.serverStreamMsgReceived.WithLabelValues(string(r.rpcType), r.serviceName, r.methodName).Inc() +} + +func (r *serverReporter) SentMessage() { + r.metrics.serverStreamMsgSent.WithLabelValues(string(r.rpcType), r.serviceName, r.methodName).Inc() +} + +func (r *serverReporter) Handled(code codes.Code) { + r.metrics.serverHandledCounter.WithLabelValues(string(r.rpcType), r.serviceName, r.methodName, code.String()).Inc() + if r.metrics.serverHandledHistogramEnabled { + r.metrics.serverHandledHistogram.WithLabelValues(string(r.rpcType), r.serviceName, r.methodName).Observe(time.Since(r.startTime).Seconds()) + } +} diff --git a/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/util.go b/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/util.go new file mode 100644 index 000000000..7987de35f --- /dev/null +++ b/vendor/github.com/grpc-ecosystem/go-grpc-prometheus/util.go @@ -0,0 +1,50 @@ +// Copyright 2016 Michal Witkowski. All Rights Reserved. +// See LICENSE for licensing terms. + +package grpc_prometheus + +import ( + "strings" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" +) + +type grpcType string + +const ( + Unary grpcType = "unary" + ClientStream grpcType = "client_stream" + ServerStream grpcType = "server_stream" + BidiStream grpcType = "bidi_stream" +) + +var ( + allCodes = []codes.Code{ + codes.OK, codes.Canceled, codes.Unknown, codes.InvalidArgument, codes.DeadlineExceeded, codes.NotFound, + codes.AlreadyExists, codes.PermissionDenied, codes.Unauthenticated, codes.ResourceExhausted, + codes.FailedPrecondition, codes.Aborted, codes.OutOfRange, codes.Unimplemented, codes.Internal, + codes.Unavailable, codes.DataLoss, + } +) + +func splitMethodName(fullMethodName string) (string, string) { + fullMethodName = strings.TrimPrefix(fullMethodName, "/") // remove leading slash + if i := strings.Index(fullMethodName, "/"); i >= 0 { + return fullMethodName[:i], fullMethodName[i+1:] + } + return "unknown", "unknown" +} + +func typeFromMethodInfo(mInfo *grpc.MethodInfo) grpcType { + if !mInfo.IsClientStream && !mInfo.IsServerStream { + return Unary + } + if mInfo.IsClientStream && !mInfo.IsServerStream { + return ClientStream + } + if !mInfo.IsClientStream && mInfo.IsServerStream { + return ServerStream + } + return BidiStream +}