mirrors/ceph-csi
1
0
Fork 0
mirror of https://github.com/ceph/ceph-csi.git synced 2024-11-09 16:00:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Deprecate grpc metrics in ceph-csi

Browse Source 
As kubernetes CSI sidecar is exposing the
GRPC mertics we can make use of the same in
ceph-csi we dont need to expose our own.

update: #881

Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>
This commit is contained in:
Madhu Rajanna 2020-03-31 11:29:29 +05:30 committed by mergify[bot]
parent f2e98a19e8
commit bcd646ee55
21 changed files with 12 additions and 363 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
charts/ceph-csi-cephfs/templates/nodeplugin-daemonset.yaml
View File

@ -76,11 +76,6 @@ spec:
- "--type=cephfs"
- "--nodeserver=true"
- "--pidlimit=-1"
{{- if .Values.nodeplugin.grpcMetrics.enabled }}
- "--metricsport={{ .Values.nodeplugin.grpcMetrics.containerPort }}"
- "--metricspath=/metrics"
- "--enablegrpcmetrics=true"
{{- end }}
{{- if .Values.nodeplugin.forcecephkernelclient }}
- "--forcecephkernelclient={{ .Values.nodeplugin.forcecephkernelclient }}"
{{- end }}

41
charts/ceph-csi-cephfs/templates/nodeplugin-grpc-service.yaml
View File

@ -1,41 +0,0 @@
{{- if .Values.nodeplugin.grpcMetrics.service.enabled -}}
apiVersion: v1
kind: Service
metadata:
{{- if .Values.nodeplugin.grpcMetrics.service.annotations }}
annotations:
{{ toYaml .Values.nodeplugin.grpcMetrics.service.annotations | indent 4 }}
{{- end }}
name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }}-grpc-metrics
namespace: {{ .Release.Namespace }}
labels:
app: {{ include "ceph-csi-cephfs.fullname" . }}
chart: {{ include "ceph-csi-cephfs.chart" . }}
component: {{ .Values.nodeplugin.name }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
spec:
{{- if .Values.nodeplugin.grpcMetrics.service.clusterIP }}
clusterIP: "{{ .Values.nodeplugin.grpcMetrics.service.clusterIP }}"
{{- end }}
{{- if .Values.nodeplugin.grpcMetrics.service.externalIPs }}
externalIPs:
{{ toYaml .Values.nodeplugin.grpcMetrics.service.externalIPs | indent 4 }}
{{- end }}
{{- if .Values.nodeplugin.grpcMetrics.service.loadBalancerIP }}
loadBalancerIP: "{{ .Values.nodeplugin.grpcMetrics.service.loadBalancerIP }}"
{{- end }}
{{- if .Values.nodeplugin.grpcMetrics.service.loadBalancerSourceRanges }}
loadBalancerSourceRanges:
{{ toYaml .Values.nodeplugin.grpcMetrics.service.loadBalancerSourceRanges | indent 4 }}
{{- end }}
ports:
- name: grpc-metrics
port: {{ .Values.nodeplugin.grpcMetrics.service.servicePort }}
targetPort: {{ .Values.nodeplugin.grpcMetrics.containerPort }}
selector:
app: {{ include "ceph-csi-cephfs.name" . }}
component: {{ .Values.nodeplugin.name }}
release: {{ .Release.Name }}
type: "{{ .Values.nodeplugin.grpcMetrics.service.type }}"
{{- end -}}

5
charts/ceph-csi-cephfs/templates/provisioner-deployment.yaml
View File

@ -90,11 +90,6 @@ spec:
- "--type=cephfs"
- "--controllerserver=true"
- "--pidlimit=-1"
{{- if .Values.provisioner.grpcMetrics.enabled }}
- "--metricsport={{ .Values.nodeplugin.grpcMetrics.containerPort }}"
- "--metricspath=/metrics"
- "--enablegrpcmetrics=true"
{{- end }}
- "--endpoint=$(CSI_ENDPOINT)"
- "--v=5"
- "--drivername=$(DRIVER_NAME)"

41
charts/ceph-csi-cephfs/templates/provisioner-grpc-service.yaml
View File

@ -1,41 +0,0 @@
{{- if .Values.provisioner.grpcMetrics.service.enabled -}}
apiVersion: v1
kind: Service
metadata:
{{- if .Values.provisioner.grpcMetrics.service.annotations }}
annotations:
{{ toYaml .Values.provisioner.grpcMetrics.service.annotations | indent 4 }}
{{- end }}
name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }}-grpc-metrics
namespace: {{ .Release.Namespace }}
labels:
app: {{ include "ceph-csi-cephfs.fullname" . }}
chart: {{ include "ceph-csi-cephfs.chart" . }}
component: {{ .Values.provisioner.name }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
spec:
{{- if .Values.provisioner.grpcMetrics.service.clusterIP }}
clusterIP: "{{ .Values.provisioner.grpcMetrics.service.clusterIP }}"
{{- end }}
{{- if .Values.provisioner.grpcMetrics.service.externalIPs }}
externalIPs:
{{ toYaml .Values.provisioner.grpcMetrics.service.externalIPs | indent 4 }}
{{- end }}
{{- if .Values.provisioner.grpcMetrics.service.loadBalancerIP }}
loadBalancerIP: "{{ .Values.provisioner.grpcMetrics.service.loadBalancerIP }}"
{{- end }}
{{- if .Values.provisioner.grpcMetrics.service.loadBalancerSourceRanges }}
loadBalancerSourceRanges:
{{ toYaml .Values.provisioner.grpcMetrics.service.loadBalancerSourceRanges | indent 4 }}
{{- end }}
ports:
- name: grpc-metrics
port: {{ .Values.provisioner.grpcMetrics.service.servicePort }}
targetPort: {{ .Values.provisioner.grpcMetrics.containerPort }}
selector:
app: {{ include "ceph-csi-cephfs.name" . }}
component: {{ .Values.provisioner.name }}
release: {{ .Release.Name }}
type: "{{ .Values.provisioner.grpcMetrics.service.type }}"
{{- end -}}

62
charts/ceph-csi-cephfs/values.yaml
View File

@ -63,37 +63,6 @@ nodeplugin:
loadBalancerIP: ""
loadBalancerSourceRanges: []
grpcMetrics:
# Metrics only available for cephcis/cephsi => 1.2.0
# Specifies whether grpc metrics should be exposed
enabled: true
# The port of the container to expose the metrics
containerPort: 8091
service:
# Specifies whether a service should be created for the metrics
enabled: true
# The port to use for the service
servicePort: 8090
type: ClusterIP
# Annotations for the service
# Example:
# annotations:
# prometheus.io/scrape: "true"
# prometheus.io/port: "9090"
annotations: {}
clusterIP: ""
## List of IP addresses at which the stats-exporter service is available
## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips
##
externalIPs: []
loadBalancerIP: ""
loadBalancerSourceRanges: []
registrar:
image:
repository: quay.io/k8scsi/csi-node-driver-registrar
@ -160,37 +129,6 @@ provisioner:
loadBalancerIP: ""
loadBalancerSourceRanges: []
grpcMetrics:
# Metrics only available for cephcsi/cephcsi => 1.2.0
# Specifies whether grpc metrics should be exposed
enabled: true
# The port of the container to expose the metrics
containerPort: 8091
service:
# Specifies whether a service should be created for the metrics
enabled: true
# The port to use for the service
servicePort: 8090
type: ClusterIP
# Annotations for the service
# Example:
# annotations:
# prometheus.io/scrape: "true"
# prometheus.io/port: "9090"
annotations: {}
clusterIP: ""
## List of IP addresses at which the stats-exporter service is available
## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips
##
externalIPs: []
loadBalancerIP: ""
loadBalancerSourceRanges: []
provisioner:
image:
repository: quay.io/k8scsi/csi-provisioner

5
charts/ceph-csi-rbd/templates/nodeplugin-daemonset.yaml
View File

@ -77,11 +77,6 @@ spec:
- "--type=rbd"
- "--nodeserver=true"
- "--pidlimit=-1"
{{- if .Values.nodeplugin.grpcMetrics.enabled }}
- "--metricsport={{ .Values.nodeplugin.grpcMetrics.containerPort }}"
- "--metricspath=/metrics"
- "--enablegrpcmetrics=true"
{{- end }}
- "--endpoint=$(CSI_ENDPOINT)"
- "--v=5"
- "--drivername=$(DRIVER_NAME)"

41
charts/ceph-csi-rbd/templates/nodeplugin-grpc-service.yaml
View File

@ -1,41 +0,0 @@
{{- if .Values.nodeplugin.grpcMetrics.service.enabled -}}
apiVersion: v1
kind: Service
metadata:
{{- if .Values.nodeplugin.grpcMetrics.service.annotations }}
annotations:
{{ toYaml .Values.nodeplugin.grpcMetrics.service.annotations | indent 4 }}
{{- end }}
name: {{ include "ceph-csi-rbd.nodeplugin.fullname" . }}-grpc-metrics
namespace: {{ .Release.Namespace }}
labels:
app: {{ include "ceph-csi-rbd.fullname" . }}
chart: {{ include "ceph-csi-rbd.chart" . }}
component: {{ .Values.nodeplugin.name }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
spec:
{{- if .Values.nodeplugin.grpcMetrics.service.clusterIP }}
clusterIP: "{{ .Values.nodeplugin.grpcMetrics.service.clusterIP }}"
{{- end }}
{{- if .Values.nodeplugin.grpcMetrics.service.externalIPs }}
externalIPs:
{{ toYaml .Values.nodeplugin.grpcMetrics.service.externalIPs | indent 4 }}
{{- end }}
{{- if .Values.nodeplugin.grpcMetrics.service.loadBalancerIP }}
loadBalancerIP: "{{ .Values.nodeplugin.grpcMetrics.service.loadBalancerIP }}"
{{- end }}
{{- if .Values.nodeplugin.grpcMetrics.service.loadBalancerSourceRanges }}
loadBalancerSourceRanges:
{{ toYaml .Values.nodeplugin.grpcMetrics.service.loadBalancerSourceRanges | indent 4 }}
{{- end }}
ports:
- name: grpc-metrics
port: {{ .Values.nodeplugin.grpcMetrics.service.servicePort }}
targetPort: {{ .Values.nodeplugin.grpcMetrics.containerPort }}
selector:
app: {{ include "ceph-csi-rbd.name" . }}
component: {{ .Values.nodeplugin.name }}
release: {{ .Release.Name }}
type: "{{ .Values.nodeplugin.grpcMetrics.service.type }}"
{{- end -}}

5
charts/ceph-csi-rbd/templates/provisioner-deployment.yaml
View File

@ -106,11 +106,6 @@ spec:
- "--type=rbd"
- "--controllerserver=true"
- "--pidlimit=-1"
{{- if .Values.provisioner.grpcMetrics.enabled }}
- "--metricsport={{ .Values.nodeplugin.grpcMetrics.containerPort }}"
- "--metricspath=/metrics"
- "--enablegrpcmetrics=true"
{{- end }}
- "--endpoint=$(CSI_ENDPOINT)"
- "--v=5"
- "--drivername=$(DRIVER_NAME)"

41
charts/ceph-csi-rbd/templates/provisioner-grpc-service.yaml
View File

@ -1,41 +0,0 @@
{{- if .Values.provisioner.grpcMetrics.service.enabled -}}
apiVersion: v1
kind: Service
metadata:
{{- if .Values.provisioner.grpcMetrics.service.annotations }}
annotations:
{{ toYaml .Values.provisioner.grpcMetrics.service.annotations | indent 4 }}
{{- end }}
name: {{ include "ceph-csi-rbd.provisioner.fullname" . }}-grpc-metrics
namespace: {{ .Release.Namespace }}
labels:
app: {{ include "ceph-csi-rbd.fullname" . }}
chart: {{ include "ceph-csi-rbd.chart" . }}
component: {{ .Values.provisioner.name }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
spec:
{{- if .Values.provisioner.grpcMetrics.service.clusterIP }}
clusterIP: "{{ .Values.provisioner.grpcMetrics.service.clusterIP }}"
{{- end }}
{{- if .Values.provisioner.grpcMetrics.service.externalIPs }}
externalIPs:
{{ toYaml .Values.provisioner.grpcMetrics.service.externalIPs | indent 4 }}
{{- end }}
{{- if .Values.provisioner.grpcMetrics.service.loadBalancerIP }}
loadBalancerIP: "{{ .Values.provisioner.grpcMetrics.service.loadBalancerIP }}"
{{- end }}
{{- if .Values.provisioner.grpcMetrics.service.loadBalancerSourceRanges }}
loadBalancerSourceRanges:
{{ toYaml .Values.provisioner.grpcMetrics.service.loadBalancerSourceRanges | indent 4 }}
{{- end }}
ports:
- name: grpc-metrics
port: {{ .Values.provisioner.grpcMetrics.service.servicePort }}
targetPort: {{ .Values.provisioner.grpcMetrics.containerPort }}
selector:
app: {{ include "ceph-csi-rbd.name" . }}
component: {{ .Values.provisioner.name }}
release: {{ .Release.Name }}
type: "{{ .Values.provisioner.grpcMetrics.service.type }}"
{{- end -}}

62
charts/ceph-csi-rbd/values.yaml
View File

@ -77,37 +77,6 @@ nodeplugin:
loadBalancerIP: ""
loadBalancerSourceRanges: []
grpcMetrics:
# Metrics only available for cephcsi/cephcsi => 1.2.0
# Specifies whether grpc metrics should be exposed
enabled: true
# The port of the container to expose the metrics
containerPort: 8090
service:
# Specifies whether a service should be created for the metrics
enabled: true
# The port to use for the service
servicePort: 8090
type: ClusterIP
# Annotations for the service
# Example:
# annotations:
# prometheus.io/scrape: "true"
# prometheus.io/port: "8090"
annotations: {}
clusterIP: ""
## List of IP addresses at which the stats-exporter service is available
## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips
##
externalIPs: []
loadBalancerIP: ""
loadBalancerSourceRanges: []
registrar:
image:
repository: quay.io/k8scsi/csi-node-driver-registrar
@ -170,37 +139,6 @@ provisioner:
loadBalancerIP: ""
loadBalancerSourceRanges: []
grpcMetrics:
# Metrics only available for cephcis/cephsi => 1.2.0
# Specifies whether grpc metrics should be exposed
enabled: true
# The port of the container to expose the metrics
containerPort: 8090
service:
# Specifies whether a service should be created for the metrics
enabled: true
# The port to use for the service
servicePort: 8090
type: ClusterIP
# Annotations for the service
# Example:
# annotations:
# prometheus.io/scrape: "true"
# prometheus.io/port: "8090"
annotations: {}
clusterIP: ""
## List of IP addresses at which the stats-exporter service is available
## Ref: https://kubernetes.io/docs/user-guide/services/#external-ips
##
externalIPs: []
loadBalancerIP: ""
loadBalancerSourceRanges: []
provisioner:
image:
repository: quay.io/k8scsi/csi-provisioner

4
cmd/cephcsi.go
View File

@ -70,9 +70,9 @@ func init() {
flag.DurationVar(&conf.PollTime, "polltime", time.Second*60, "time interval in seconds between each poll")
flag.DurationVar(&conf.PoolTimeout, "timeout", time.Second*3, "probe timeout in seconds")
flag.BoolVar(&conf.EnableGRPCMetrics, "enablegrpcmetrics", false, "enable grpc metrics")
flag.BoolVar(&conf.EnableGRPCMetrics, "enablegrpcmetrics", false, "[DEPRECATED] enable grpc metrics")
flag.StringVar(&conf.HistogramOption, "histogramoption", "0.5,2,6",
"Histogram option for grpc metrics, should be comma separated value, ex:= 0.5,2,6 where start=0.5 factor=2, count=6")
"[DEPRECATED] Histogram option for grpc metrics, should be comma separated value, ex:= 0.5,2,6 where start=0.5 factor=2, count=6")
flag.BoolVar(&conf.Version, "version", false, "Print cephcsi version information")

7
deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml
View File

@ -13,10 +13,6 @@ spec:
port: 8080
protocol: TCP
targetPort: 8681
- name: grpc-metrics
port: 8090
protocol: TCP
targetPort: 8091
---
kind: Deployment
@ -94,9 +90,6 @@ spec:
- "--drivername=cephfs.csi.ceph.com"
- "--metadatastorage=k8s_configmap"
- "--pidlimit=-1"
- "--metricsport=8091"
- "--metricspath=/metrics"
- "--enablegrpcmetrics=false"
env:
- name: POD_IP
valueFrom:

9
deploy/cephfs/kubernetes/csi-cephfsplugin.yaml
View File

@ -63,9 +63,6 @@ spec:
- "--v=5"
- "--drivername=cephfs.csi.ceph.com"
- "--metadatastorage=k8s_configmap"
- "--metricsport=8090"
- "--metricspath=/metrics"
- "--enablegrpcmetrics=false"
env:
- name: POD_IP
valueFrom:
@ -163,7 +160,7 @@ spec:
medium: "Memory"
}
---
# This is a service to expose the liveness and grpc metrics
# This is a service to expose the liveness metrics
apiVersion: v1
kind: Service
metadata:
@ -176,9 +173,5 @@ spec:
port: 8080
protocol: TCP
targetPort: 8681
- name: grpc-metrics
port: 8090
protocol: TCP
targetPort: 8091
selector:
app: csi-cephfsplugin

7
deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml
View File

@ -13,10 +13,6 @@ spec:
port: 8080
protocol: TCP
targetPort: 8680
- name: grpc-metrics
port: 8090
protocol: TCP
targetPort: 8090
---
kind: Deployment
@ -113,9 +109,6 @@ spec:
- "--v=5"
- "--drivername=rbd.csi.ceph.com"
- "--pidlimit=-1"
- "--metricsport=8090"
- "--metricspath=/metrics"
- "--enablegrpcmetrics=false"
env:
- name: POD_IP
valueFrom:

9
deploy/rbd/kubernetes/csi-rbdplugin.yaml
View File

@ -63,9 +63,6 @@ spec:
- "--endpoint=$(CSI_ENDPOINT)"
- "--v=5"
- "--drivername=rbd.csi.ceph.com"
- "--metricsport=8090"
- "--metricspath=/metrics"
- "--enablegrpcmetrics=false"
env:
- name: POD_IP
valueFrom:
@ -168,7 +165,7 @@ spec:
medium: "Memory"
}
---
# This is a service to expose the liveness and grpc metrics
# This is a service to expose the liveness metrics
apiVersion: v1
kind: Service
metadata:
@ -181,9 +178,5 @@ spec:
port: 8080
protocol: TCP
targetPort: 8680
- name: grpc-metrics
port: 8090
protocol: TCP
targetPort: 8090
selector:
app: csi-rbdplugin

6
docs/deploy-cephfs.md
View File

@ -53,12 +53,12 @@ that should be resolved in v14.2.3.
| `--pluginpath` | "/var/lib/kubelet/plugins/" | The location of cephcsi plugin on host |
| `--metadatastorage` | _empty_ | Points to where older (1.0.0 or older plugin versions) metadata about provisioned volumes are kept, as file or in as k8s configmap (`node` or `k8s_configmap` respectively) |
| `--pidlimit` | _0_ | Configure the PID limit in cgroups. The container runtime can restrict the number of processes/tasks which can cause problems while provisioning (or deleting) a large number of volumes. A value of `-1` configures the limit to the maximum, `0` does not configure limits at all. |
| `--metricsport` | `8080` | TCP port for /grpc metrics requests |
| `--metricsport` | `8080` | TCP port for liveness metrics requests |
| `--metricspath` | `/metrics` | Path of prometheus endpoint where metrics will be available |
| `--enablegrpcmetrics` | `false` | Enable grpc metrics collection and start prometheus server |
| `--enablegrpcmetrics` | `false` | [Deprecated] Enable grpc metrics collection and start prometheus server |
| `--polltime` | `60s` | Time interval in between each poll |
| `--timeout` | `3s` | Probe timeout in seconds |
| `--histogramoption` | `0.5,2,6` | Histogram option for grpc metrics, should be comma separated value (ex:= "0.5,2,6" where start=0.5 factor=2, count=6) |
| `--histogramoption` | `0.5,2,6` | [Deprecated] Histogram option for grpc metrics, should be comma separated value (ex:= "0.5,2,6" where start=0.5 factor=2, count=6) |
| `--forcecephkernelclient` | `false` | Force enabling Ceph Kernel clients for mounting on kernels < 4.17 |
**NOTE:** The parameter `-forcecephkernelclient` enables the Kernel

6
docs/deploy-rbd.md
View File

@ -35,12 +35,12 @@ make image-cephcsi
| `--instanceid` | "default" | Unique ID distinguishing this instance of Ceph CSI among other instances, when sharing Ceph clusters across CSI instances for provisioning |
| `--metadatastorage` | _empty_ | Points to where legacy (1.0.0 or older plugin versions) metadata about provisioned volumes are kept, as file or in as k8s configmap (`node` or `k8s_configmap` respectively) |
| `--pidlimit` | _0_ | Configure the PID limit in cgroups. The container runtime can restrict the number of processes/tasks which can cause problems while provisioning (or deleting) a large number of volumes. A value of `-1` configures the limit to the maximum, `0` does not configure limits at all. |
| `--metricsport` | `8080` | TCP port for liveness/grpc metrics requests |
| `--metricsport` | `8080` | TCP port for liveness metrics requests |
| `--metricspath` | `"/metrics"` | Path of prometheus endpoint where metrics will be available |
| `--enablegrpcmetrics` | `false` | Enable grpc metrics collection and start prometheus server |
| `--enablegrpcmetrics` | `false` | [Deprecated] Enable grpc metrics collection and start prometheus server |
| `--polltime` | `"60s"` | Time interval in between each poll |
| `--timeout` | `"3s"` | Probe timeout in seconds |
| `--histogramoption` | `0.5,2,6` | Histogram option for grpc metrics, should be comma separated value (ex:= "0.5,2,6" where start=0.5 factor=2, count=6) |
| `--histogramoption` | `0.5,2,6` | [Deprecated] Histogram option for grpc metrics, should be comma separated value (ex:= "0.5,2,6" where start=0.5 factor=2, count=6) |
**Available volume parameters:**

14
docs/metrics.md
View File

@ -2,7 +2,6 @@
- [Metrics](#metrics)
- [Liveness](#liveness)
- [GRPC metrics](#grpc-metrics)
## Liveness
@ -34,16 +33,3 @@ ports will be used for additional CSI pods.
Note: You may need to open the ports used in your firewall depending on how you
cluster is setup.
## GRPC metrics
grpc metrics are intended to be collected by prometheus but can be accesesed
through a GET request to a specific pod ip.
Each CSI pod has a service to expose the endpoint to prometheus. By default rbd
pods run on port 8090 and cephfs 8091.
These can be changed if desired or if multiple ceph clusters are deployed more
ports will be used for additional CSI pods.
Note: You may need to open the ports used in your firewall depending on how you
cluster is setup.

3
examples/service-monitor.yaml
View File

@ -20,6 +20,3 @@ spec:
- port: http-metrics
path: /metrics
interval: 5s
- port: grpc-metrics
path: /metrics
interval: 5s

1
pkg/cephfs/driver.go
View File

@ -151,6 +151,7 @@ func (fs *Driver) Run(conf *util.Config, cachePersister util.CachePersister) {
server := csicommon.NewNonBlockingGRPCServer()
server.Start(conf.Endpoint, conf.HistogramOption, fs.is, fs.cs, fs.ns, conf.EnableGRPCMetrics)
if conf.EnableGRPCMetrics {
klog.Warning("EnableGRPCMetrics is deprecated")
go util.StartMetricsServer(conf)
}
server.Wait()

1
pkg/rbd/driver.go
View File

@ -154,6 +154,7 @@ func (r *Driver) Run(conf *util.Config, cachePersister util.CachePersister) {
s := csicommon.NewNonBlockingGRPCServer()
s.Start(conf.Endpoint, conf.HistogramOption, r.ids, r.cs, r.ns, conf.EnableGRPCMetrics)
if conf.EnableGRPCMetrics {
klog.Warning("EnableGRPCMetrics is deprecated")
go util.StartMetricsServer(conf)
}
s.Wait()