Provide options to pass in Ceph cluster-id

This commit provides the option to pass in Ceph cluster-id instead
of a MON list from the storage class.

This helps in moving towards a stateless CSI implementation.

Tested the following,
- PV provisioning and staging using cluster-id in storage class
- PV provisioning and staging using MON list in storage class

Did not test,
- snapshot operations in either forms of the storage class

Signed-off-by: ShyamsundarR <srangana@redhat.com>

examples/rbd/snapshotclass.yaml

@@ -6,6 +6,12 @@ metadata:
 snapshotter: rbd.csi.ceph.com
 parameters:
   pool: rbd
+  # Comma separated list of Ceph monitors
+  # if using FQDN, make sure csi plugin's dns policy is appropriate.
   monitors: mon1:port,mon2:port,...
+  # OR,
+  # Ceph cluster fsid, of the cluster to provision storage from
+  # clusterID: <ceph-fsid>
+
   csi.storage.k8s.io/snapshotter-secret-name: csi-rbd-secret
   csi.storage.k8s.io/snapshotter-secret-namespace: default

examples/rbd/storageclass.yaml

@@ -8,7 +8,10 @@ parameters:
    # Comma separated list of Ceph monitors
    # if using FQDN, make sure csi plugin's dns policy is appropriate.
    monitors: mon1:port,mon2:port,...
-
+   # OR,
+   # Ceph cluster fsid, of the cluster to provision storage from
+   # clusterID: <ceph-fsid>
+   # OR,
    # if "monitors" parameter is not set, driver to get monitors from same
    # secret as admin/user credentials. "monValueFromSecret" provides the
    # key in the secret whose value is the mons

examples/rbd/template-ceph-cluster-ID-config.yaml

@@ -0,0 +1,22 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ceph-cluster-<cluster-fsid>
+  namespace: default
+data:
+  cluster-config: |
+    {
+      "version": 1,
+      "cluster-config": {
+        "cluster-fsid": "<ceph-fsid>",
+        "monitors": [
+          "<IP/DNS:port>",
+          "<IP/DNS:port>"
+        ],
+        "pools": [
+          "<pool-name>",
+          "<pool-name>"
+        ]
+      }
+    }

examples/rbd/template-ceph-cluster-ID-provisioner-secret.yaml

@@ -0,0 +1,19 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  # The <cluster-fsid> is used by the CSI plugin to uniquely identify and use a
+  #   Ceph cluster, hence the value MUST match the output of the following
+  #   command.
+  #   - Output of: `ceph fsid`
+  name: ceph-cluster-<cluster-fsid>-provisioner-secret
+  namespace: default
+data:
+  # Base64 encoded ID of the admin name
+  #   - Typically output of: `echo -n "<admin-id>" | base64`
+  # Substitute the entire string including angle braces, with the base64 value
+  subjectid: <BASE64-ENCODED-ID>
+  # Credentials of the above admin/user
+  #   - Output of: `ceph auth get-key client.admin | base64`
+  # Substitute the entire string including angle braces, with the base64 value
+  credentials: <BASE64-ENCODED-PASSWORD>

examples/rbd/template-ceph-cluster-ID-publish-secret.yaml

@@ -0,0 +1,19 @@
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  # The <cluster-fsid> is used by the CSI plugin to uniquely identify and use a
+  #   Ceph cluster, hence the value MUST match the output of the following
+  #   command.
+  #   - Output of: `ceph fsid`
+  name: ceph-cluster-<cluster-fsid>-publish-secret
+  namespace: default
+data:
+  # Base64 encoded ID of the admin name
+  #   - Typically output of: `echo -n "<admin-id>" | base64`
+  # Substitute the entire string including angle braces, with the base64 value
+  subjectid: <BASE64-ENCODED-ID>
+  # Credentials of the above admin/user
+  #   - Output of: `ceph auth get-key client.admin | base64`
+  # Substitute the entire string including angle braces, with the base64 value
+  credentials: <BASE64-ENCODED-PASSWORD>

examples/rbd/template-csi-rbdplugin-patch.yaml

@@ -0,0 +1,33 @@
+---
+# This is a patch to the existing daemonset deployment of CSI rbdplugin.
+# This is to be used when adding a new Ceph cluster to the CSI plugin.
+# NOTE: Update csi-rbdplugin-provisioner StatefulSet as well with similar patch
+# Post substituting the <cluster-fsid> in all places execute,
+#   `kubectl patch daemonset csi-rbdplugin --patch\
+#         "$(cat template-csi-rbdplugin-patch.yaml)"`
+#   to patch the statefulset deployment.
+spec:
+  template:
+    spec:
+      containers:
+        - name: csi-rbdplugin
+          volumeMounts:
+            - name: provisioner-secret-<cluster-fsid>
+              mountPath: "/etc/ceph-cluster-<cluster-fsid>-provisioner-secret"
+              readOnly: true
+            - name: publish-secret-<cluster-fsid>
+              mountPath: "/etc/ceph-cluster-<cluster-fsid>-publish-secret"
+              readOnly: true
+            - name: ceph-cluster-<cluster-fsid>
+              mountPath: "/etc/ceph-cluster-<cluster-fsid>/"
+              readOnly: true
+      volumes:
+        - name: provisioner-secret-<cluster-fsid>
+          secret:
+            secretName: ceph-cluster-<cluster-fsid>-provisioner-secret
+        - name: publish-secret-<cluster-fsid>
+          secret:
+            secretName: ceph-cluster-<cluster-fsid>-publish-secret
+        - name: ceph-cluster-<cluster-fsid>
+          configMap:
+            name: ceph-cluster-<cluster-fsid>

examples/rbd/template-csi-rbdplugin-provisioner-patch.yaml

@@ -0,0 +1,33 @@
+---
+# This is a patch to the existing statefulset deployment of CSI rbdplugin.
+# This is to be used when adding a new Ceph cluster to the CSI plugin.
+# NOTE: Update csi-rbdplugin DaemonSet as well with similar patch
+# Post substituting the <cluster-fsid> in all places execute,
+#   `kubectl patch statefulset csi-rbdplugin-provisioner --patch\
+#         "$(cat template-csi-rbdplugin-provisioner-patch.yaml)"`
+#   to patch the statefulset deployment.
+spec:
+  template:
+    spec:
+      containers:
+        - name: csi-rbdplugin
+          volumeMounts:
+            - name: provisioner-secret-<cluster-fsid>
+              mountPath: "/etc/ceph-cluster-<cluster-fsid>-provisioner-secret"
+              readOnly: true
+            - name: publish-secret-<cluster-fsid>
+              mountPath: "/etc/ceph-cluster-<cluster-fsid>-publish-secret"
+              readOnly: true
+            - name: ceph-cluster-<cluster-fsid>
+              mountPath: "/etc/ceph-cluster-<cluster-fsid>/"
+              readOnly: true
+      volumes:
+        - name: provisioner-secret-<cluster-fsid>
+          secret:
+            secretName: ceph-cluster-<cluster-fsid>-provisioner-secret
+        - name: publish-secret-<cluster-fsid>
+          secret:
+            secretName: ceph-cluster-<cluster-fsid>-publish-secret
+        - name: ceph-cluster-<cluster-fsid>
+          configMap:
+            name: ceph-cluster-<cluster-fsid>