code update

.gitignore

@@ -6,6 +6,9 @@
 # docker build
 /deploy/docker/rbdplugin
 
+# rbdplugin executable
+rbdplugin
+
 # Emacs save files
 *~
 \#*\#

Dockerfile

@@ -7,6 +7,6 @@ RUN yum  install -y centos-release-ceph && \
     yum install -y ceph-common e2fsprogs && \ 
     yum clean all
 
-COPY rbdplugin /rbdplugin
+COPY _output/rbdplugin /rbdplugin
 RUN chmod +x /rbdplugin
 ENTRYPOINT ["/rbdplugin"]

Gopkg.lock

@@ -2,10 +2,10 @@
 
 
 [[projects]]
-  branch = "master"
   name = "github.com/container-storage-interface/spec"
-  packages = ["lib/go/csi"]
+  packages = ["lib/go/csi/v0"]
-  revision = "7ab01a90da87f9fef3ee1de0494962fdefaf7db7"
+  revision = "35d9f9d77954980e449e52c3f3e43c21bd8171f5"
+  version = "v0.2.0"
 
 [[projects]]
   branch = "master"
@@ -14,16 +14,22 @@
   revision = "23def4e6c14b4da8ac2ed8007337bc5eb5007998"
 
 [[projects]]
-  branch = "master"
   name = "github.com/golang/protobuf"
-  packages = ["proto","ptypes","ptypes/any","ptypes/duration","ptypes/timestamp"]
+  packages = [
-  revision = "1e59b77b52bf8e4b449a57e6f79f21226d571845"
+    "proto",
+    "ptypes",
+    "ptypes/any",
+    "ptypes/duration",
+    "ptypes/timestamp"
+  ]
+  revision = "925541529c1fa6821df4e44ce2723319eb2be768"
+  version = "v1.0.0"
 
 [[projects]]
   branch = "master"
   name = "github.com/kubernetes-csi/drivers"
   packages = ["pkg/csi-common"]
-  revision = "d1ab787ad5510df08a3a98a091a41adeae4647b4"
+  revision = "1853bd0038cd634f277efda5c6548766a2a51ff3"
 
 [[projects]]
   name = "github.com/pborman/uuid"
@@ -34,54 +40,110 @@
 [[projects]]
   branch = "master"
   name = "golang.org/x/net"
-  packages = ["context","http2","http2/hpack","idna","internal/timeseries","lex/httplex","trace"]
+  packages = [
-  revision = "42fe2e1c20de1054d3d30f82cc9fb5b41e2e3767"
+    "context",
+    "http2",
+    "http2/hpack",
+    "idna",
+    "internal/timeseries",
+    "lex/httplex",
+    "trace"
+  ]
+  revision = "d25186b37f34ebdbbea8f488ef055638dfab272d"
 
 [[projects]]
   branch = "master"
   name = "golang.org/x/sys"
   packages = ["unix"]
-  revision = "1792d66dc88e503d3cb2400578221cdf1f7fe26f"
+  revision = "dd2ff4accc098aceecb86b36eaa7829b2a17b1c9"
 
 [[projects]]
-  branch = "master"
   name = "golang.org/x/text"
-  packages = ["collate","collate/build","internal/colltab","internal/gen","internal/tag","internal/triegen","internal/ucd","language","secure/bidirule","transform","unicode/bidi","unicode/cldr","unicode/norm","unicode/rangetable"]
+  packages = [
-  revision = "e19ae1496984b1c655b8044a65c0300a3c878dd3"
+    "collate",
+    "collate/build",
+    "internal/colltab",
+    "internal/gen",
+    "internal/tag",
+    "internal/triegen",
+    "internal/ucd",
+    "language",
+    "secure/bidirule",
+    "transform",
+    "unicode/bidi",
+    "unicode/cldr",
+    "unicode/norm",
+    "unicode/rangetable"
+  ]
+  revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0"
+  version = "v0.3.0"
 
 [[projects]]
   branch = "master"
   name = "google.golang.org/genproto"
   packages = ["googleapis/rpc/status"]
-  revision = "a8101f21cf983e773d0c1133ebc5424792003214"
+  revision = "df60624c1e9b9d2973e889c7a1cff73155da81c4"
 
 [[projects]]
   name = "google.golang.org/grpc"
-  packages = [".","balancer","balancer/base","balancer/roundrobin","codes","connectivity","credentials","encoding","grpclb/grpc_lb_v1/messages","grpclog","internal","keepalive","metadata","naming","peer","resolver","resolver/dns","resolver/passthrough","stats","status","tap","transport"]
+  packages = [
-  revision = "f3955b8e9e244dd4dd4bc4f7b7a23a8445400a76"
+    ".",
-  version = "v1.9.0"
+    "balancer",
+    "balancer/base",
+    "balancer/roundrobin",
+    "codes",
+    "connectivity",
+    "credentials",
+    "encoding",
+    "encoding/proto",
+    "grpclb/grpc_lb_v1/messages",
+    "grpclog",
+    "internal",
+    "keepalive",
+    "metadata",
+    "naming",
+    "peer",
+    "resolver",
+    "resolver/dns",
+    "resolver/passthrough",
+    "stats",
+    "status",
+    "tap",
+    "transport"
+  ]
+  revision = "8e4536a86ab602859c20df5ebfd0bd4228d08655"
+  version = "v1.10.0"
 
 [[projects]]
-  branch = "release-1.9"
   name = "k8s.io/apimachinery"
-  packages = ["pkg/util/runtime","pkg/util/sets","pkg/util/wait"]
+  packages = [
-  revision = "68f9c3a1feb3140df59c67ced62d3a5df8e6c9c2"
+    "pkg/util/runtime",
+    "pkg/util/sets",
+    "pkg/util/wait"
+  ]
+  revision = "302974c03f7e50f16561ba237db776ab93594ef6"
+  version = "kubernetes-1.10.0-beta.1"
 
 [[projects]]
   name = "k8s.io/kubernetes"
-  packages = ["pkg/util/io","pkg/util/keymutex","pkg/util/mount","pkg/util/nsenter"]
+  packages = [
-  revision = "3a1c9449a956b6026f075fa3134ff92f7d55f812"
+    "pkg/util/io",
-  version = "v1.9.1"
+    "pkg/util/keymutex",
+    "pkg/util/mount",
+    "pkg/util/nsenter"
+  ]
+  revision = "37555e6d24c2f951c40660ea59a80fa251982005"
+  version = "v1.10.0-beta.1"
 
 [[projects]]
   branch = "master"
   name = "k8s.io/utils"
   packages = ["exec"]
-  revision = "a99a3e11a96751670db62ba77c6d278d1136931e"
+  revision = "258e2a2fa64568210fbd6267cf1d8fd87c3cb86e"
 
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "8908f89154f277d98fd83b22edf73652d4c4e37bbd827bf11d9605c58ae3fd0e"
+  inputs-digest = "d409396ee410b5443abac732df725cb70ef9e0c940c1b807c0ff2698bab5d102"
   solver-name = "gps-cdcl"
   solver-version = 1

Gopkg.toml

@@ -1,23 +1,31 @@
+[[constraint]]
+  name = "github.com/container-storage-interface/spec"
+  version = "~0.2.0"
+
 [[constraint]]
   branch = "master"
-  name = "github.com/container-storage-interface/spec"
+  name = "github.com/kubernetes-csi/drivers"
 
 [[constraint]]
   branch = "master"
   name = "github.com/golang/glog"
 
-[[constraint]]
+[[override]]
-  name = "google.golang.org/grpc"
+  revision = "5db89f0ca68677abc5eefce8f2a0a772c98ba52d"
-  version = "1.7.2"
+  name = "github.com/docker/distribution"
 
 [[constraint]]
-  name = "github.com/docker/distribution"
+  name = "google.golang.org/grpc"
-  revision = "edc3ab29cdff8694dd6feb85cfeb4b5f1b38ed9c"
+  version = "1.10.0"
+
+[[constraint]]
+  version = "kubernetes-1.10.0-beta.1"
+  name = "k8s.io/apimachinery"
 
 [[constraint]]
   name = "k8s.io/kubernetes"
-  version = "v1.9.1"
+  version = "v1.10.0-beta.1"
 
-[[constraint]]
+[[override]]
-  name = "k8s.io/apimachinery"
+  version = "kubernetes-1.10.0-beta.1"
-  version = "kubernetes-1.9.1"
+  name = "k8s.io/api"

Makefile

@@ -28,12 +28,10 @@ rbdplugin:
 	CGO_ENABLED=0 GOOS=linux go build -a -ldflags '-extldflags "-static"' -o  _output/rbdplugin ./rbd
 
 container: rbdplugin 
-	cp _output/rbdplugin  deploy/docker
+	docker build -t $(IMAGE_NAME):$(IMAGE_VERSION) .
-	docker build -t $(IMAGE_NAME):$(IMAGE_VERSION) deploy/docker
 
 push-container: container
 	docker push $(IMAGE_NAME):$(IMAGE_VERSION)
 clean:
 	go clean -r -x
-	rm -f deploy/docker/rbdplugin
 	-rm -rf _output

README.md

@@ -110,7 +110,7 @@ The following output should be displayed:
 ```
 NAMESPACE     NAME                                                READY     STATUS    RESTARTS   AGE          
 default       csi-attacher-0                                      1/1       Running   0          1d           
-default       csi-nodeplugin-rbdplugin-qxqtl                      2/2       Running   0          1d           
+default       csi-rbdplugin-qxqtl                                 2/2       Running   0          1d           
 default       csi-provisioner-0                                   1/1       Running   0          1d           
 ```
 

deploy/kubernetes/csi-attacher.yaml

@@ -12,6 +12,9 @@ apiVersion: rbac.authorization.k8s.io/v1
 metadata:
   name: external-attacher-runner
 rules:
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["get", "list", "watch", "update"]
   - apiGroups: [""]
     resources: ["persistentvolumes"]
     verbs: ["get", "list", "watch", "update"]
@@ -66,19 +69,19 @@ spec:
       serviceAccount: csi-attacher
       containers:
         - name: csi-attacher
-          image: docker.io/k8scsi/csi-attacher:latest
+          image: quay.io/k8scsi/csi-attacher:v0.2.0
           args:
             - "--v=5"
             - "--csi-address=$(ADDRESS)"
           env:
             - name: ADDRESS
-              value: /var/lib/kubelet/plugins/rbdplugin/csi.sock
+              value: /var/lib/kubelet/plugins/csi-rbdplugin/csi.sock
           imagePullPolicy: "IfNotPresent"
           volumeMounts:
             - name: socket-dir
-              mountPath: /var/lib/kubelet/plugins/rbdplugin
+              mountPath: /var/lib/kubelet/plugins/csi-rbdplugin
       volumes:
         - name: socket-dir
           hostPath:
-            path: /var/lib/kubelet/plugins/rbdplugin
+            path: /var/lib/kubelet/plugins/csi-rbdplugin
             type: DirectoryOrCreate

deploy/kubernetes/csi-provisioner.yaml

@@ -18,6 +18,9 @@ apiVersion: rbac.authorization.k8s.io/v1
 metadata:
   name: external-provisioner-runner
 rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "list"]
   - apiGroups: [""]
     resources: ["persistentvolumes"]
     verbs: ["get", "list", "watch", "create", "delete"]
@@ -75,20 +78,20 @@ spec:
       serviceAccount: csi-provisioner
       containers:
         - name: csi-provisioner
-          image: quay.io/k8scsi/csi-provisioner:latest
+          image: quay.io/k8scsi/csi-provisioner:v0.2.0
           args:
-            - "--provisioner=rbdplugin"
+            - "--provisioner=csi-rbdplugin"
             - "--csi-address=$(ADDRESS)"
             - "--v=5"
           env:
             - name: ADDRESS
-              value: /var/lib/kubelet/plugins/rbdplugin/csi.sock
+              value: /var/lib/kubelet/plugins/csi-rbdplugin/csi.sock
           imagePullPolicy: "IfNotPresent"
           volumeMounts:
             - name: socket-dir
-              mountPath: /var/lib/kubelet/plugins/rbdplugin
+              mountPath: /var/lib/kubelet/plugins/csi-rbdplugin
       volumes:
         - name: socket-dir
           hostPath:
-            path: /var/lib/kubelet/plugins/rbdplugin
+            path: /var/lib/kubelet/plugins/csi-rbdplugin
             type: DirectoryOrCreate

deploy/kubernetes/rbd-secrets.yaml

@@ -1,19 +1,10 @@
 apiVersion: v1
 kind: Secret
 metadata:
-  name: ceph-secret-admin
+  name: csi-ceph-secret
-  namespace: kube-system 
+  namespace: default 
-type: "kubernetes.io/rbd"
 data:
 #Please note this value is base64 encoded.
-  key: QVFDZUhPMVpJTFBQRFJBQTd6dzNkNzZicGxrdlR3em9vc3lidkE9PQo=
+# Key value corresponds to a user name defined in ceph cluster
-type: kubernetes.io/rbd
+  admin: QVFDZUhPMVpJTFBQRFJBQTd6dzNkNzZicGxrdlR3em9vc3lidkE9PQo=
----
+  kubernetes: QVFDZDR1MVoxSDI0QnhBQWFxdmZIRnFuMSs0RFZlK1pRZ0ZmUEE9PQo=
-apiVersion: v1
-kind: Secret
-metadata:
-  name: ceph-secret-user
-type: "kubernetes.io/rbd"  
-data:
-#Please note this value is base64 encoded.
-  key: QVFDZDR1MVoxSDI0QnhBQWFxdmZIRnFuMSs0RFZlK1pRZ0ZmUEE9PQo=

deploy/kubernetes/rbd-storage-class.yaml

@@ -1,13 +1,11 @@
 apiVersion: storage.k8s.io/v1
 kind: StorageClass
 metadata:
-   name: rbd
+   name: csi-rbd
-provisioner: rbdplugin 
+provisioner: csi-rbdplugin 
 parameters:
     monitors: 192.168.80.233:6789
     pool: kubernetes
-    adminID: admin
+    csiProvisionerSecretName: csi-ceph-secret 
-    adminSecret: AQAmsGBap4EoBhAAET/Hc7fBqAZj/cy7cDcoQA==
+    csiProvisionerSecretNamespace: default 
-    userID: kube
-    userSecret: AQAMgXhVwBCeDhAA9nlPaFyfUSatGD4drFWDvQ==
 reclaimPolicy: Delete 

deploy/kubernetes/rbdplugin.yaml

@@ -3,20 +3,17 @@
 apiVersion: v1
 kind: ServiceAccount
 metadata:
-  name: csi-nodeplugin
+  name: csi-rbdplugin
 
 ---
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
-  name: csi-nodeplugin
+  name: csi-rbdplugin
 rules:
   - apiGroups: [""]
     resources: ["nodes"]
     verbs: ["get", "list", "update"]
-  - apiGroups: [""]
-    resources: ["secrets"]
-    verbs: ["get", "list"]
   - apiGroups: [""]
     resources: ["namespaces"]
     verbs: ["get", "list"]
@@ -30,14 +27,14 @@ rules:
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
-  name: csi-nodeplugin
+  name: csi-rbdplugin
 subjects:
   - kind: ServiceAccount
-    name: csi-nodeplugin
+    name: csi-rbdplugin
     namespace: default
 roleRef:
   kind: ClusterRole
-  name: csi-nodeplugin
+  name: csi-rbdplugin
   apiGroup: rbac.authorization.k8s.io          
 
 ---
@@ -47,35 +44,35 @@ roleRef:
 kind: DaemonSet
 apiVersion: apps/v1beta2
 metadata:
-  name: csi-nodeplugin-rbdplugin
+  name: csi-rbdplugin
 spec:
   selector:
     matchLabels:
-      app: csi-nodeplugin-rbdplugin
+      app: csi-rbdplugin
   template:
     metadata:
       labels:
-        app: csi-nodeplugin-rbdplugin
+        app: csi-rbdplugin
     spec:
-      serviceAccount: csi-nodeplugin
+      serviceAccount: csi-rbdplugin
       hostNetwork: true
       containers:
         - name: driver-registrar
-          image: docker.io/k8scsi/driver-registrar:latest
+          image: quay.io/k8scsi/driver-registrar:v0.2.0
           args:
             - "--v=5"
             - "--csi-address=$(ADDRESS)"
           env:
             - name: ADDRESS
-              value: /var/lib/kubelet/plugins/rbdplugin/csi.sock
+              value: /var/lib/kubelet/plugins/csi-rbdplugin/csi.sock
             - name: KUBE_NODE_NAME
               valueFrom:
                 fieldRef:
                   fieldPath: spec.nodeName
           volumeMounts:
             - name: socket-dir
-              mountPath: /var/lib/kubelet/plugins/rbdplugin
+              mountPath: /var/lib/kubelet/plugins/csi-rbdplugin
-        - name: rbdplugin 
+        - name: csi-rbdplugin 
           securityContext:
             privileged: true
             capabilities:
@@ -86,18 +83,18 @@ spec:
             - "--nodeid=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"
             - "--v=5"
-            - "--drivername=rbdplugin"
+            - "--drivername=csi-rbdplugin"
           env:
             - name: NODE_ID
               valueFrom:
                 fieldRef:
                   fieldPath: spec.nodeName
             - name: CSI_ENDPOINT
-              value: unix://var/lib/kubelet/plugins/rbdplugin/csi.sock
+              value: unix://var/lib/kubelet/plugins/csi-rbdplugin/csi.sock
           imagePullPolicy: "IfNotPresent"
           volumeMounts:
             - name: plugin-dir
-              mountPath: /var/lib/kubelet/plugins/rbdplugin
+              mountPath: /var/lib/kubelet/plugins/csi-rbdplugin
             - name: pods-mount-dir
               mountPath: /var/lib/kubelet/pods
               mountPropagation: "Bidirectional"
@@ -111,7 +108,7 @@ spec:
       volumes:
         - name: plugin-dir
           hostPath:
-            path: /var/lib/kubelet/plugins/rbdplugin
+            path: /var/lib/kubelet/plugins/csi-rbdplugin
             type: DirectoryOrCreate
         - name: pods-mount-dir
           hostPath:
@@ -119,7 +116,7 @@ spec:
             type: Directory
         - name: socket-dir
           hostPath:
-            path: /var/lib/kubelet/plugins/rbdplugin
+            path: /var/lib/kubelet/plugins/csi-rbdplugin
             type: DirectoryOrCreate
         - name: host-dev
           hostPath:

pkg/rbd/controllerserver.go

@@ -20,12 +20,13 @@ import (
 	"fmt"
 	"path"
 
+	"github.com/container-storage-interface/spec/lib/go/csi/v0"
 	"github.com/golang/glog"
+	"github.com/kubernetes-csi/drivers/pkg/csi-common"
 	"github.com/pborman/uuid"
 	"golang.org/x/net/context"
-
+	"google.golang.org/grpc/codes"
-	"github.com/container-storage-interface/spec/lib/go/csi"
+	"google.golang.org/grpc/status"
-	"github.com/kubernetes-csi/drivers/pkg/csi-common"
 )
 
 const (
@@ -36,17 +37,42 @@ type controllerServer struct {
 	*csicommon.DefaultControllerServer
 }
 
-func GetVersionString(ver *csi.Version) string {
-	return fmt.Sprintf("%d.%d.%d", ver.Major, ver.Minor, ver.Patch)
-}
-
 func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) {
-	if err := cs.Driver.ValidateControllerServiceRequest(req.Version, csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME); err != nil {
+	if err := cs.Driver.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME); err != nil {
 		glog.V(3).Infof("invalid create volume req: %v", req)
 		return nil, err
 	}
+	// Check sanity of request Name, Volume Capabilities
+	if len(req.Name) == 0 {
+		return nil, status.Error(codes.InvalidArgument, "Volume Name cannot be empty")
+	}
+	if req.VolumeCapabilities == nil {
+		return nil, status.Error(codes.InvalidArgument, "Volume Capabilities cannot be empty")
+	}
 
-	volOptions, err := getRBDVolumeOptions(req.GetParameters())
+	// Need to check for already existing volume name, and if found
+	// check for the requested capacity and already allocated capacity
+	if exVol, err := getRBDVolumeByName(req.GetName()); err == nil {
+		// Since err is nil, it means the volume with the same name already exists
+		// need to check if the size of exisiting volume is the same as in new
+		// request
+		if exVol.VolSize >= int64(req.GetCapacityRange().GetRequiredBytes()) {
+			// exisiting volume is compatible with new request and should be reused.
+			// TODO (sbezverk) Do I need to make sure that RBD volume still exists?
+			return &csi.CreateVolumeResponse{
+				Volume: &csi.Volume{
+					Id:            exVol.VolID,
+					CapacityBytes: int64(exVol.VolSize),
+					Attributes:    req.GetParameters(),
+				},
+			}, nil
+		}
+		return nil, status.Error(codes.AlreadyExists, fmt.Sprintf("Volume with the same name: %s but with different size already exist", req.GetName()))
+	}
+
+	// TODO (sbezverk) Last check for not exceeding total storage capacity
+
+	rbdVol, err := getRBDVolumeOptions(req.GetParameters())
 	if err != nil {
 		return nil, err
 	}
@@ -55,21 +81,23 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
 	volName := req.GetName()
 	uniqueID := uuid.NewUUID().String()
 	if len(volName) == 0 {
-		volName = volOptions.Pool + "-dynamic-pvc-" + uniqueID
+		volName = rbdVol.Pool + "-dynamic-pvc-" + uniqueID
 	}
-	volOptions.VolName = volName
+	rbdVol.VolName = volName
 	volumeID := "csi-rbd-" + uniqueID
+	rbdVol.VolID = volumeID
 	// Volume Size - Default is 1 GiB
 	volSizeBytes := int64(oneGB)
 	if req.GetCapacityRange() != nil {
 		volSizeBytes = int64(req.GetCapacityRange().GetRequiredBytes())
 	}
+	rbdVol.VolSize = volSizeBytes
 	volSizeGB := int(volSizeBytes / 1024 / 1024 / 1024)
 
 	// Check if there is already RBD image with requested name
-	found, _, _ := rbdStatus(volOptions)
+	found, _, _ := rbdStatus(rbdVol, req.GetControllerCreateSecrets())
 	if !found {
-		if err := createRBDImage(volOptions, volSizeGB); err != nil {
+		if err := createRBDImage(rbdVol, volSizeGB, req.GetControllerCreateSecrets()); err != nil {
 			if err != nil {
 				glog.Warningf("failed to create volume: %v", err)
 				return nil, err
@@ -77,12 +105,11 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
 		}
 		glog.V(4).Infof("create volume %s", volName)
 	}
-	// Storing volInfo into a persistent file, will need info to delete rbd image
+	// Storing volInfo into a persistent file.
-	// in ControllerUnpublishVolume
+	if err := persistVolInfo(volumeID, path.Join(PluginFolder, "controller"), rbdVol); err != nil {
-	if err := persistVolInfo(volumeID, path.Join(PluginFolder, "controller"), volOptions); err != nil {
 		glog.Warningf("rbd: failed to store volInfo with error: %v", err)
 	}
-
+	rbdVolumes[volumeID] = *rbdVol
 	return &csi.CreateVolumeResponse{
 		Volume: &csi.Volume{
 			Id:            volumeID,
@@ -93,23 +120,21 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
 }
 
 func (cs *controllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) {
-	if err := cs.Driver.ValidateControllerServiceRequest(req.Version, csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME); err != nil {
+	if err := cs.Driver.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME); err != nil {
 		glog.Warningf("invalid delete volume req: %v", req)
 		return nil, err
 	}
-
 	// For now the image get unconditionally deleted, but here retention policy can be checked
 	volumeID := req.GetVolumeId()
-	volOptions := &rbdVolumeOptions{}
+	rbdVol := &rbdVolume{}
-	if err := loadVolInfo(volumeID, path.Join(PluginFolder, "controller"), volOptions); err != nil {
+	if err := loadVolInfo(volumeID, path.Join(PluginFolder, "controller"), rbdVol); err != nil {
 		return nil, err
 	}
-
+	volName := rbdVol.VolName
-	volName := volOptions.VolName
 	// Deleting rbd image
 	glog.V(4).Infof("deleting volume %s", volName)
-	if err := deleteRBDImage(volOptions); err != nil {
+	if err := deleteRBDImage(rbdVol, req.GetControllerDeleteSecrets()); err != nil {
-		glog.V(3).Infof("failed to delete rbd image: %s/%s with error: %v", volOptions.Pool, volName, err)
+		glog.V(3).Infof("failed to delete rbd image: %s/%s with error: %v", rbdVol.Pool, volName, err)
 		return nil, err
 	}
 	// Removing persistent storage file for the unmapped volume
@@ -117,6 +142,7 @@ func (cs *controllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVol
 		return nil, err
 	}
 
+	delete(rbdVolumes, volumeID)
 	return &csi.DeleteVolumeResponse{}, nil
 }
 

pkg/rbd/nodeserver.go

@@ -24,7 +24,7 @@ import (
 	"github.com/golang/glog"
 	"golang.org/x/net/context"
 
-	"github.com/container-storage-interface/spec/lib/go/csi"
+	"github.com/container-storage-interface/spec/lib/go/csi/v0"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 
@@ -67,7 +67,7 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis
 	}
 	volOptions.VolName = volName
 	// Mapping RBD image
-	devicePath, err := attachRBDImage(volOptions)
+	devicePath, err := attachRBDImage(volOptions, req.GetNodePublishSecrets())
 	if err != nil {
 		return nil, err
 	}
@@ -130,3 +130,19 @@ func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu
 
 	return &csi.NodeUnpublishVolumeResponse{}, nil
 }
+
+func (ns *nodeServer) NodeStageVolume(
+	ctx context.Context,
+	req *csi.NodeStageVolumeRequest) (
+	*csi.NodeStageVolumeResponse, error) {
+
+	return nil, status.Error(codes.Unimplemented, "")
+}
+
+func (ns *nodeServer) NodeUnstageVolume(
+	ctx context.Context,
+	req *csi.NodeUnstageVolumeRequest) (
+	*csi.NodeUnstageVolumeResponse, error) {
+
+	return nil, status.Error(codes.Unimplemented, "")
+}

pkg/rbd/rbd.go

@@ -17,15 +17,25 @@ limitations under the License.
 package rbd
 
 import (
+	"encoding/json"
+	"io/ioutil"
+	"os"
+	"path"
+	"strings"
+
 	"github.com/golang/glog"
 
-	"github.com/container-storage-interface/spec/lib/go/csi"
+	"github.com/container-storage-interface/spec/lib/go/csi/v0"
 	"github.com/kubernetes-csi/drivers/pkg/csi-common"
 )
 
 // PluginFolder defines the location of rbdplugin
 const (
-	PluginFolder = "/var/lib/kubelet/plugins/rbdplugin"
+	PluginFolder = "/var/lib/kubelet/plugins/csi-rbdplugin"
+	// RBDUserID used as a key in credentials map to extract the key which is
+	// passed be the provisioner, the value od RBDUserID must match to the key used
+	// in Secret object.
+	RBDUserID = "admin"
 )
 
 type rbd struct {
@@ -41,13 +51,54 @@ type rbd struct {
 
 var (
 	rbdDriver *rbd
-	version   = csi.Version{
+	version   = "0.2.0"
-		Minor: 2,
-	}
 )
 
-func GetSupportedVersions() []*csi.Version {
+var rbdVolumes map[string]rbdVolume
-	return []*csi.Version{&version}
+
+// Init checks for the persistent volume file and loads all found volumes
+// into a memory structure
+func init() {
+	rbdVolumes = map[string]rbdVolume{}
+	if _, err := os.Stat(path.Join(PluginFolder, "controller")); os.IsNotExist(err) {
+		glog.Infof("rbd: folder %s not found. Creating... \n", path.Join(PluginFolder, "controller"))
+		if err := os.Mkdir(path.Join(PluginFolder, "controller"), 0755); err != nil {
+			glog.Fatalf("Failed to create a controller's volumes folder with error: %v\n", err)
+		}
+		return
+	}
+	// Since "controller" folder exists, it means the rbdplugin has already been running, it means
+	// there might be some volumes left, they must be re-inserted into rbdVolumes map
+	loadExVolumes()
+}
+
+// loadExVolumes check for any *.json files in the  PluginFolder/controller folder
+// and loads then into rbdVolumes map
+func loadExVolumes() {
+	rbdVol := rbdVolume{}
+	files, err := ioutil.ReadDir(path.Join(PluginFolder, "controller"))
+	if err != nil {
+		glog.Infof("rbd: failed to read  controller's volumes folder: %s error:%v", path.Join(PluginFolder, "controller"), err)
+		return
+	}
+	for _, f := range files {
+		if !strings.HasSuffix(f.Name(), ".json") {
+			continue
+		}
+		fp, err := os.Open(path.Join(PluginFolder, "controller", f.Name()))
+		if err != nil {
+			glog.Infof("rbd: open file: %s err %%v", f.Name(), err)
+			continue
+		}
+		decoder := json.NewDecoder(fp)
+		if err = decoder.Decode(&rbdVol); err != nil {
+			glog.Infof("rbd: decode file: %s err: %v", f.Name(), err)
+			fp.Close()
+			continue
+		}
+		rbdVolumes[rbdVol.VolID] = rbdVol
+	}
+	glog.Infof("rbd: Loaded %d volumes from %s", len(rbdVolumes), path.Join(PluginFolder, "controller"))
 }
 
 func GetRBDDriver() *rbd {
@@ -73,10 +124,10 @@ func NewNodeServer(d *csicommon.CSIDriver) *nodeServer {
 }
 
 func (rbd *rbd) Run(driverName, nodeID, endpoint string) {
-	glog.Infof("Driver: %v version: %v", driverName, GetVersionString(&version))
+	glog.Infof("Driver: %v version: %v", driverName, version)
 
 	// Initialize default library driver
-	rbd.driver = csicommon.NewCSIDriver(driverName, &version, GetSupportedVersions(), nodeID)
+	rbd.driver = csicommon.NewCSIDriver(driverName, version, nodeID)
 	if rbd.driver == nil {
 		glog.Fatalln("Failed to initialize CSI Driver.")
 	}

pkg/rbd/rbd_util.go

@@ -19,15 +19,16 @@ package rbd
 import (
 	"encoding/json"
 	"fmt"
-	"github.com/golang/glog"
 	"io/ioutil"
-	"k8s.io/apimachinery/pkg/util/wait"
-	"k8s.io/kubernetes/pkg/util/keymutex"
 	"os"
 	"os/exec"
 	"path"
 	"strings"
 	"time"
+
+	"github.com/golang/glog"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/kubernetes/pkg/util/keymutex"
 )
 
 const (
@@ -44,22 +45,29 @@ const (
 	rbdImageWatcherSteps     = 10
 )
 
-type rbdVolumeOptions struct {
+type rbdVolume struct {
 	VolName     string `json:"volName"`
+	VolID       string `json:"volID"`
 	Monitors    string `json:"monitors"`
 	Pool        string `json:"pool"`
-	AdminID       string   `json:"adminID"`
-	AdminSecret   string   `json:"adminSecret"`
-	UserID        string   `json:"userID"`
-	UserSecret    string   `json:"userSecret"`
 	ImageFormat string `json:"imageFormat"`
+	// TODO (sbezverk) check if it is used and how
 	ImageFeatures []string `json:"imageFeatures"`
+	VolSize       int64    `json:"volSize"`
 }
 
 var attachdetachMutex = keymutex.NewKeyMutex()
 
+func getRBDKey(id string, credentials map[string]string) (string, error) {
+
+	if key, ok := credentials[id]; ok {
+		return key, nil
+	}
+	return "", fmt.Errorf("RBD key for ID: %s not found", id)
+}
+
 // CreateImage creates a new ceph image with provision and volume options.
-func createRBDImage(pOpts *rbdVolumeOptions, volSz int) error {
+func createRBDImage(pOpts *rbdVolume, volSz int, credentials map[string]string) error {
 	var output []byte
 	var err error
 
@@ -68,12 +76,16 @@ func createRBDImage(pOpts *rbdVolumeOptions, volSz int) error {
 	image := pOpts.VolName
 	volSzGB := fmt.Sprintf("%dG", volSz)
 
-	if pOpts.ImageFormat == rbdImageFormat2 {
+	key, err := getRBDKey(RBDUserID, credentials)
-		glog.V(4).Infof("rbd: create %s size %s format %s (features: %s) using mon %s, pool %s id %s key %s", image, volSzGB, pOpts.ImageFormat, pOpts.ImageFeatures, mon, pOpts.Pool, pOpts.AdminID, pOpts.AdminSecret)
+	if err != nil {
-	} else {
+		return err
-		glog.V(4).Infof("rbd: create %s size %s format %s using mon %s, pool %s id %s key %s", image, volSzGB, pOpts.ImageFormat, mon, pOpts.Pool, pOpts.AdminID, pOpts.AdminSecret)
 	}
-	args := []string{"create", image, "--size", volSzGB, "--pool", pOpts.Pool, "--id", pOpts.AdminID, "-m", mon, "--key=" + pOpts.AdminSecret, "--image-format", pOpts.ImageFormat}
+	if pOpts.ImageFormat == rbdImageFormat2 {
+		glog.V(4).Infof("rbd: create %s size %s format %s (features: %s) using mon %s, pool %s id %s key %s", image, volSzGB, pOpts.ImageFormat, pOpts.ImageFeatures, mon, pOpts.Pool, RBDUserID, key)
+	} else {
+		glog.V(4).Infof("rbd: create %s size %s format %s using mon %s, pool %s id %s key %s", image, volSzGB, pOpts.ImageFormat, mon, pOpts.Pool, RBDUserID, key)
+	}
+	args := []string{"create", image, "--size", volSzGB, "--pool", pOpts.Pool, "--id", RBDUserID, "-m", mon, "--key=" + key, "--image-format", pOpts.ImageFormat}
 	if pOpts.ImageFormat == rbdImageFormat2 {
 		// if no image features is provided, it results in empty string
 		// which disable all RBD image format 2 features as we expected
@@ -91,22 +103,20 @@ func createRBDImage(pOpts *rbdVolumeOptions, volSz int) error {
 
 // rbdStatus checks if there is watcher on the image.
 // It returns true if there is a watcher onthe image, otherwise returns false.
-func rbdStatus(b *rbdVolumeOptions) (bool, string, error) {
+func rbdStatus(pOpts *rbdVolume, credentials map[string]string) (bool, string, error) {
 	var err error
 	var output string
 	var cmd []byte
 
-	image := b.VolName
+	image := pOpts.VolName
 	// If we don't have admin id/secret (e.g. attaching), fallback to user id/secret.
-	id := b.AdminID
+	key, err := getRBDKey(RBDUserID, credentials)
-	secret := b.AdminSecret
+	if err != nil {
-	if id == "" {
+		return false, "", err
-		id = b.UserID
-		secret = b.UserSecret
 	}
 
-	glog.V(4).Infof("rbd: status %s using mon %s, pool %s id %s key %s", image, b.Monitors, b.Pool, id, secret)
+	glog.V(4).Infof("rbd: status %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, RBDUserID, key)
-	args := []string{"status", image, "--pool", b.Pool, "-m", b.Monitors, "--id", id, "--key=" + secret}
+	args := []string{"status", image, "--pool", pOpts.Pool, "-m", pOpts.Monitors, "--id", RBDUserID, "--key=" + key}
 	cmd, err = execCommand("rbd", args)
 	output = string(cmd)
 
@@ -133,10 +143,10 @@ func rbdStatus(b *rbdVolumeOptions) (bool, string, error) {
 }
 
 // DeleteImage deletes a ceph image with provision and volume options.
-func deleteRBDImage(b *rbdVolumeOptions) error {
+func deleteRBDImage(pOpts *rbdVolume, credentials map[string]string) error {
 	var output []byte
-	image := b.VolName
+	image := pOpts.VolName
-	found, _, err := rbdStatus(b)
+	found, _, err := rbdStatus(pOpts, credentials)
 	if err != nil {
 		return err
 	}
@@ -144,15 +154,13 @@ func deleteRBDImage(b *rbdVolumeOptions) error {
 		glog.Info("rbd is still being used ", image)
 		return fmt.Errorf("rbd %s is still being used", image)
 	}
-	id := b.AdminID
+	key, err := getRBDKey(RBDUserID, credentials)
-	secret := b.AdminSecret
+	if err != nil {
-	if id == "" {
+		return err
-		id = b.UserID
-		secret = b.UserSecret
 	}
 
-	glog.V(4).Infof("rbd: rm %s using mon %s, pool %s id %s key %s", image, b.Monitors, b.Pool, id, secret)
+	glog.V(4).Infof("rbd: rm %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, RBDUserID, key)
-	args := []string{"rm", image, "--pool", b.Pool, "--id", id, "-m", b.Monitors, "--key=" + secret}
+	args := []string{"rm", image, "--pool", pOpts.Pool, "--id", RBDUserID, "-m", pOpts.Monitors, "--key=" + key}
 	output, err = execCommand("rbd", args)
 	if err == nil {
 		return nil
@@ -166,42 +174,26 @@ func execCommand(command string, args []string) ([]byte, error) {
 	return cmd.CombinedOutput()
 }
 
-func getRBDVolumeOptions(volOptions map[string]string) (*rbdVolumeOptions, error) {
+func getRBDVolumeOptions(volOptions map[string]string) (*rbdVolume, error) {
-	rbdVolume := &rbdVolumeOptions{}
 	var ok bool
-	rbdVolume.AdminID, ok = volOptions["adminID"]
+	rbdVol := &rbdVolume{}
-	if !ok {
+	rbdVol.Pool, ok = volOptions["pool"]
-		return nil, fmt.Errorf("Missing required parameter adminID")
-	}
-	rbdVolume.AdminSecret, ok = volOptions["adminSecret"]
-	if !ok {
-		return nil, fmt.Errorf("Missing required parameter adminSecret")
-	}
-	rbdVolume.Pool, ok = volOptions["pool"]
 	if !ok {
 		return nil, fmt.Errorf("Missing required parameter pool")
 	}
-	rbdVolume.Monitors, ok = volOptions["monitors"]
+	rbdVol.Monitors, ok = volOptions["monitors"]
 	if !ok {
 		return nil, fmt.Errorf("Missing required parameter monitors")
 	}
-	rbdVolume.UserID, ok = volOptions["userID"]
+	rbdVol.ImageFormat, ok = volOptions["imageFormat"]
 	if !ok {
-		return nil, fmt.Errorf("Missing required parameter userID")
+		rbdVol.ImageFormat = "2"
-	}
-	rbdVolume.UserSecret, ok = volOptions["userSecret"]
-	if !ok {
-		return nil, fmt.Errorf("Missing required parameter userSecret")
-	}
-	rbdVolume.ImageFormat, ok = volOptions["imageFormat"]
-	if !ok {
-		rbdVolume.ImageFormat = "2"
 	}
 
-	return rbdVolume, nil
+	return rbdVol, nil
 }
 
-func attachRBDImage(volOptions *rbdVolumeOptions) (string, error) {
+func attachRBDImage(volOptions *rbdVolume, credentials map[string]string) (string, error) {
 	var err error
 	var output []byte
 
@@ -222,7 +214,7 @@ func attachRBDImage(volOptions *rbdVolumeOptions) (string, error) {
 			Steps:    rbdImageWatcherSteps,
 		}
 		err := wait.ExponentialBackoff(backoff, func() (bool, error) {
-			used, rbdOutput, err := rbdStatus(volOptions)
+			used, rbdOutput, err := rbdStatus(volOptions, credentials)
 			if err != nil {
 				return false, fmt.Errorf("fail to check rbd image status with: (%v), rbd output: (%s)", err, rbdOutput)
 			}
@@ -238,11 +230,13 @@ func attachRBDImage(volOptions *rbdVolumeOptions) (string, error) {
 		}
 
 		glog.V(1).Infof("rbd: map mon %s", volOptions.Monitors)
-		id := volOptions.UserID
+		key, err := getRBDKey(RBDUserID, credentials)
-		secret := volOptions.UserSecret
+		if err != nil {
+			return "", err
+		}
 
 		output, err = execCommand("rbd", []string{
-			"map", image, "--pool", volOptions.Pool, "--id", id, "-m", volOptions.Monitors, "--key=" + secret})
+			"map", image, "--pool", volOptions.Pool, "--id", RBDUserID, "-m", volOptions.Monitors, "--key=" + key})
 		if err != nil {
 			glog.V(1).Infof("rbd: map error %v, rbd output: %s", err, string(output))
 			return "", fmt.Errorf("rbd: map failed %v, rbd output: %s", err, string(output))
@@ -322,23 +316,24 @@ func waitForPath(pool, image string, maxRetries int) (string, bool) {
 	return "", false
 }
 
-func persistVolInfo(image string, persistentStoragePath string, volInfo *rbdVolumeOptions) error {
+func persistVolInfo(image string, persistentStoragePath string, volInfo *rbdVolume) error {
 	file := path.Join(persistentStoragePath, image+".json")
 	fp, err := os.Create(file)
 	if err != nil {
+		glog.Errorf("rbd: failed to create persistent storage file %s with error: %v\n", file, err)
 		return fmt.Errorf("rbd: create err %s/%s", file, err)
 	}
 	defer fp.Close()
-
 	encoder := json.NewEncoder(fp)
 	if err = encoder.Encode(volInfo); err != nil {
-		return fmt.Errorf("rbd: encode err: %v.", err)
+		glog.Errorf("rbd: failed to encode volInfo: %+v for file: %s with error: %v\n", volInfo, file, err)
+		return fmt.Errorf("rbd: encode err: %v", err)
 	}
-
+	glog.Infof("rbd: successfully saved volInfo: %+v into file: %s\n", volInfo, file)
 	return nil
 }
 
-func loadVolInfo(image string, persistentStoragePath string, volInfo *rbdVolumeOptions) error {
+func loadVolInfo(image string, persistentStoragePath string, volInfo *rbdVolume) error {
 	file := path.Join(persistentStoragePath, image+".json")
 	fp, err := os.Open(file)
 	if err != nil {
@@ -356,11 +351,28 @@ func loadVolInfo(image string, persistentStoragePath string, volInfo *rbdVolumeO
 
 func deleteVolInfo(image string, persistentStoragePath string) error {
 	file := path.Join(persistentStoragePath, image+".json")
+	glog.Infof("rbd: Deleting file for Volume: %s at: %s resulting path: %+v\n", image, persistentStoragePath, file)
 	err := os.Remove(file)
 	if err != nil {
 		if err != os.ErrNotExist {
-			return fmt.Errorf("rbd: open err %s/%s", file, err)
+			return fmt.Errorf("rbd: error removing file: %s/%s", file, err)
 		}
 	}
 	return nil
 }
+
+func getRBDVolumeByID(volumeID string) (rbdVolume, error) {
+	if rbdVol, ok := rbdVolumes[volumeID]; ok {
+		return rbdVol, nil
+	}
+	return rbdVolume{}, fmt.Errorf("volume id %s does not exit in the volumes list", volumeID)
+}
+
+func getRBDVolumeByName(volName string) (rbdVolume, error) {
+	for _, rbdVol := range rbdVolumes {
+		if rbdVol.VolName == volName {
+			return rbdVol, nil
+		}
+	}
+	return rbdVolume{}, fmt.Errorf("volume name %s does not exit in the volumes list", volName)
+}

rbd/main.go

@@ -31,7 +31,7 @@ func init() {
 
 var (
 	endpoint   = flag.String("endpoint", "unix://tmp/csi.sock", "CSI endpoint")
-	driverName = flag.String("drivername", "rbdplugin", "name of the driver")
+	driverName = flag.String("drivername", "csi-rbdplugin", "name of the driver")
 	nodeID     = flag.String("nodeid", "", "node id")
 )