diff --git a/.gitignore b/.gitignore index a20ef19ba..83354949f 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,9 @@ # docker build /deploy/docker/rbdplugin +# rbdplugin executable +rbdplugin + # Emacs save files *~ \#*\# diff --git a/deploy/docker/Dockerfile b/Dockerfile similarity index 89% rename from deploy/docker/Dockerfile rename to Dockerfile index 8a35757e4..2f8537c19 100644 --- a/deploy/docker/Dockerfile +++ b/Dockerfile @@ -7,6 +7,6 @@ RUN yum install -y centos-release-ceph && \ yum install -y ceph-common e2fsprogs && \ yum clean all -COPY rbdplugin /rbdplugin +COPY _output/rbdplugin /rbdplugin RUN chmod +x /rbdplugin ENTRYPOINT ["/rbdplugin"] diff --git a/Gopkg.lock b/Gopkg.lock index 8b92f56da..f7aea2634 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -2,10 +2,10 @@ [[projects]] - branch = "master" name = "github.com/container-storage-interface/spec" - packages = ["lib/go/csi"] - revision = "7ab01a90da87f9fef3ee1de0494962fdefaf7db7" + packages = ["lib/go/csi/v0"] + revision = "35d9f9d77954980e449e52c3f3e43c21bd8171f5" + version = "v0.2.0" [[projects]] branch = "master" @@ -14,16 +14,22 @@ revision = "23def4e6c14b4da8ac2ed8007337bc5eb5007998" [[projects]] - branch = "master" name = "github.com/golang/protobuf" - packages = ["proto","ptypes","ptypes/any","ptypes/duration","ptypes/timestamp"] - revision = "1e59b77b52bf8e4b449a57e6f79f21226d571845" + packages = [ + "proto", + "ptypes", + "ptypes/any", + "ptypes/duration", + "ptypes/timestamp" + ] + revision = "925541529c1fa6821df4e44ce2723319eb2be768" + version = "v1.0.0" [[projects]] branch = "master" name = "github.com/kubernetes-csi/drivers" packages = ["pkg/csi-common"] - revision = "d1ab787ad5510df08a3a98a091a41adeae4647b4" + revision = "1853bd0038cd634f277efda5c6548766a2a51ff3" [[projects]] name = "github.com/pborman/uuid" @@ -34,54 +40,110 @@ [[projects]] branch = "master" name = "golang.org/x/net" - packages = ["context","http2","http2/hpack","idna","internal/timeseries","lex/httplex","trace"] - revision = "42fe2e1c20de1054d3d30f82cc9fb5b41e2e3767" + packages = [ + "context", + "http2", + "http2/hpack", + "idna", + "internal/timeseries", + "lex/httplex", + "trace" + ] + revision = "d25186b37f34ebdbbea8f488ef055638dfab272d" [[projects]] branch = "master" name = "golang.org/x/sys" packages = ["unix"] - revision = "1792d66dc88e503d3cb2400578221cdf1f7fe26f" + revision = "dd2ff4accc098aceecb86b36eaa7829b2a17b1c9" [[projects]] - branch = "master" name = "golang.org/x/text" - packages = ["collate","collate/build","internal/colltab","internal/gen","internal/tag","internal/triegen","internal/ucd","language","secure/bidirule","transform","unicode/bidi","unicode/cldr","unicode/norm","unicode/rangetable"] - revision = "e19ae1496984b1c655b8044a65c0300a3c878dd3" + packages = [ + "collate", + "collate/build", + "internal/colltab", + "internal/gen", + "internal/tag", + "internal/triegen", + "internal/ucd", + "language", + "secure/bidirule", + "transform", + "unicode/bidi", + "unicode/cldr", + "unicode/norm", + "unicode/rangetable" + ] + revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" + version = "v0.3.0" [[projects]] branch = "master" name = "google.golang.org/genproto" packages = ["googleapis/rpc/status"] - revision = "a8101f21cf983e773d0c1133ebc5424792003214" + revision = "df60624c1e9b9d2973e889c7a1cff73155da81c4" [[projects]] name = "google.golang.org/grpc" - packages = [".","balancer","balancer/base","balancer/roundrobin","codes","connectivity","credentials","encoding","grpclb/grpc_lb_v1/messages","grpclog","internal","keepalive","metadata","naming","peer","resolver","resolver/dns","resolver/passthrough","stats","status","tap","transport"] - revision = "f3955b8e9e244dd4dd4bc4f7b7a23a8445400a76" - version = "v1.9.0" + packages = [ + ".", + "balancer", + "balancer/base", + "balancer/roundrobin", + "codes", + "connectivity", + "credentials", + "encoding", + "encoding/proto", + "grpclb/grpc_lb_v1/messages", + "grpclog", + "internal", + "keepalive", + "metadata", + "naming", + "peer", + "resolver", + "resolver/dns", + "resolver/passthrough", + "stats", + "status", + "tap", + "transport" + ] + revision = "8e4536a86ab602859c20df5ebfd0bd4228d08655" + version = "v1.10.0" [[projects]] - branch = "release-1.9" name = "k8s.io/apimachinery" - packages = ["pkg/util/runtime","pkg/util/sets","pkg/util/wait"] - revision = "68f9c3a1feb3140df59c67ced62d3a5df8e6c9c2" + packages = [ + "pkg/util/runtime", + "pkg/util/sets", + "pkg/util/wait" + ] + revision = "302974c03f7e50f16561ba237db776ab93594ef6" + version = "kubernetes-1.10.0-beta.1" [[projects]] name = "k8s.io/kubernetes" - packages = ["pkg/util/io","pkg/util/keymutex","pkg/util/mount","pkg/util/nsenter"] - revision = "3a1c9449a956b6026f075fa3134ff92f7d55f812" - version = "v1.9.1" + packages = [ + "pkg/util/io", + "pkg/util/keymutex", + "pkg/util/mount", + "pkg/util/nsenter" + ] + revision = "37555e6d24c2f951c40660ea59a80fa251982005" + version = "v1.10.0-beta.1" [[projects]] branch = "master" name = "k8s.io/utils" packages = ["exec"] - revision = "a99a3e11a96751670db62ba77c6d278d1136931e" + revision = "258e2a2fa64568210fbd6267cf1d8fd87c3cb86e" [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "8908f89154f277d98fd83b22edf73652d4c4e37bbd827bf11d9605c58ae3fd0e" + inputs-digest = "d409396ee410b5443abac732df725cb70ef9e0c940c1b807c0ff2698bab5d102" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 2ef13999d..edb066925 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -1,23 +1,31 @@ +[[constraint]] + name = "github.com/container-storage-interface/spec" + version = "~0.2.0" + [[constraint]] branch = "master" - name = "github.com/container-storage-interface/spec" + name = "github.com/kubernetes-csi/drivers" [[constraint]] branch = "master" name = "github.com/golang/glog" -[[constraint]] - name = "google.golang.org/grpc" - version = "1.7.2" +[[override]] + revision = "5db89f0ca68677abc5eefce8f2a0a772c98ba52d" + name = "github.com/docker/distribution" [[constraint]] - name = "github.com/docker/distribution" - revision = "edc3ab29cdff8694dd6feb85cfeb4b5f1b38ed9c" + name = "google.golang.org/grpc" + version = "1.10.0" + +[[constraint]] + version = "kubernetes-1.10.0-beta.1" + name = "k8s.io/apimachinery" [[constraint]] name = "k8s.io/kubernetes" - version = "v1.9.1" + version = "v1.10.0-beta.1" -[[constraint]] - name = "k8s.io/apimachinery" - version = "kubernetes-1.9.1" +[[override]] + version = "kubernetes-1.10.0-beta.1" + name = "k8s.io/api" diff --git a/Makefile b/Makefile index 7b2744342..b921112e6 100644 --- a/Makefile +++ b/Makefile @@ -28,12 +28,10 @@ rbdplugin: CGO_ENABLED=0 GOOS=linux go build -a -ldflags '-extldflags "-static"' -o _output/rbdplugin ./rbd container: rbdplugin - cp _output/rbdplugin deploy/docker - docker build -t $(IMAGE_NAME):$(IMAGE_VERSION) deploy/docker + docker build -t $(IMAGE_NAME):$(IMAGE_VERSION) . push-container: container docker push $(IMAGE_NAME):$(IMAGE_VERSION) clean: go clean -r -x - rm -f deploy/docker/rbdplugin -rm -rf _output diff --git a/README.md b/README.md index aae282085..18cbe2cc0 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ The following output should be displayed: ``` NAMESPACE NAME READY STATUS RESTARTS AGE default csi-attacher-0 1/1 Running 0 1d -default csi-nodeplugin-rbdplugin-qxqtl 2/2 Running 0 1d +default csi-rbdplugin-qxqtl 2/2 Running 0 1d default csi-provisioner-0 1/1 Running 0 1d ``` diff --git a/deploy/kubernetes/csi-attacher.yaml b/deploy/kubernetes/csi-attacher.yaml index 290eddf7d..142890a7d 100644 --- a/deploy/kubernetes/csi-attacher.yaml +++ b/deploy/kubernetes/csi-attacher.yaml @@ -12,6 +12,9 @@ apiVersion: rbac.authorization.k8s.io/v1 metadata: name: external-attacher-runner rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "update"] - apiGroups: [""] resources: ["persistentvolumes"] verbs: ["get", "list", "watch", "update"] @@ -66,19 +69,19 @@ spec: serviceAccount: csi-attacher containers: - name: csi-attacher - image: docker.io/k8scsi/csi-attacher:latest + image: quay.io/k8scsi/csi-attacher:v0.2.0 args: - "--v=5" - "--csi-address=$(ADDRESS)" env: - name: ADDRESS - value: /var/lib/kubelet/plugins/rbdplugin/csi.sock + value: /var/lib/kubelet/plugins/csi-rbdplugin/csi.sock imagePullPolicy: "IfNotPresent" volumeMounts: - name: socket-dir - mountPath: /var/lib/kubelet/plugins/rbdplugin + mountPath: /var/lib/kubelet/plugins/csi-rbdplugin volumes: - name: socket-dir hostPath: - path: /var/lib/kubelet/plugins/rbdplugin + path: /var/lib/kubelet/plugins/csi-rbdplugin type: DirectoryOrCreate diff --git a/deploy/kubernetes/csi-provisioner.yaml b/deploy/kubernetes/csi-provisioner.yaml index 6758e3a92..fb9ed0170 100644 --- a/deploy/kubernetes/csi-provisioner.yaml +++ b/deploy/kubernetes/csi-provisioner.yaml @@ -18,6 +18,9 @@ apiVersion: rbac.authorization.k8s.io/v1 metadata: name: external-provisioner-runner rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] - apiGroups: [""] resources: ["persistentvolumes"] verbs: ["get", "list", "watch", "create", "delete"] @@ -75,20 +78,20 @@ spec: serviceAccount: csi-provisioner containers: - name: csi-provisioner - image: quay.io/k8scsi/csi-provisioner:latest + image: quay.io/k8scsi/csi-provisioner:v0.2.0 args: - - "--provisioner=rbdplugin" + - "--provisioner=csi-rbdplugin" - "--csi-address=$(ADDRESS)" - "--v=5" env: - name: ADDRESS - value: /var/lib/kubelet/plugins/rbdplugin/csi.sock + value: /var/lib/kubelet/plugins/csi-rbdplugin/csi.sock imagePullPolicy: "IfNotPresent" volumeMounts: - name: socket-dir - mountPath: /var/lib/kubelet/plugins/rbdplugin + mountPath: /var/lib/kubelet/plugins/csi-rbdplugin volumes: - name: socket-dir hostPath: - path: /var/lib/kubelet/plugins/rbdplugin + path: /var/lib/kubelet/plugins/csi-rbdplugin type: DirectoryOrCreate diff --git a/deploy/kubernetes/rbd-secrets.yaml b/deploy/kubernetes/rbd-secrets.yaml index 160d9b87b..7f13e2681 100644 --- a/deploy/kubernetes/rbd-secrets.yaml +++ b/deploy/kubernetes/rbd-secrets.yaml @@ -1,19 +1,10 @@ apiVersion: v1 kind: Secret metadata: - name: ceph-secret-admin - namespace: kube-system -type: "kubernetes.io/rbd" + name: csi-ceph-secret + namespace: default data: #Please note this value is base64 encoded. - key: QVFDZUhPMVpJTFBQRFJBQTd6dzNkNzZicGxrdlR3em9vc3lidkE9PQo= -type: kubernetes.io/rbd ---- -apiVersion: v1 -kind: Secret -metadata: - name: ceph-secret-user -type: "kubernetes.io/rbd" -data: -#Please note this value is base64 encoded. - key: QVFDZDR1MVoxSDI0QnhBQWFxdmZIRnFuMSs0RFZlK1pRZ0ZmUEE9PQo= +# Key value corresponds to a user name defined in ceph cluster + admin: QVFDZUhPMVpJTFBQRFJBQTd6dzNkNzZicGxrdlR3em9vc3lidkE9PQo= + kubernetes: QVFDZDR1MVoxSDI0QnhBQWFxdmZIRnFuMSs0RFZlK1pRZ0ZmUEE9PQo= \ No newline at end of file diff --git a/deploy/kubernetes/rbd-storage-class.yaml b/deploy/kubernetes/rbd-storage-class.yaml index c5fb54c8c..eb9afeef7 100644 --- a/deploy/kubernetes/rbd-storage-class.yaml +++ b/deploy/kubernetes/rbd-storage-class.yaml @@ -1,13 +1,11 @@ apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: - name: rbd -provisioner: rbdplugin + name: csi-rbd +provisioner: csi-rbdplugin parameters: monitors: 192.168.80.233:6789 pool: kubernetes - adminID: admin - adminSecret: AQAmsGBap4EoBhAAET/Hc7fBqAZj/cy7cDcoQA== - userID: kube - userSecret: AQAMgXhVwBCeDhAA9nlPaFyfUSatGD4drFWDvQ== + csiProvisionerSecretName: csi-ceph-secret + csiProvisionerSecretNamespace: default reclaimPolicy: Delete diff --git a/deploy/kubernetes/rbdplugin.yaml b/deploy/kubernetes/rbdplugin.yaml index d5f60cabd..4eef8e86b 100644 --- a/deploy/kubernetes/rbdplugin.yaml +++ b/deploy/kubernetes/rbdplugin.yaml @@ -3,20 +3,17 @@ apiVersion: v1 kind: ServiceAccount metadata: - name: csi-nodeplugin + name: csi-rbdplugin --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: csi-nodeplugin + name: csi-rbdplugin rules: - apiGroups: [""] resources: ["nodes"] verbs: ["get", "list", "update"] - - apiGroups: [""] - resources: ["secrets"] - verbs: ["get", "list"] - apiGroups: [""] resources: ["namespaces"] verbs: ["get", "list"] @@ -30,14 +27,14 @@ rules: kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: csi-nodeplugin + name: csi-rbdplugin subjects: - kind: ServiceAccount - name: csi-nodeplugin + name: csi-rbdplugin namespace: default roleRef: kind: ClusterRole - name: csi-nodeplugin + name: csi-rbdplugin apiGroup: rbac.authorization.k8s.io --- @@ -47,35 +44,35 @@ roleRef: kind: DaemonSet apiVersion: apps/v1beta2 metadata: - name: csi-nodeplugin-rbdplugin + name: csi-rbdplugin spec: selector: matchLabels: - app: csi-nodeplugin-rbdplugin + app: csi-rbdplugin template: metadata: labels: - app: csi-nodeplugin-rbdplugin + app: csi-rbdplugin spec: - serviceAccount: csi-nodeplugin + serviceAccount: csi-rbdplugin hostNetwork: true containers: - name: driver-registrar - image: docker.io/k8scsi/driver-registrar:latest + image: quay.io/k8scsi/driver-registrar:v0.2.0 args: - "--v=5" - "--csi-address=$(ADDRESS)" env: - name: ADDRESS - value: /var/lib/kubelet/plugins/rbdplugin/csi.sock + value: /var/lib/kubelet/plugins/csi-rbdplugin/csi.sock - name: KUBE_NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName volumeMounts: - name: socket-dir - mountPath: /var/lib/kubelet/plugins/rbdplugin - - name: rbdplugin + mountPath: /var/lib/kubelet/plugins/csi-rbdplugin + - name: csi-rbdplugin securityContext: privileged: true capabilities: @@ -86,18 +83,18 @@ spec: - "--nodeid=$(NODE_ID)" - "--endpoint=$(CSI_ENDPOINT)" - "--v=5" - - "--drivername=rbdplugin" + - "--drivername=csi-rbdplugin" env: - name: NODE_ID valueFrom: fieldRef: fieldPath: spec.nodeName - name: CSI_ENDPOINT - value: unix://var/lib/kubelet/plugins/rbdplugin/csi.sock + value: unix://var/lib/kubelet/plugins/csi-rbdplugin/csi.sock imagePullPolicy: "IfNotPresent" volumeMounts: - name: plugin-dir - mountPath: /var/lib/kubelet/plugins/rbdplugin + mountPath: /var/lib/kubelet/plugins/csi-rbdplugin - name: pods-mount-dir mountPath: /var/lib/kubelet/pods mountPropagation: "Bidirectional" @@ -111,7 +108,7 @@ spec: volumes: - name: plugin-dir hostPath: - path: /var/lib/kubelet/plugins/rbdplugin + path: /var/lib/kubelet/plugins/csi-rbdplugin type: DirectoryOrCreate - name: pods-mount-dir hostPath: @@ -119,7 +116,7 @@ spec: type: Directory - name: socket-dir hostPath: - path: /var/lib/kubelet/plugins/rbdplugin + path: /var/lib/kubelet/plugins/csi-rbdplugin type: DirectoryOrCreate - name: host-dev hostPath: @@ -129,4 +126,4 @@ spec: path: /sys - name: lib-modules hostPath: - path: /lib/modules + path: /lib/modules \ No newline at end of file diff --git a/pkg/rbd/controllerserver.go b/pkg/rbd/controllerserver.go index 3073c3e5a..4c986e776 100644 --- a/pkg/rbd/controllerserver.go +++ b/pkg/rbd/controllerserver.go @@ -20,12 +20,13 @@ import ( "fmt" "path" + "github.com/container-storage-interface/spec/lib/go/csi/v0" "github.com/golang/glog" + "github.com/kubernetes-csi/drivers/pkg/csi-common" "github.com/pborman/uuid" "golang.org/x/net/context" - - "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/kubernetes-csi/drivers/pkg/csi-common" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) const ( @@ -36,17 +37,42 @@ type controllerServer struct { *csicommon.DefaultControllerServer } -func GetVersionString(ver *csi.Version) string { - return fmt.Sprintf("%d.%d.%d", ver.Major, ver.Minor, ver.Patch) -} - func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) { - if err := cs.Driver.ValidateControllerServiceRequest(req.Version, csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME); err != nil { + if err := cs.Driver.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME); err != nil { glog.V(3).Infof("invalid create volume req: %v", req) return nil, err } + // Check sanity of request Name, Volume Capabilities + if len(req.Name) == 0 { + return nil, status.Error(codes.InvalidArgument, "Volume Name cannot be empty") + } + if req.VolumeCapabilities == nil { + return nil, status.Error(codes.InvalidArgument, "Volume Capabilities cannot be empty") + } - volOptions, err := getRBDVolumeOptions(req.GetParameters()) + // Need to check for already existing volume name, and if found + // check for the requested capacity and already allocated capacity + if exVol, err := getRBDVolumeByName(req.GetName()); err == nil { + // Since err is nil, it means the volume with the same name already exists + // need to check if the size of exisiting volume is the same as in new + // request + if exVol.VolSize >= int64(req.GetCapacityRange().GetRequiredBytes()) { + // exisiting volume is compatible with new request and should be reused. + // TODO (sbezverk) Do I need to make sure that RBD volume still exists? + return &csi.CreateVolumeResponse{ + Volume: &csi.Volume{ + Id: exVol.VolID, + CapacityBytes: int64(exVol.VolSize), + Attributes: req.GetParameters(), + }, + }, nil + } + return nil, status.Error(codes.AlreadyExists, fmt.Sprintf("Volume with the same name: %s but with different size already exist", req.GetName())) + } + + // TODO (sbezverk) Last check for not exceeding total storage capacity + + rbdVol, err := getRBDVolumeOptions(req.GetParameters()) if err != nil { return nil, err } @@ -55,21 +81,23 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol volName := req.GetName() uniqueID := uuid.NewUUID().String() if len(volName) == 0 { - volName = volOptions.Pool + "-dynamic-pvc-" + uniqueID + volName = rbdVol.Pool + "-dynamic-pvc-" + uniqueID } - volOptions.VolName = volName + rbdVol.VolName = volName volumeID := "csi-rbd-" + uniqueID + rbdVol.VolID = volumeID // Volume Size - Default is 1 GiB volSizeBytes := int64(oneGB) if req.GetCapacityRange() != nil { volSizeBytes = int64(req.GetCapacityRange().GetRequiredBytes()) } + rbdVol.VolSize = volSizeBytes volSizeGB := int(volSizeBytes / 1024 / 1024 / 1024) // Check if there is already RBD image with requested name - found, _, _ := rbdStatus(volOptions) + found, _, _ := rbdStatus(rbdVol, req.GetControllerCreateSecrets()) if !found { - if err := createRBDImage(volOptions, volSizeGB); err != nil { + if err := createRBDImage(rbdVol, volSizeGB, req.GetControllerCreateSecrets()); err != nil { if err != nil { glog.Warningf("failed to create volume: %v", err) return nil, err @@ -77,12 +105,11 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol } glog.V(4).Infof("create volume %s", volName) } - // Storing volInfo into a persistent file, will need info to delete rbd image - // in ControllerUnpublishVolume - if err := persistVolInfo(volumeID, path.Join(PluginFolder, "controller"), volOptions); err != nil { + // Storing volInfo into a persistent file. + if err := persistVolInfo(volumeID, path.Join(PluginFolder, "controller"), rbdVol); err != nil { glog.Warningf("rbd: failed to store volInfo with error: %v", err) } - + rbdVolumes[volumeID] = *rbdVol return &csi.CreateVolumeResponse{ Volume: &csi.Volume{ Id: volumeID, @@ -93,23 +120,21 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol } func (cs *controllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) { - if err := cs.Driver.ValidateControllerServiceRequest(req.Version, csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME); err != nil { + if err := cs.Driver.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME); err != nil { glog.Warningf("invalid delete volume req: %v", req) return nil, err } - // For now the image get unconditionally deleted, but here retention policy can be checked volumeID := req.GetVolumeId() - volOptions := &rbdVolumeOptions{} - if err := loadVolInfo(volumeID, path.Join(PluginFolder, "controller"), volOptions); err != nil { + rbdVol := &rbdVolume{} + if err := loadVolInfo(volumeID, path.Join(PluginFolder, "controller"), rbdVol); err != nil { return nil, err } - - volName := volOptions.VolName + volName := rbdVol.VolName // Deleting rbd image glog.V(4).Infof("deleting volume %s", volName) - if err := deleteRBDImage(volOptions); err != nil { - glog.V(3).Infof("failed to delete rbd image: %s/%s with error: %v", volOptions.Pool, volName, err) + if err := deleteRBDImage(rbdVol, req.GetControllerDeleteSecrets()); err != nil { + glog.V(3).Infof("failed to delete rbd image: %s/%s with error: %v", rbdVol.Pool, volName, err) return nil, err } // Removing persistent storage file for the unmapped volume @@ -117,6 +142,7 @@ func (cs *controllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVol return nil, err } + delete(rbdVolumes, volumeID) return &csi.DeleteVolumeResponse{}, nil } diff --git a/pkg/rbd/nodeserver.go b/pkg/rbd/nodeserver.go index cbbed8792..7091ca99c 100644 --- a/pkg/rbd/nodeserver.go +++ b/pkg/rbd/nodeserver.go @@ -24,7 +24,7 @@ import ( "github.com/golang/glog" "golang.org/x/net/context" - "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/container-storage-interface/spec/lib/go/csi/v0" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -67,7 +67,7 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis } volOptions.VolName = volName // Mapping RBD image - devicePath, err := attachRBDImage(volOptions) + devicePath, err := attachRBDImage(volOptions, req.GetNodePublishSecrets()) if err != nil { return nil, err } @@ -130,3 +130,19 @@ func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu return &csi.NodeUnpublishVolumeResponse{}, nil } + +func (ns *nodeServer) NodeStageVolume( + ctx context.Context, + req *csi.NodeStageVolumeRequest) ( + *csi.NodeStageVolumeResponse, error) { + + return nil, status.Error(codes.Unimplemented, "") +} + +func (ns *nodeServer) NodeUnstageVolume( + ctx context.Context, + req *csi.NodeUnstageVolumeRequest) ( + *csi.NodeUnstageVolumeResponse, error) { + + return nil, status.Error(codes.Unimplemented, "") +} diff --git a/pkg/rbd/rbd.go b/pkg/rbd/rbd.go index 3ecf40141..569e97219 100644 --- a/pkg/rbd/rbd.go +++ b/pkg/rbd/rbd.go @@ -17,15 +17,25 @@ limitations under the License. package rbd import ( + "encoding/json" + "io/ioutil" + "os" + "path" + "strings" + "github.com/golang/glog" - "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/container-storage-interface/spec/lib/go/csi/v0" "github.com/kubernetes-csi/drivers/pkg/csi-common" ) // PluginFolder defines the location of rbdplugin const ( - PluginFolder = "/var/lib/kubelet/plugins/rbdplugin" + PluginFolder = "/var/lib/kubelet/plugins/csi-rbdplugin" + // RBDUserID used as a key in credentials map to extract the key which is + // passed be the provisioner, the value od RBDUserID must match to the key used + // in Secret object. + RBDUserID = "admin" ) type rbd struct { @@ -41,13 +51,54 @@ type rbd struct { var ( rbdDriver *rbd - version = csi.Version{ - Minor: 2, - } + version = "0.2.0" ) -func GetSupportedVersions() []*csi.Version { - return []*csi.Version{&version} +var rbdVolumes map[string]rbdVolume + +// Init checks for the persistent volume file and loads all found volumes +// into a memory structure +func init() { + rbdVolumes = map[string]rbdVolume{} + if _, err := os.Stat(path.Join(PluginFolder, "controller")); os.IsNotExist(err) { + glog.Infof("rbd: folder %s not found. Creating... \n", path.Join(PluginFolder, "controller")) + if err := os.Mkdir(path.Join(PluginFolder, "controller"), 0755); err != nil { + glog.Fatalf("Failed to create a controller's volumes folder with error: %v\n", err) + } + return + } + // Since "controller" folder exists, it means the rbdplugin has already been running, it means + // there might be some volumes left, they must be re-inserted into rbdVolumes map + loadExVolumes() +} + +// loadExVolumes check for any *.json files in the PluginFolder/controller folder +// and loads then into rbdVolumes map +func loadExVolumes() { + rbdVol := rbdVolume{} + files, err := ioutil.ReadDir(path.Join(PluginFolder, "controller")) + if err != nil { + glog.Infof("rbd: failed to read controller's volumes folder: %s error:%v", path.Join(PluginFolder, "controller"), err) + return + } + for _, f := range files { + if !strings.HasSuffix(f.Name(), ".json") { + continue + } + fp, err := os.Open(path.Join(PluginFolder, "controller", f.Name())) + if err != nil { + glog.Infof("rbd: open file: %s err %%v", f.Name(), err) + continue + } + decoder := json.NewDecoder(fp) + if err = decoder.Decode(&rbdVol); err != nil { + glog.Infof("rbd: decode file: %s err: %v", f.Name(), err) + fp.Close() + continue + } + rbdVolumes[rbdVol.VolID] = rbdVol + } + glog.Infof("rbd: Loaded %d volumes from %s", len(rbdVolumes), path.Join(PluginFolder, "controller")) } func GetRBDDriver() *rbd { @@ -73,10 +124,10 @@ func NewNodeServer(d *csicommon.CSIDriver) *nodeServer { } func (rbd *rbd) Run(driverName, nodeID, endpoint string) { - glog.Infof("Driver: %v version: %v", driverName, GetVersionString(&version)) + glog.Infof("Driver: %v version: %v", driverName, version) // Initialize default library driver - rbd.driver = csicommon.NewCSIDriver(driverName, &version, GetSupportedVersions(), nodeID) + rbd.driver = csicommon.NewCSIDriver(driverName, version, nodeID) if rbd.driver == nil { glog.Fatalln("Failed to initialize CSI Driver.") } diff --git a/pkg/rbd/rbd_util.go b/pkg/rbd/rbd_util.go index 698572437..58a68916d 100644 --- a/pkg/rbd/rbd_util.go +++ b/pkg/rbd/rbd_util.go @@ -19,15 +19,16 @@ package rbd import ( "encoding/json" "fmt" - "github.com/golang/glog" "io/ioutil" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/kubernetes/pkg/util/keymutex" "os" "os/exec" "path" "strings" "time" + + "github.com/golang/glog" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/kubernetes/pkg/util/keymutex" ) const ( @@ -44,22 +45,29 @@ const ( rbdImageWatcherSteps = 10 ) -type rbdVolumeOptions struct { - VolName string `json:"volName"` - Monitors string `json:"monitors"` - Pool string `json:"pool"` - AdminID string `json:"adminID"` - AdminSecret string `json:"adminSecret"` - UserID string `json:"userID"` - UserSecret string `json:"userSecret"` - ImageFormat string `json:"imageFormat"` +type rbdVolume struct { + VolName string `json:"volName"` + VolID string `json:"volID"` + Monitors string `json:"monitors"` + Pool string `json:"pool"` + ImageFormat string `json:"imageFormat"` + // TODO (sbezverk) check if it is used and how ImageFeatures []string `json:"imageFeatures"` + VolSize int64 `json:"volSize"` } var attachdetachMutex = keymutex.NewKeyMutex() +func getRBDKey(id string, credentials map[string]string) (string, error) { + + if key, ok := credentials[id]; ok { + return key, nil + } + return "", fmt.Errorf("RBD key for ID: %s not found", id) +} + // CreateImage creates a new ceph image with provision and volume options. -func createRBDImage(pOpts *rbdVolumeOptions, volSz int) error { +func createRBDImage(pOpts *rbdVolume, volSz int, credentials map[string]string) error { var output []byte var err error @@ -68,12 +76,16 @@ func createRBDImage(pOpts *rbdVolumeOptions, volSz int) error { image := pOpts.VolName volSzGB := fmt.Sprintf("%dG", volSz) - if pOpts.ImageFormat == rbdImageFormat2 { - glog.V(4).Infof("rbd: create %s size %s format %s (features: %s) using mon %s, pool %s id %s key %s", image, volSzGB, pOpts.ImageFormat, pOpts.ImageFeatures, mon, pOpts.Pool, pOpts.AdminID, pOpts.AdminSecret) - } else { - glog.V(4).Infof("rbd: create %s size %s format %s using mon %s, pool %s id %s key %s", image, volSzGB, pOpts.ImageFormat, mon, pOpts.Pool, pOpts.AdminID, pOpts.AdminSecret) + key, err := getRBDKey(RBDUserID, credentials) + if err != nil { + return err } - args := []string{"create", image, "--size", volSzGB, "--pool", pOpts.Pool, "--id", pOpts.AdminID, "-m", mon, "--key=" + pOpts.AdminSecret, "--image-format", pOpts.ImageFormat} + if pOpts.ImageFormat == rbdImageFormat2 { + glog.V(4).Infof("rbd: create %s size %s format %s (features: %s) using mon %s, pool %s id %s key %s", image, volSzGB, pOpts.ImageFormat, pOpts.ImageFeatures, mon, pOpts.Pool, RBDUserID, key) + } else { + glog.V(4).Infof("rbd: create %s size %s format %s using mon %s, pool %s id %s key %s", image, volSzGB, pOpts.ImageFormat, mon, pOpts.Pool, RBDUserID, key) + } + args := []string{"create", image, "--size", volSzGB, "--pool", pOpts.Pool, "--id", RBDUserID, "-m", mon, "--key=" + key, "--image-format", pOpts.ImageFormat} if pOpts.ImageFormat == rbdImageFormat2 { // if no image features is provided, it results in empty string // which disable all RBD image format 2 features as we expected @@ -91,22 +103,20 @@ func createRBDImage(pOpts *rbdVolumeOptions, volSz int) error { // rbdStatus checks if there is watcher on the image. // It returns true if there is a watcher onthe image, otherwise returns false. -func rbdStatus(b *rbdVolumeOptions) (bool, string, error) { +func rbdStatus(pOpts *rbdVolume, credentials map[string]string) (bool, string, error) { var err error var output string var cmd []byte - image := b.VolName + image := pOpts.VolName // If we don't have admin id/secret (e.g. attaching), fallback to user id/secret. - id := b.AdminID - secret := b.AdminSecret - if id == "" { - id = b.UserID - secret = b.UserSecret + key, err := getRBDKey(RBDUserID, credentials) + if err != nil { + return false, "", err } - glog.V(4).Infof("rbd: status %s using mon %s, pool %s id %s key %s", image, b.Monitors, b.Pool, id, secret) - args := []string{"status", image, "--pool", b.Pool, "-m", b.Monitors, "--id", id, "--key=" + secret} + glog.V(4).Infof("rbd: status %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, RBDUserID, key) + args := []string{"status", image, "--pool", pOpts.Pool, "-m", pOpts.Monitors, "--id", RBDUserID, "--key=" + key} cmd, err = execCommand("rbd", args) output = string(cmd) @@ -133,10 +143,10 @@ func rbdStatus(b *rbdVolumeOptions) (bool, string, error) { } // DeleteImage deletes a ceph image with provision and volume options. -func deleteRBDImage(b *rbdVolumeOptions) error { +func deleteRBDImage(pOpts *rbdVolume, credentials map[string]string) error { var output []byte - image := b.VolName - found, _, err := rbdStatus(b) + image := pOpts.VolName + found, _, err := rbdStatus(pOpts, credentials) if err != nil { return err } @@ -144,15 +154,13 @@ func deleteRBDImage(b *rbdVolumeOptions) error { glog.Info("rbd is still being used ", image) return fmt.Errorf("rbd %s is still being used", image) } - id := b.AdminID - secret := b.AdminSecret - if id == "" { - id = b.UserID - secret = b.UserSecret + key, err := getRBDKey(RBDUserID, credentials) + if err != nil { + return err } - glog.V(4).Infof("rbd: rm %s using mon %s, pool %s id %s key %s", image, b.Monitors, b.Pool, id, secret) - args := []string{"rm", image, "--pool", b.Pool, "--id", id, "-m", b.Monitors, "--key=" + secret} + glog.V(4).Infof("rbd: rm %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, RBDUserID, key) + args := []string{"rm", image, "--pool", pOpts.Pool, "--id", RBDUserID, "-m", pOpts.Monitors, "--key=" + key} output, err = execCommand("rbd", args) if err == nil { return nil @@ -166,42 +174,26 @@ func execCommand(command string, args []string) ([]byte, error) { return cmd.CombinedOutput() } -func getRBDVolumeOptions(volOptions map[string]string) (*rbdVolumeOptions, error) { - rbdVolume := &rbdVolumeOptions{} +func getRBDVolumeOptions(volOptions map[string]string) (*rbdVolume, error) { var ok bool - rbdVolume.AdminID, ok = volOptions["adminID"] - if !ok { - return nil, fmt.Errorf("Missing required parameter adminID") - } - rbdVolume.AdminSecret, ok = volOptions["adminSecret"] - if !ok { - return nil, fmt.Errorf("Missing required parameter adminSecret") - } - rbdVolume.Pool, ok = volOptions["pool"] + rbdVol := &rbdVolume{} + rbdVol.Pool, ok = volOptions["pool"] if !ok { return nil, fmt.Errorf("Missing required parameter pool") } - rbdVolume.Monitors, ok = volOptions["monitors"] + rbdVol.Monitors, ok = volOptions["monitors"] if !ok { return nil, fmt.Errorf("Missing required parameter monitors") } - rbdVolume.UserID, ok = volOptions["userID"] + rbdVol.ImageFormat, ok = volOptions["imageFormat"] if !ok { - return nil, fmt.Errorf("Missing required parameter userID") - } - rbdVolume.UserSecret, ok = volOptions["userSecret"] - if !ok { - return nil, fmt.Errorf("Missing required parameter userSecret") - } - rbdVolume.ImageFormat, ok = volOptions["imageFormat"] - if !ok { - rbdVolume.ImageFormat = "2" + rbdVol.ImageFormat = "2" } - return rbdVolume, nil + return rbdVol, nil } -func attachRBDImage(volOptions *rbdVolumeOptions) (string, error) { +func attachRBDImage(volOptions *rbdVolume, credentials map[string]string) (string, error) { var err error var output []byte @@ -222,7 +214,7 @@ func attachRBDImage(volOptions *rbdVolumeOptions) (string, error) { Steps: rbdImageWatcherSteps, } err := wait.ExponentialBackoff(backoff, func() (bool, error) { - used, rbdOutput, err := rbdStatus(volOptions) + used, rbdOutput, err := rbdStatus(volOptions, credentials) if err != nil { return false, fmt.Errorf("fail to check rbd image status with: (%v), rbd output: (%s)", err, rbdOutput) } @@ -238,11 +230,13 @@ func attachRBDImage(volOptions *rbdVolumeOptions) (string, error) { } glog.V(1).Infof("rbd: map mon %s", volOptions.Monitors) - id := volOptions.UserID - secret := volOptions.UserSecret + key, err := getRBDKey(RBDUserID, credentials) + if err != nil { + return "", err + } output, err = execCommand("rbd", []string{ - "map", image, "--pool", volOptions.Pool, "--id", id, "-m", volOptions.Monitors, "--key=" + secret}) + "map", image, "--pool", volOptions.Pool, "--id", RBDUserID, "-m", volOptions.Monitors, "--key=" + key}) if err != nil { glog.V(1).Infof("rbd: map error %v, rbd output: %s", err, string(output)) return "", fmt.Errorf("rbd: map failed %v, rbd output: %s", err, string(output)) @@ -322,23 +316,24 @@ func waitForPath(pool, image string, maxRetries int) (string, bool) { return "", false } -func persistVolInfo(image string, persistentStoragePath string, volInfo *rbdVolumeOptions) error { +func persistVolInfo(image string, persistentStoragePath string, volInfo *rbdVolume) error { file := path.Join(persistentStoragePath, image+".json") fp, err := os.Create(file) if err != nil { + glog.Errorf("rbd: failed to create persistent storage file %s with error: %v\n", file, err) return fmt.Errorf("rbd: create err %s/%s", file, err) } defer fp.Close() - encoder := json.NewEncoder(fp) if err = encoder.Encode(volInfo); err != nil { - return fmt.Errorf("rbd: encode err: %v.", err) + glog.Errorf("rbd: failed to encode volInfo: %+v for file: %s with error: %v\n", volInfo, file, err) + return fmt.Errorf("rbd: encode err: %v", err) } - + glog.Infof("rbd: successfully saved volInfo: %+v into file: %s\n", volInfo, file) return nil } -func loadVolInfo(image string, persistentStoragePath string, volInfo *rbdVolumeOptions) error { +func loadVolInfo(image string, persistentStoragePath string, volInfo *rbdVolume) error { file := path.Join(persistentStoragePath, image+".json") fp, err := os.Open(file) if err != nil { @@ -356,11 +351,28 @@ func loadVolInfo(image string, persistentStoragePath string, volInfo *rbdVolumeO func deleteVolInfo(image string, persistentStoragePath string) error { file := path.Join(persistentStoragePath, image+".json") + glog.Infof("rbd: Deleting file for Volume: %s at: %s resulting path: %+v\n", image, persistentStoragePath, file) err := os.Remove(file) if err != nil { if err != os.ErrNotExist { - return fmt.Errorf("rbd: open err %s/%s", file, err) + return fmt.Errorf("rbd: error removing file: %s/%s", file, err) } } return nil } + +func getRBDVolumeByID(volumeID string) (rbdVolume, error) { + if rbdVol, ok := rbdVolumes[volumeID]; ok { + return rbdVol, nil + } + return rbdVolume{}, fmt.Errorf("volume id %s does not exit in the volumes list", volumeID) +} + +func getRBDVolumeByName(volName string) (rbdVolume, error) { + for _, rbdVol := range rbdVolumes { + if rbdVol.VolName == volName { + return rbdVol, nil + } + } + return rbdVolume{}, fmt.Errorf("volume name %s does not exit in the volumes list", volName) +} diff --git a/rbd/main.go b/rbd/main.go index 6a2a12a40..6a2adcca0 100644 --- a/rbd/main.go +++ b/rbd/main.go @@ -31,7 +31,7 @@ func init() { var ( endpoint = flag.String("endpoint", "unix://tmp/csi.sock", "CSI endpoint") - driverName = flag.String("drivername", "rbdplugin", "name of the driver") + driverName = flag.String("drivername", "csi-rbdplugin", "name of the driver") nodeID = flag.String("nodeid", "", "node id") )