diff --git a/Gopkg.lock b/Gopkg.lock new file mode 100644 index 000000000..88fd48088 --- /dev/null +++ b/Gopkg.lock @@ -0,0 +1,231 @@ +# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. + + +[[projects]] + name = "github.com/PuerkitoBio/purell" + packages = ["."] + revision = "0bcb03f4b4d0a9428594752bd2a3b9aa0a9d4bd4" + version = "v1.1.0" + +[[projects]] + branch = "master" + name = "github.com/PuerkitoBio/urlesc" + packages = ["."] + revision = "de5bf2ad457846296e2031421a34e2568e304e35" + +[[projects]] + name = "github.com/container-storage-interface/spec" + packages = ["lib/go/csi"] + revision = "9e88e4bfabeca1b8e4810555815f112159292ada" + version = "v0.1.0" + +[[projects]] + name = "github.com/emicklei/go-restful" + packages = [".","log"] + revision = "5741799b275a3c4a5a9623a993576d7545cf7b5c" + version = "v2.4.0" + +[[projects]] + name = "github.com/ghodss/yaml" + packages = ["."] + revision = "0ca9ea5df5451ffdf184b4428c902747c2c11cd7" + version = "v1.0.0" + +[[projects]] + branch = "master" + name = "github.com/go-openapi/jsonpointer" + packages = ["."] + revision = "779f45308c19820f1a69e9a4cd965f496e0da10f" + +[[projects]] + branch = "master" + name = "github.com/go-openapi/jsonreference" + packages = ["."] + revision = "36d33bfe519efae5632669801b180bf1a245da3b" + +[[projects]] + branch = "master" + name = "github.com/go-openapi/spec" + packages = ["."] + revision = "fa03337d7da5735229ee8f5e9d5d0b996014b7f8" + +[[projects]] + branch = "master" + name = "github.com/go-openapi/swag" + packages = ["."] + revision = "84f4bee7c0a6db40e3166044c7983c1c32125429" + +[[projects]] + name = "github.com/gogo/protobuf" + packages = ["proto","sortkeys"] + revision = "342cbe0a04158f6dcb03ca0079991a51a4248c02" + version = "v0.5" + +[[projects]] + branch = "master" + name = "github.com/golang/glog" + packages = ["."] + revision = "23def4e6c14b4da8ac2ed8007337bc5eb5007998" + +[[projects]] + branch = "master" + name = "github.com/golang/protobuf" + packages = ["proto","ptypes","ptypes/any","ptypes/duration","ptypes/timestamp"] + revision = "1e59b77b52bf8e4b449a57e6f79f21226d571845" + +[[projects]] + branch = "master" + name = "github.com/google/btree" + packages = ["."] + revision = "316fb6d3f031ae8f4d457c6c5186b9e3ded70435" + +[[projects]] + branch = "master" + name = "github.com/google/gofuzz" + packages = ["."] + revision = "24818f796faf91cd76ec7bddd72458fbced7a6c1" + +[[projects]] + name = "github.com/googleapis/gnostic" + packages = ["OpenAPIv2","compiler","extensions"] + revision = "ee43cbb60db7bd22502942cccbc39059117352ab" + version = "v0.1.0" + +[[projects]] + branch = "master" + name = "github.com/gregjones/httpcache" + packages = [".","diskcache"] + revision = "2bcd89a1743fd4b373f7370ce8ddc14dfbd18229" + +[[projects]] + name = "github.com/json-iterator/go" + packages = ["."] + revision = "f7279a603edee96fe7764d3de9c6ff8cf9970994" + version = "1.0.4" + +[[projects]] + branch = "master" + name = "github.com/juju/ratelimit" + packages = ["."] + revision = "59fac5042749a5afb9af70e813da1dd5474f0167" + +[[projects]] + branch = "master" + name = "github.com/kubernetes-csi/drivers" + packages = ["pkg/csi-common"] + revision = "822ddbb41799f02d17e9662d0e34530f7e8061dd" + +[[projects]] + branch = "master" + name = "github.com/mailru/easyjson" + packages = ["buffer","jlexer","jwriter"] + revision = "32fa128f234d041f196a9f3e0fea5ac9772c08e1" + +[[projects]] + name = "github.com/pborman/uuid" + packages = ["."] + revision = "e790cca94e6cc75c7064b1332e63811d4aae1a53" + version = "v1.1" + +[[projects]] + branch = "master" + name = "github.com/petar/GoLLRB" + packages = ["llrb"] + revision = "53be0d36a84c2a886ca057d34b6aa4468df9ccb4" + +[[projects]] + name = "github.com/peterbourgon/diskv" + packages = ["."] + revision = "5f041e8faa004a95c88a202771f4cc3e991971e6" + version = "v2.0.1" + +[[projects]] + name = "github.com/spf13/pflag" + packages = ["."] + revision = "e57e3eeb33f795204c1ca35f56c44f83227c6e66" + version = "v1.0.0" + +[[projects]] + branch = "master" + name = "golang.org/x/net" + packages = ["context","http2","http2/hpack","idna","internal/timeseries","lex/httplex","trace"] + revision = "42fe2e1c20de1054d3d30f82cc9fb5b41e2e3767" + +[[projects]] + branch = "master" + name = "golang.org/x/sys" + packages = ["unix"] + revision = "1792d66dc88e503d3cb2400578221cdf1f7fe26f" + +[[projects]] + branch = "master" + name = "golang.org/x/text" + packages = ["collate","collate/build","internal/colltab","internal/gen","internal/tag","internal/triegen","internal/ucd","language","secure/bidirule","transform","unicode/bidi","unicode/cldr","unicode/norm","unicode/rangetable","width"] + revision = "e19ae1496984b1c655b8044a65c0300a3c878dd3" + +[[projects]] + branch = "master" + name = "google.golang.org/genproto" + packages = ["googleapis/rpc/status"] + revision = "a8101f21cf983e773d0c1133ebc5424792003214" + +[[projects]] + name = "google.golang.org/grpc" + packages = [".","balancer","balancer/base","balancer/roundrobin","codes","connectivity","credentials","encoding","grpclb/grpc_lb_v1/messages","grpclog","internal","keepalive","metadata","naming","peer","resolver","resolver/dns","resolver/passthrough","stats","status","tap","transport"] + revision = "f3955b8e9e244dd4dd4bc4f7b7a23a8445400a76" + version = "v1.9.0" + +[[projects]] + name = "gopkg.in/inf.v0" + packages = ["."] + revision = "3887ee99ecf07df5b447e9b00d9c0b2adaa9f3e4" + version = "v0.9.0" + +[[projects]] + branch = "v2" + name = "gopkg.in/yaml.v2" + packages = ["."] + revision = "d670f9405373e636a5a2765eea47fac0c9bc91a4" + +[[projects]] + branch = "master" + name = "k8s.io/api" + packages = ["admissionregistration/v1alpha1","admissionregistration/v1beta1","apps/v1","apps/v1beta1","apps/v1beta2","authentication/v1","authentication/v1beta1","authorization/v1","authorization/v1beta1","autoscaling/v1","autoscaling/v2beta1","batch/v1","batch/v1beta1","batch/v2alpha1","certificates/v1beta1","core/v1","events/v1beta1","extensions/v1beta1","networking/v1","policy/v1beta1","rbac/v1","rbac/v1alpha1","rbac/v1beta1","scheduling/v1alpha1","settings/v1alpha1","storage/v1","storage/v1alpha1","storage/v1beta1"] + revision = "57d7f151236665c12202a51c21bc939eb5d5ba91" + +[[projects]] + branch = "release-1.9" + name = "k8s.io/apimachinery" + packages = ["pkg/api/errors","pkg/api/meta","pkg/api/resource","pkg/apis/meta/v1","pkg/apis/meta/v1/unstructured","pkg/apis/meta/v1alpha1","pkg/conversion","pkg/conversion/queryparams","pkg/fields","pkg/labels","pkg/runtime","pkg/runtime/schema","pkg/runtime/serializer","pkg/runtime/serializer/json","pkg/runtime/serializer/protobuf","pkg/runtime/serializer/recognizer","pkg/runtime/serializer/streaming","pkg/runtime/serializer/versioning","pkg/selection","pkg/types","pkg/util/clock","pkg/util/errors","pkg/util/framer","pkg/util/intstr","pkg/util/json","pkg/util/net","pkg/util/runtime","pkg/util/sets","pkg/util/validation","pkg/util/validation/field","pkg/util/wait","pkg/util/yaml","pkg/version","pkg/watch","third_party/forked/golang/reflect"] + revision = "68f9c3a1feb3140df59c67ced62d3a5df8e6c9c2" + +[[projects]] + name = "k8s.io/client-go" + packages = ["discovery","kubernetes","kubernetes/scheme","kubernetes/typed/admissionregistration/v1alpha1","kubernetes/typed/admissionregistration/v1beta1","kubernetes/typed/apps/v1","kubernetes/typed/apps/v1beta1","kubernetes/typed/apps/v1beta2","kubernetes/typed/authentication/v1","kubernetes/typed/authentication/v1beta1","kubernetes/typed/authorization/v1","kubernetes/typed/authorization/v1beta1","kubernetes/typed/autoscaling/v1","kubernetes/typed/autoscaling/v2beta1","kubernetes/typed/batch/v1","kubernetes/typed/batch/v1beta1","kubernetes/typed/batch/v2alpha1","kubernetes/typed/certificates/v1beta1","kubernetes/typed/core/v1","kubernetes/typed/events/v1beta1","kubernetes/typed/extensions/v1beta1","kubernetes/typed/networking/v1","kubernetes/typed/policy/v1beta1","kubernetes/typed/rbac/v1","kubernetes/typed/rbac/v1alpha1","kubernetes/typed/rbac/v1beta1","kubernetes/typed/scheduling/v1alpha1","kubernetes/typed/settings/v1alpha1","kubernetes/typed/storage/v1","kubernetes/typed/storage/v1alpha1","kubernetes/typed/storage/v1beta1","pkg/version","rest","rest/watch","tools/clientcmd/api","tools/metrics","tools/reference","transport","util/cert","util/flowcontrol","util/integer"] + revision = "78700dec6369ba22221b72770783300f143df150" + version = "v6.0.0" + +[[projects]] + branch = "master" + name = "k8s.io/kube-openapi" + packages = ["pkg/common"] + revision = "a07b7bbb58e7fdc5144f8d7046331d29fc9ad3b3" + +[[projects]] + name = "k8s.io/kubernetes" + packages = ["pkg/util/io","pkg/util/keymutex","pkg/util/mount","pkg/util/nsenter"] + revision = "3a1c9449a956b6026f075fa3134ff92f7d55f812" + version = "v1.9.1" + +[[projects]] + branch = "master" + name = "k8s.io/utils" + packages = ["exec"] + revision = "a99a3e11a96751670db62ba77c6d278d1136931e" + +[solve-meta] + analyzer-name = "dep" + analyzer-version = 1 + inputs-digest = "b1c8bd120bec9cbdabfe8c0971602ed9a8dc3da6bde54c8f27805bb79be80b58" + solver-name = "gps-cdcl" + solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml new file mode 100644 index 000000000..d8c3660c2 --- /dev/null +++ b/Gopkg.toml @@ -0,0 +1,23 @@ +[[constraint]] + version = "v0.1" + name = "github.com/container-storage-interface/spec" + +[[constraint]] + branch = "master" + name = "github.com/golang/glog" + +[[constraint]] + name = "google.golang.org/grpc" + version = "1.7.2" + +[[constraint]] + name = "github.com/docker/distribution" + revision = "edc3ab29cdff8694dd6feb85cfeb4b5f1b38ed9c" + +[[constraint]] + name = "k8s.io/kubernetes" + version = "v1.9.1" + +[[constraint]] + name = "k8s.io/apimachinery" + version = "kubernetes-1.9.1" diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..a06568f7a --- /dev/null +++ b/Makefile @@ -0,0 +1,36 @@ +# Copyright 2018 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +.PHONY: all rbdplugin + +IMAGE_NAME=csi_images/rbdplugin +IMAGE_VERSION=latest + +all: rbdplugin + +test: + go test github.com/ceph/ceph-csi/pkg/... -cover + go vet github.com/ceph/ceph-csi/pkg/... + +rbdplugin: + if [ ! -d ./vendor ]; then dep ensure; fi + go build -i -o _output/rbdplugin ./rbd + +container: rbdplugin + cp _output/rbdplugin deploy/docker + docker build -t $(IMAGE_NAME):$(IMAGE_VERSION) deploy/docker + +clean: + go clean -r -x + -rm -rf _output diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile new file mode 100644 index 000000000..8a35757e4 --- /dev/null +++ b/deploy/docker/Dockerfile @@ -0,0 +1,12 @@ +FROM centos:7 +LABEL maintainers="Kubernetes Authors" +LABEL description="RBD CSI Plugin" + +ENV CEPH_VERSION "luminous" +RUN yum install -y centos-release-ceph && \ + yum install -y ceph-common e2fsprogs && \ + yum clean all + +COPY rbdplugin /rbdplugin +RUN chmod +x /rbdplugin +ENTRYPOINT ["/rbdplugin"] diff --git a/deploy/docker/rbdplugin b/deploy/docker/rbdplugin new file mode 100755 index 000000000..86d3a4437 Binary files /dev/null and b/deploy/docker/rbdplugin differ diff --git a/deploy/kubernetes/rbdplugin.yaml b/deploy/kubernetes/rbdplugin.yaml new file mode 100644 index 000000000..2ca814340 --- /dev/null +++ b/deploy/kubernetes/rbdplugin.yaml @@ -0,0 +1,129 @@ +# This YAML defines all API objects to create RBAC roles for csi node plugin. + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-nodeplugin + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-nodeplugin +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-nodeplugin +subjects: + - kind: ServiceAccount + name: csi-nodeplugin + namespace: default +roleRef: + kind: ClusterRole + name: csi-nodeplugin + apiGroup: rbac.authorization.k8s.io + +--- +# This YAML file contains driver-registrar & csi driver nodeplugin API objects, +# which are necessary to run csi nodeplugin for rbd. + +kind: DaemonSet +apiVersion: apps/v1beta2 +metadata: + name: csi-nodeplugin-rbdplugin +spec: + selector: + matchLabels: + app: csi-nodeplugin-rbdplugin + template: + metadata: + labels: + app: csi-nodeplugin-rbdplugin + spec: + serviceAccount: csi-nodeplugin + hostNetwork: true + containers: + - name: driver-registrar + image: csi_images/driver-registrar:latest + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + env: + - name: ADDRESS + value: /var/lib/kubelet/plugins/rbdplugin/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: socket-dir + mountPath: /var/lib/kubelet/plugins/rbdplugin + - name: rbdplugin + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + image: csi_images/rbdplugin:latest + args : + - "--nodeid=$(NODE_ID)" + - "--endpoint=$(CSI_ENDPOINT)" + - "--v=5" + - "--drivername=rbdplugin" + env: + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: unix://var/lib/kubelet/plugins/rbdplugin/csi.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: plugin-dir + mountPath: /var/lib/kubelet/plugins/rbdplugin + - name: pods-mount-dir + mountPath: /var/lib/kubelet/pods + mountPropagation: "Bidirectional" + - mountPath: /dev + name: host-dev + - mountPath: /sys + name: host-sys + - mountPath: /lib/modules + name: lib-modules + readOnly: true + volumes: + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/rbdplugin + type: DirectoryOrCreate + - name: pods-mount-dir + hostPath: + path: /var/lib/kubelet/pods + type: Directory + - name: socket-dir + hostPath: + path: /var/lib/kubelet/plugins/rbdplugin + type: DirectoryOrCreate + - name: host-dev + hostPath: + path: /dev + - name: host-sys + hostPath: + path: /sys + - name: lib-modules + hostPath: + path: /lib/modules diff --git a/pkg/rbd/controllerserver.go b/pkg/rbd/controllerserver.go new file mode 100644 index 000000000..d4a50c049 --- /dev/null +++ b/pkg/rbd/controllerserver.go @@ -0,0 +1,152 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rbd + +import ( + "fmt" + "path" + + "github.com/golang/glog" + "github.com/pborman/uuid" + "golang.org/x/net/context" + + "k8s.io/client-go/kubernetes" + + "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/kubernetes-csi/drivers/pkg/csi-common" +) + +const ( + oneGB = 1073741824 +) + +type controllerServer struct { + *csicommon.DefaultControllerServer + clientSet *kubernetes.Clientset +} + +func GetVersionString(ver *csi.Version) string { + return fmt.Sprintf("%d.%d.%d", ver.Major, ver.Minor, ver.Patch) +} + +func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) { + if err := cs.Driver.ValidateControllerServiceRequest(req.Version, csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME); err != nil { + glog.V(3).Infof("invalid create volume req: %v", req) + return nil, err + } + // Volume Name + volName := req.GetName() + if len(volName) == 0 { + volName = uuid.NewUUID().String() + } + + // Volume Size - Default is 1 GiB + volSizeBytes := int64(oneGB) + if req.GetCapacityRange() != nil { + volSizeBytes = int64(req.GetCapacityRange().GetRequiredBytes()) + } + volSizeGB := int(volSizeBytes / 1024 / 1024 / 1024) + + volOptions, err := getRBDVolumeOptions(req.Parameters, cs.clientSet) + if err != nil { + return nil, err + } + // Check if there is already RBD image with requested name + found, _, _ := rbdStatus(volName, volOptions) + if !found { + if err := createRBDImage(volName, volSizeGB, volOptions); err != nil { + if err != nil { + glog.Warningf("failed to create volume: %v", err) + return nil, err + } + } + glog.V(4).Infof("create volume %s", volName) + } + // Storing volInfo into a persistent file, will need info to delete rbd image + // in ControllerUnpublishVolume + if err := persistVolInfo(volName, path.Join(PluginFolder, "controller"), volOptions); err != nil { + glog.Warningf("rbd: failed to store volInfo with error: %v", err) + } + + return &csi.CreateVolumeResponse{ + VolumeInfo: &csi.VolumeInfo{ + Id: volName, + CapacityBytes: uint64(volSizeBytes), + Attributes: req.GetParameters(), + }, + }, nil +} + +func (cs *controllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) { + if err := cs.Driver.ValidateControllerServiceRequest(req.Version, csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME); err != nil { + glog.Warningf("invalid delete volume req: %v", req) + return nil, err + } + + return &csi.DeleteVolumeResponse{}, nil +} + +func (cs *controllerServer) ValidateVolumeCapabilities(ctx context.Context, req *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error) { + for _, cap := range req.VolumeCapabilities { + if cap.GetAccessMode().GetMode() != csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER { + return &csi.ValidateVolumeCapabilitiesResponse{Supported: false, Message: ""}, nil + } + } + return &csi.ValidateVolumeCapabilitiesResponse{Supported: true, Message: ""}, nil +} + +func (cs *controllerServer) ControllerUnpublishVolume(ctx context.Context, req *csi.ControllerUnpublishVolumeRequest) (*csi.ControllerUnpublishVolumeResponse, error) { + + // For now the image get unconditionally deleted, but here retention policy can be checked + volName := req.GetVolumeId() + volOptions := &rbdVolumeOptions{} + if err := loadVolInfo(volName, path.Join(PluginFolder, "controller"), volOptions); err != nil { + return nil, err + } + + // Recover rbd secret key value, for now by k8s specific call + id := volOptions.AdminID + secretName := volOptions.AdminSecretName + secretNamespace := volOptions.AdminSecretNamespace + if id == "" { + secretName = volOptions.UserSecretName + secretNamespace = volOptions.UserSecretNamespace + } + if key, err := parseStorageClassSecret(secretName, secretNamespace, cs.clientSet); err != nil { + return nil, err + } else { + volOptions.adminSecret = key + } + + // Deleting rbd image + glog.V(4).Infof("deleting volume %s", volName) + if err := deleteRBDImage(volName, volOptions); err != nil { + glog.V(3).Infof("failed to delete rbd image: %s/%s with error: %v", volOptions.Pool, volName, err) + return nil, err + } + // Removing persistent storage file for the unmapped volume + if err := deleteVolInfo(volName, path.Join(PluginFolder, "controller")); err != nil { + return nil, err + } + + return &csi.ControllerUnpublishVolumeResponse{}, nil +} + +func (cs *controllerServer) ControllerPublishVolume(ctx context.Context, req *csi.ControllerPublishVolumeRequest) (*csi.ControllerPublishVolumeResponse, error) { + + return &csi.ControllerPublishVolumeResponse{}, nil +} diff --git a/pkg/rbd/identityserver.go b/pkg/rbd/identityserver.go new file mode 100644 index 000000000..d410d8548 --- /dev/null +++ b/pkg/rbd/identityserver.go @@ -0,0 +1,25 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rbd + +import ( + "github.com/kubernetes-csi/drivers/pkg/csi-common" +) + +type identityServer struct { + *csicommon.DefaultIdentityServer +} diff --git a/pkg/rbd/nodeserver.go b/pkg/rbd/nodeserver.go new file mode 100644 index 000000000..f561b464d --- /dev/null +++ b/pkg/rbd/nodeserver.go @@ -0,0 +1,144 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rbd + +import ( + "os" + "path" + + "github.com/golang/glog" + "golang.org/x/net/context" + + "github.com/container-storage-interface/spec/lib/go/csi" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/pkg/util/mount" + + "github.com/kubernetes-csi/drivers/pkg/csi-common" +) + +type nodeServer struct { + *csicommon.DefaultNodeServer + clientSet *kubernetes.Clientset +} + +func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) { + targetPath := req.GetTargetPath() + + notMnt, err := mount.New("").IsLikelyNotMountPoint(targetPath) + if err != nil { + if os.IsNotExist(err) { + if err = os.MkdirAll(targetPath, 0750); err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + notMnt = true + } else { + return nil, status.Error(codes.Internal, err.Error()) + } + } + + if !notMnt { + return &csi.NodePublishVolumeResponse{}, nil + } + volOptions, err := getRBDVolumeOptions(req.VolumeAttributes, ns.clientSet) + if err != nil { + return nil, err + } + + // Mapping RBD image + devicePath, err := attachRBDImage(req.GetVolumeId(), volOptions) + if err != nil { + return nil, err + } + glog.V(4).Infof("rbd image: %s/%s was succesfully mapped at %s\n", req.GetVolumeId(), volOptions.Pool, devicePath) + fsType := req.GetVolumeCapability().GetMount().GetFsType() + + readOnly := req.GetReadonly() + attrib := req.GetVolumeAttributes() + mountFlags := req.GetVolumeCapability().GetMount().GetMountFlags() + + glog.V(4).Infof("target %v\nfstype %v\ndevice %v\nreadonly %v\nattributes %v\n mountflags %v\n", + targetPath, fsType, devicePath, readOnly, attrib, mountFlags) + + options := []string{} + if readOnly { + options = append(options, "ro") + } + + diskMounter := &mount.SafeFormatAndMount{Interface: mount.New(""), Exec: mount.NewOsExec()} + if err := diskMounter.FormatAndMount(devicePath, targetPath, fsType, options); err != nil { + return nil, err + } + // Storing rbd device path + + volOptions.ImageMapping = map[string]string{req.GetVolumeId(): devicePath} + // Storing volInfo into a persistent file + if err := persistVolInfo(req.GetVolumeId(), path.Join(PluginFolder, "node"), volOptions); err != nil { + glog.Warningf("rbd: failed to store volInfo with error: %v", err) + } + + return &csi.NodePublishVolumeResponse{}, nil +} + +func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) { + targetPath := req.GetTargetPath() + volName := req.GetVolumeId() + volOptions := &rbdVolumeOptions{} + if err := loadVolInfo(volName, path.Join(PluginFolder, "node"), volOptions); err != nil { + return nil, err + } + + // Recover rbd secret key value, for now by k8s specific call + id := volOptions.AdminID + secretName := volOptions.AdminSecretName + secretNamespace := volOptions.AdminSecretNamespace + if id == "" { + secretName = volOptions.UserSecretName + secretNamespace = volOptions.UserSecretNamespace + } + if key, err := parseStorageClassSecret(secretName, secretNamespace, ns.clientSet); err != nil { + return nil, err + } else { + volOptions.adminSecret = key + } + + notMnt, err := mount.New("").IsLikelyNotMountPoint(targetPath) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + if notMnt { + return nil, status.Error(codes.NotFound, "Volume not mounted") + } + // Unmounting the image + err = mount.New("").Unmount(req.GetTargetPath()) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + // Unmapping rbd device + glog.V(4).Infof("deleting volume %s", volName) + if err := detachRBDImage(volName, volOptions); err != nil { + glog.V(3).Infof("failed to unmap rbd device: %s with error: %v", volOptions.ImageMapping[volName], err) + return nil, err + } + // Removing persistent storage file for the unmapped volume + if err := deleteVolInfo(volName, path.Join(PluginFolder, "node")); err != nil { + return nil, err + } + + return &csi.NodeUnpublishVolumeResponse{}, nil +} diff --git a/pkg/rbd/rbd.go b/pkg/rbd/rbd.go new file mode 100644 index 000000000..2273cdedc --- /dev/null +++ b/pkg/rbd/rbd.go @@ -0,0 +1,99 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rbd + +import ( + "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/golang/glog" + + "github.com/kubernetes-csi/drivers/pkg/csi-common" + "k8s.io/client-go/kubernetes" +) + +// PluginFolder defines the location of rbdplugin +const ( + PluginFolder = "/var/lib/kubelet/plugins/rbdplugin" +) + +type rbd struct { + driver *csicommon.CSIDriver + + ids *identityServer + ns *nodeServer + cs *controllerServer + + cap []*csi.VolumeCapability_AccessMode + cscap []*csi.ControllerServiceCapability +} + +var ( + rbdDriver *rbd + version = csi.Version{ + Minor: 1, + } +) + +func GetSupportedVersions() []*csi.Version { + return []*csi.Version{&version} +} + +func GetRBDDriver() *rbd { + return &rbd{} +} + +func NewIdentityServer(d *csicommon.CSIDriver) *identityServer { + return &identityServer{ + DefaultIdentityServer: csicommon.NewDefaultIdentityServer(d), + } +} + +func NewControllerServer(d *csicommon.CSIDriver, clientSet *kubernetes.Clientset) *controllerServer { + return &controllerServer{ + DefaultControllerServer: csicommon.NewDefaultControllerServer(d), + clientSet: clientSet, + } +} + +func NewNodeServer(d *csicommon.CSIDriver, clientSet *kubernetes.Clientset) *nodeServer { + return &nodeServer{ + DefaultNodeServer: csicommon.NewDefaultNodeServer(d), + clientSet: clientSet, + } +} + +func (rbd *rbd) Run(driverName, nodeID, endpoint string, clientSet *kubernetes.Clientset) { + glog.Infof("Driver: %v version: %v", driverName, GetVersionString(&version)) + + // Initialize default library driver + rbd.driver = csicommon.NewCSIDriver(driverName, &version, GetSupportedVersions(), nodeID) + if rbd.driver == nil { + glog.Fatalln("Failed to initialize CSI Driver.") + } + rbd.driver.AddControllerServiceCapabilities([]csi.ControllerServiceCapability_RPC_Type{ + csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME, + csi.ControllerServiceCapability_RPC_PUBLISH_UNPUBLISH_VOLUME, + }) + rbd.driver.AddVolumeCapabilityAccessModes([]csi.VolumeCapability_AccessMode_Mode{csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER}) + + // Create GRPC servers + rbd.ids = NewIdentityServer(rbd.driver) + rbd.ns = NewNodeServer(rbd.driver, clientSet) + rbd.cs = NewControllerServer(rbd.driver, clientSet) + s := csicommon.NewNonBlockingGRPCServer() + s.Start(endpoint, rbd.ids, rbd.cs, rbd.ns) + s.Wait() +} diff --git a/pkg/rbd/rbd_util.go b/pkg/rbd/rbd_util.go new file mode 100644 index 000000000..55ccd5b5f --- /dev/null +++ b/pkg/rbd/rbd_util.go @@ -0,0 +1,420 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rbd + +import ( + "encoding/json" + "fmt" + "github.com/golang/glog" + "io/ioutil" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/pkg/util/keymutex" + "os" + "os/exec" + "path" + "strings" + "time" +) + +const ( + imageWatcherStr = "watcher=" + rbdImageFormat1 = "1" + rbdImageFormat2 = "2" + imageSizeStr = "size " + sizeDivStr = " MB in" + kubeLockMagic = "kubelet_lock_magic_" + // The following three values are used for 30 seconds timeout + // while waiting for RBD Watcher to expire. + rbdImageWatcherInitDelay = 1 * time.Second + rbdImageWatcherFactor = 1.4 + rbdImageWatcherSteps = 10 +) + +type rbdVolumeOptions struct { + Monitors string `json:"monitors"` + Pool string `json:"pool"` + AdminSecretName string `json:"adminSecret"` + AdminSecretNamespace string `json:"adminSecretNamespace"` + AdminID string `json:"adminID"` + UserID string `json:"userID"` + UserSecretName string `json:"userSecret"` + UserSecretNamespace string `json:"userSecretNamespace"` + ImageFormat string `json:"imageFormat"` + ImageFeatures []string `json:"imageFeatures"` + ImageMapping map[string]string `json:"imageMapping"` + adminSecret string + userSecret string +} + +var attachdetachMutex = keymutex.NewKeyMutex() + +// CreateImage creates a new ceph image with provision and volume options. +func createRBDImage(image string, volSz int, pOpts *rbdVolumeOptions) error { + var output []byte + var err error + + // rbd create + mon := pOpts.Monitors + volSzGB := fmt.Sprintf("%dG", volSz) + + if pOpts.ImageFormat == rbdImageFormat2 { + glog.V(4).Infof("rbd: create %s size %s format %s (features: %s) using mon %s, pool %s id %s key %s", image, volSzGB, pOpts.ImageFormat, pOpts.ImageFeatures, mon, pOpts.Pool, pOpts.AdminID, pOpts.adminSecret) + } else { + glog.V(4).Infof("rbd: create %s size %s format %s using mon %s, pool %s id %s key %s", image, volSzGB, pOpts.ImageFormat, mon, pOpts.Pool, pOpts.AdminID, pOpts.adminSecret) + } + args := []string{"create", image, "--size", volSzGB, "--pool", pOpts.Pool, "--id", pOpts.AdminID, "-m", mon, "--key=" + pOpts.adminSecret, "--image-format", pOpts.ImageFormat} + if pOpts.ImageFormat == rbdImageFormat2 { + // if no image features is provided, it results in empty string + // which disable all RBD image format 2 features as we expected + features := strings.Join(pOpts.ImageFeatures, ",") + args = append(args, "--image-feature", features) + } + output, err = execCommand("rbd", args) + + if err != nil { + return fmt.Errorf("failed to create rbd image: %v, command output: %s", err, string(output)) + } + + return nil +} + +// rbdStatus checks if there is watcher on the image. +// It returns true if there is a watcher onthe image, otherwise returns false. +func rbdStatus(image string, b *rbdVolumeOptions) (bool, string, error) { + var err error + var output string + var cmd []byte + + // If we don't have admin id/secret (e.g. attaching), fallback to user id/secret. + id := b.AdminID + secret := b.adminSecret + if id == "" { + id = b.UserID + secret = b.userSecret + } + + glog.V(4).Infof("rbd: status %s using mon %s, pool %s id %s key %s", image, b.Monitors, b.Pool, id, secret) + args := []string{"status", image, "--pool", b.Pool, "-m", b.Monitors, "--id", id, "--key=" + secret} + cmd, err = execCommand("rbd", args) + output = string(cmd) + + if err, ok := err.(*exec.Error); ok { + if err.Err == exec.ErrNotFound { + glog.Errorf("rbd cmd not found") + // fail fast if command not found + return false, output, err + } + } + + // If command never succeed, returns its last error. + if err != nil { + return false, output, err + } + + if strings.Contains(output, imageWatcherStr) { + glog.V(4).Infof("rbd: watchers on %s: %s", image, output) + return true, output, nil + } else { + glog.Warningf("rbd: no watchers on %s", image) + return false, output, nil + } +} + +// DeleteImage deletes a ceph image with provision and volume options. +func deleteRBDImage(image string, b *rbdVolumeOptions) error { + var output []byte + found, _, err := rbdStatus(image, b) + if err != nil { + return err + } + if found { + glog.Info("rbd is still being used ", image) + return fmt.Errorf("rbd %s is still being used", image) + } + id := b.AdminID + secret := b.adminSecret + if id == "" { + id = b.UserID + secret = b.userSecret + } + + glog.V(4).Infof("rbd: rm %s using mon %s, pool %s id %s key %s", image, b.Monitors, b.Pool, id, secret) + args := []string{"rm", image, "--pool", b.Pool, "--id", id, "-m", b.Monitors, "--key=" + secret} + output, err = execCommand("rbd", args) + if err == nil { + return nil + } + glog.Errorf("failed to delete rbd image: %v, command output: %s", err, string(output)) + return err +} + +func execCommand(command string, args []string) ([]byte, error) { + cmd := exec.Command(command, args...) + return cmd.CombinedOutput() +} + +func getRBDVolumeOptions(volOptions map[string]string, client *kubernetes.Clientset) (*rbdVolumeOptions, error) { + rbdVolume := &rbdVolumeOptions{} + var ok bool + var err error + rbdVolume.AdminID, ok = volOptions["adminId"] + if !ok { + return nil, fmt.Errorf("Missing required parameter adminId") + } + rbdVolume.AdminSecretName, ok = volOptions["adminSecretName"] + if !ok { + return nil, fmt.Errorf("Missing required parameter adminSecretName") + } + rbdVolume.AdminSecretNamespace, ok = volOptions["adminSecretNamespace"] + if !ok { + rbdVolume.AdminSecretNamespace = "default" + } + rbdVolume.adminSecret, err = parseStorageClassSecret(rbdVolume.AdminSecretName, rbdVolume.AdminSecretNamespace, client) + if err != nil { + return nil, fmt.Errorf("Failed to retrieve Admin secret %v", err) + } + rbdVolume.Pool, ok = volOptions["pool"] + if !ok { + return nil, fmt.Errorf("Missing required parameter pool") + } + rbdVolume.Monitors, ok = volOptions["monitors"] + if !ok { + return nil, fmt.Errorf("Missing required parameter monitors") + } + if err != nil { + return nil, err + } + rbdVolume.UserID, ok = volOptions["userId"] + if !ok { + return nil, fmt.Errorf("Missing required parameter userId") + } + rbdVolume.UserSecretName, ok = volOptions["userSecretName"] + if ok { + rbdVolume.UserSecretNamespace, ok = volOptions["userSecretNamespace"] + if !ok { + rbdVolume.UserSecretNamespace = "default" + } + rbdVolume.userSecret, err = parseStorageClassSecret(rbdVolume.UserSecretName, rbdVolume.UserSecretNamespace, client) + if err != nil { + glog.Errorf("failed to retrieve user's secret: %s/%s (%v)", rbdVolume.UserSecretName, rbdVolume.UserSecretNamespace, err) + } + } + rbdVolume.ImageFormat, ok = volOptions["imageFormat"] + if !ok { + rbdVolume.ImageFormat = "2" + } + + return rbdVolume, nil +} + +func parseStorageClassSecret(secretName string, namespace string, client *kubernetes.Clientset) (string, error) { + if client == nil { + return "", fmt.Errorf("Cannot get kube client") + } + secrets, err := client.CoreV1().Secrets(namespace).Get(secretName, metav1.GetOptions{}) + if err != nil { + return "", err + } + secret := "" + for k, v := range secrets.Data { + if k == secretName { + return string(v), nil + } + secret = string(v) + } + + return secret, nil +} + +func attachRBDImage(image string, volOptions *rbdVolumeOptions) (string, error) { + var err error + var output []byte + + devicePath, found := waitForPath(volOptions.Pool, image, 1) + if !found { + attachdetachMutex.LockKey(string(volOptions.Pool + image)) + defer attachdetachMutex.UnlockKey(string(volOptions.Pool + image)) + + _, err = execCommand("modprobe", []string{"rbd"}) + if err != nil { + glog.Warningf("rbd: failed to load rbd kernel module:%v", err) + } + + backoff := wait.Backoff{ + Duration: rbdImageWatcherInitDelay, + Factor: rbdImageWatcherFactor, + Steps: rbdImageWatcherSteps, + } + err := wait.ExponentialBackoff(backoff, func() (bool, error) { + used, rbdOutput, err := rbdStatus(image, volOptions) + if err != nil { + return false, fmt.Errorf("fail to check rbd image status with: (%v), rbd output: (%s)", err, rbdOutput) + } + return !used, nil + }) + // return error if rbd image has not become available for the specified timeout + if err == wait.ErrWaitTimeout { + return "", fmt.Errorf("rbd image %s/%s is still being used", volOptions.Pool, image) + } + // return error if any other errors were encountered during wating for the image to becme avialble + if err != nil { + return "", err + } + + glog.V(1).Infof("rbd: map mon %s", volOptions.Monitors) + // If we don't have admin id/secret (e.g. attaching), fallback to user id/secret. + id := volOptions.AdminID + secret := volOptions.adminSecret + if id == "" { + id = volOptions.UserID + secret = volOptions.userSecret + } + + output, err = execCommand("rbd", []string{ + "map", image, "--pool", volOptions.Pool, "--id", id, "-m", volOptions.Monitors, "--key=" + secret}) + if err != nil { + glog.V(1).Infof("rbd: map error %v, rbd output: %s", err, string(output)) + return "", fmt.Errorf("rbd: map failed %v, rbd output: %s", err, string(output)) + } + devicePath, found = waitForPath(volOptions.Pool, image, 10) + if !found { + return "", fmt.Errorf("Could not map image %s/%s, Timeout after 10s", volOptions.Pool, image) + } + } + + return devicePath, nil +} + +func detachRBDImage(image string, volOptions *rbdVolumeOptions) error { + var err error + var output []byte + + glog.V(1).Infof("rbd: unmap device %s", volOptions.ImageMapping[image]) + // If we don't have admin id/secret (e.g. attaching), fallback to user id/secret. + id := volOptions.AdminID + secret := volOptions.adminSecret + if id == "" { + id = volOptions.UserID + secret = volOptions.userSecret + } + + output, err = execCommand("rbd", []string{ + "unmap", volOptions.ImageMapping[image], "--id", id, "--key=" + secret}) + if err != nil { + glog.V(1).Infof("rbd: unmap error %v, rbd output: %s", err, string(output)) + return fmt.Errorf("rbd: unmap failed %v, rbd output: %s", err, string(output)) + } + + return nil +} + +func getDevFromImageAndPool(pool, image string) (string, bool) { + // /sys/bus/rbd/devices/X/name and /sys/bus/rbd/devices/X/pool + sys_path := "/sys/bus/rbd/devices" + if dirs, err := ioutil.ReadDir(sys_path); err == nil { + for _, f := range dirs { + name := f.Name() + // first match pool, then match name + poolFile := path.Join(sys_path, name, "pool") + poolBytes, err := ioutil.ReadFile(poolFile) + if err != nil { + glog.V(4).Infof("Error reading %s: %v", poolFile, err) + continue + } + if strings.TrimSpace(string(poolBytes)) != pool { + glog.V(4).Infof("Device %s is not %q: %q", name, pool, string(poolBytes)) + continue + } + imgFile := path.Join(sys_path, name, "name") + imgBytes, err := ioutil.ReadFile(imgFile) + if err != nil { + glog.V(4).Infof("Error reading %s: %v", imgFile, err) + continue + } + if strings.TrimSpace(string(imgBytes)) != image { + glog.V(4).Infof("Device %s is not %q: %q", name, image, string(imgBytes)) + continue + } + // found a match, check if device exists + devicePath := "/dev/rbd" + name + if _, err := os.Lstat(devicePath); err == nil { + return devicePath, true + } + } + } + return "", false +} + +// stat a path, if not exists, retry maxRetries times +func waitForPath(pool, image string, maxRetries int) (string, bool) { + for i := 0; i < maxRetries; i++ { + devicePath, found := getDevFromImageAndPool(pool, image) + if found { + return devicePath, true + } + if i == maxRetries-1 { + break + } + time.Sleep(time.Second) + } + return "", false +} + +func persistVolInfo(image string, persistentStoragePath string, volInfo *rbdVolumeOptions) error { + file := path.Join(persistentStoragePath, image+".json") + fp, err := os.Create(file) + if err != nil { + return fmt.Errorf("rbd: create err %s/%s", file, err) + } + defer fp.Close() + + encoder := json.NewEncoder(fp) + if err = encoder.Encode(volInfo); err != nil { + return fmt.Errorf("rbd: encode err: %v.", err) + } + + return nil +} + +func loadVolInfo(image string, persistentStoragePath string, volInfo *rbdVolumeOptions) error { + file := path.Join(persistentStoragePath, image+".json") + fp, err := os.Open(file) + if err != nil { + return fmt.Errorf("rbd: open err %s/%s", file, err) + } + defer fp.Close() + + decoder := json.NewDecoder(fp) + if err = decoder.Decode(volInfo); err != nil { + return fmt.Errorf("rbd: decode err: %v.", err) + } + + return nil +} + +func deleteVolInfo(image string, persistentStoragePath string) error { + file := path.Join(persistentStoragePath, image+".json") + err := os.Remove(file) + if err != nil { + if err != os.ErrNotExist { + return fmt.Errorf("rbd: open err %s/%s", file, err) + } + } + return nil +} diff --git a/rbd/main.go b/rbd/main.go new file mode 100644 index 000000000..bf327123f --- /dev/null +++ b/rbd/main.go @@ -0,0 +1,80 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "github.com/golang/glog" + "os" + "path" + + "github.com/ceph/ceph-csi/pkg/rbd" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" +) + +func init() { + flag.Set("logtostderr", "true") +} + +var ( + endpoint = flag.String("endpoint", "unix://tmp/csi.sock", "CSI endpoint") + driverName = flag.String("drivername", "rbdplugin", "name of the driver") + nodeID = flag.String("nodeid", "", "node id") +) + +func main() { + flag.Parse() + + // creates the in-cluster config + config, err := rest.InClusterConfig() + if err != nil { + panic(err.Error()) + } + // creates the clientset + clientSet, err := kubernetes.NewForConfig(config) + if err != nil { + panic(err.Error()) + } + + if err := createPersistentStorage(path.Join(rbd.PluginFolder, "controller")); err != nil { + glog.Errorf("failed to create persistent storage for controller %v", err) + os.Exit(1) + } + if err := createPersistentStorage(path.Join(rbd.PluginFolder, "node")); err != nil { + glog.Errorf("failed to create persistent storage for node %v", err) + os.Exit(1) + } + + handle(clientSet) + os.Exit(0) +} + +func handle(clientSet *kubernetes.Clientset) { + driver := rbd.GetRBDDriver() + driver.Run(*driverName, *nodeID, *endpoint, clientSet) +} + +func createPersistentStorage(persistentStoragePath string) error { + if _, err := os.Stat(persistentStoragePath); os.IsNotExist(err) { + if err := os.MkdirAll(persistentStoragePath, os.FileMode(0755)); err != nil { + return err + } + } else { + } + return nil +} diff --git a/vendor/github.com/container-storage-interface/spec/lib/go/csi/csi.pb.go b/vendor/github.com/container-storage-interface/spec/lib/go/csi/csi.pb.go index 7f53a1cb4..7e9e8cfed 100644 --- a/vendor/github.com/container-storage-interface/spec/lib/go/csi/csi.pb.go +++ b/vendor/github.com/container-storage-interface/spec/lib/go/csi/csi.pb.go @@ -73,11 +73,9 @@ type VolumeCapability_AccessMode_Mode int32 const ( VolumeCapability_AccessMode_UNKNOWN VolumeCapability_AccessMode_Mode = 0 - // Can only be published once as read/write on a single node, at - // any given time. + // Can be published as read/write at one node at a time. VolumeCapability_AccessMode_SINGLE_NODE_WRITER VolumeCapability_AccessMode_Mode = 1 - // Can only be published once as readonly on a single node, at - // any given time. + // Can be published as readonly at one node at a time. VolumeCapability_AccessMode_SINGLE_NODE_READER_ONLY VolumeCapability_AccessMode_Mode = 2 // Can be published as readonly at multiple nodes simultaneously. VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY VolumeCapability_AccessMode_Mode = 3 @@ -1346,8 +1344,6 @@ type NodePublishVolumeRequest struct { // The path to which the volume will be published. It MUST be an // absolute path in the root filesystem of the process serving this // request. The CO SHALL ensure uniqueness of target_path per volume. - // The CO SHALL ensure that the path exists, and that the process - // serving the request has `read` and `write` permissions to the path. // This is a REQUIRED field. TargetPath string `protobuf:"bytes,4,opt,name=target_path,json=targetPath" json:"target_path,omitempty"` // The capability of the volume the CO expects the volume to have. @@ -1466,6 +1462,10 @@ type NodeUnpublishVolumeRequest struct { // sensitive and MUST be treated as such (not logged, etc.) by the CO. // This field is OPTIONAL. UserCredentials map[string]string `protobuf:"bytes,4,rep,name=user_credentials,json=userCredentials" json:"user_credentials,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + // Attributes of the volume to publish. This field is OPTIONAL and + // MUST match the attributes of the VolumeInfo identified by + // `volume_id`. + VolumeAttributes map[string]string `protobuf:"bytes,5,rep,name=volume_attributes,json=volumeAttributes" json:"volume_attributes,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` } func (m *NodeUnpublishVolumeRequest) Reset() { *m = NodeUnpublishVolumeRequest{} } @@ -1501,6 +1501,13 @@ func (m *NodeUnpublishVolumeRequest) GetUserCredentials() map[string]string { return nil } +func (m *NodeUnpublishVolumeRequest) GetVolumeAttributes() map[string]string { + if m != nil { + return m.VolumeAttributes + } + return nil +} + type NodeUnpublishVolumeResponse struct { }