code update

This commit is contained in:
Serguei Bezverkhi 2018-03-06 17:33:57 -05:00
parent e9033989a0
commit 1ff1f8be30
16 changed files with 367 additions and 199 deletions

3
.gitignore vendored
View File

@ -6,6 +6,9 @@
# docker build
/deploy/docker/rbdplugin
# rbdplugin executable
rbdplugin
# Emacs save files
*~
\#*\#

View File

@ -7,6 +7,6 @@ RUN yum install -y centos-release-ceph && \
yum install -y ceph-common e2fsprogs && \
yum clean all
COPY rbdplugin /rbdplugin
COPY _output/rbdplugin /rbdplugin
RUN chmod +x /rbdplugin
ENTRYPOINT ["/rbdplugin"]

112
Gopkg.lock generated
View File

@ -2,10 +2,10 @@
[[projects]]
branch = "master"
name = "github.com/container-storage-interface/spec"
packages = ["lib/go/csi"]
revision = "7ab01a90da87f9fef3ee1de0494962fdefaf7db7"
packages = ["lib/go/csi/v0"]
revision = "35d9f9d77954980e449e52c3f3e43c21bd8171f5"
version = "v0.2.0"
[[projects]]
branch = "master"
@ -14,16 +14,22 @@
revision = "23def4e6c14b4da8ac2ed8007337bc5eb5007998"
[[projects]]
branch = "master"
name = "github.com/golang/protobuf"
packages = ["proto","ptypes","ptypes/any","ptypes/duration","ptypes/timestamp"]
revision = "1e59b77b52bf8e4b449a57e6f79f21226d571845"
packages = [
"proto",
"ptypes",
"ptypes/any",
"ptypes/duration",
"ptypes/timestamp"
]
revision = "925541529c1fa6821df4e44ce2723319eb2be768"
version = "v1.0.0"
[[projects]]
branch = "master"
name = "github.com/kubernetes-csi/drivers"
packages = ["pkg/csi-common"]
revision = "d1ab787ad5510df08a3a98a091a41adeae4647b4"
revision = "1853bd0038cd634f277efda5c6548766a2a51ff3"
[[projects]]
name = "github.com/pborman/uuid"
@ -34,54 +40,110 @@
[[projects]]
branch = "master"
name = "golang.org/x/net"
packages = ["context","http2","http2/hpack","idna","internal/timeseries","lex/httplex","trace"]
revision = "42fe2e1c20de1054d3d30f82cc9fb5b41e2e3767"
packages = [
"context",
"http2",
"http2/hpack",
"idna",
"internal/timeseries",
"lex/httplex",
"trace"
]
revision = "d25186b37f34ebdbbea8f488ef055638dfab272d"
[[projects]]
branch = "master"
name = "golang.org/x/sys"
packages = ["unix"]
revision = "1792d66dc88e503d3cb2400578221cdf1f7fe26f"
revision = "dd2ff4accc098aceecb86b36eaa7829b2a17b1c9"
[[projects]]
branch = "master"
name = "golang.org/x/text"
packages = ["collate","collate/build","internal/colltab","internal/gen","internal/tag","internal/triegen","internal/ucd","language","secure/bidirule","transform","unicode/bidi","unicode/cldr","unicode/norm","unicode/rangetable"]
revision = "e19ae1496984b1c655b8044a65c0300a3c878dd3"
packages = [
"collate",
"collate/build",
"internal/colltab",
"internal/gen",
"internal/tag",
"internal/triegen",
"internal/ucd",
"language",
"secure/bidirule",
"transform",
"unicode/bidi",
"unicode/cldr",
"unicode/norm",
"unicode/rangetable"
]
revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0"
version = "v0.3.0"
[[projects]]
branch = "master"
name = "google.golang.org/genproto"
packages = ["googleapis/rpc/status"]
revision = "a8101f21cf983e773d0c1133ebc5424792003214"
revision = "df60624c1e9b9d2973e889c7a1cff73155da81c4"
[[projects]]
name = "google.golang.org/grpc"
packages = [".","balancer","balancer/base","balancer/roundrobin","codes","connectivity","credentials","encoding","grpclb/grpc_lb_v1/messages","grpclog","internal","keepalive","metadata","naming","peer","resolver","resolver/dns","resolver/passthrough","stats","status","tap","transport"]
revision = "f3955b8e9e244dd4dd4bc4f7b7a23a8445400a76"
version = "v1.9.0"
packages = [
".",
"balancer",
"balancer/base",
"balancer/roundrobin",
"codes",
"connectivity",
"credentials",
"encoding",
"encoding/proto",
"grpclb/grpc_lb_v1/messages",
"grpclog",
"internal",
"keepalive",
"metadata",
"naming",
"peer",
"resolver",
"resolver/dns",
"resolver/passthrough",
"stats",
"status",
"tap",
"transport"
]
revision = "8e4536a86ab602859c20df5ebfd0bd4228d08655"
version = "v1.10.0"
[[projects]]
branch = "release-1.9"
name = "k8s.io/apimachinery"
packages = ["pkg/util/runtime","pkg/util/sets","pkg/util/wait"]
revision = "68f9c3a1feb3140df59c67ced62d3a5df8e6c9c2"
packages = [
"pkg/util/runtime",
"pkg/util/sets",
"pkg/util/wait"
]
revision = "302974c03f7e50f16561ba237db776ab93594ef6"
version = "kubernetes-1.10.0-beta.1"
[[projects]]
name = "k8s.io/kubernetes"
packages = ["pkg/util/io","pkg/util/keymutex","pkg/util/mount","pkg/util/nsenter"]
revision = "3a1c9449a956b6026f075fa3134ff92f7d55f812"
version = "v1.9.1"
packages = [
"pkg/util/io",
"pkg/util/keymutex",
"pkg/util/mount",
"pkg/util/nsenter"
]
revision = "37555e6d24c2f951c40660ea59a80fa251982005"
version = "v1.10.0-beta.1"
[[projects]]
branch = "master"
name = "k8s.io/utils"
packages = ["exec"]
revision = "a99a3e11a96751670db62ba77c6d278d1136931e"
revision = "258e2a2fa64568210fbd6267cf1d8fd87c3cb86e"
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
inputs-digest = "8908f89154f277d98fd83b22edf73652d4c4e37bbd827bf11d9605c58ae3fd0e"
inputs-digest = "d409396ee410b5443abac732df725cb70ef9e0c940c1b807c0ff2698bab5d102"
solver-name = "gps-cdcl"
solver-version = 1

View File

@ -1,23 +1,31 @@
[[constraint]]
name = "github.com/container-storage-interface/spec"
version = "~0.2.0"
[[constraint]]
branch = "master"
name = "github.com/container-storage-interface/spec"
name = "github.com/kubernetes-csi/drivers"
[[constraint]]
branch = "master"
name = "github.com/golang/glog"
[[constraint]]
name = "google.golang.org/grpc"
version = "1.7.2"
[[override]]
revision = "5db89f0ca68677abc5eefce8f2a0a772c98ba52d"
name = "github.com/docker/distribution"
[[constraint]]
name = "github.com/docker/distribution"
revision = "edc3ab29cdff8694dd6feb85cfeb4b5f1b38ed9c"
name = "google.golang.org/grpc"
version = "1.10.0"
[[constraint]]
version = "kubernetes-1.10.0-beta.1"
name = "k8s.io/apimachinery"
[[constraint]]
name = "k8s.io/kubernetes"
version = "v1.9.1"
version = "v1.10.0-beta.1"
[[constraint]]
name = "k8s.io/apimachinery"
version = "kubernetes-1.9.1"
[[override]]
version = "kubernetes-1.10.0-beta.1"
name = "k8s.io/api"

View File

@ -28,12 +28,10 @@ rbdplugin:
CGO_ENABLED=0 GOOS=linux go build -a -ldflags '-extldflags "-static"' -o _output/rbdplugin ./rbd
container: rbdplugin
cp _output/rbdplugin deploy/docker
docker build -t $(IMAGE_NAME):$(IMAGE_VERSION) deploy/docker
docker build -t $(IMAGE_NAME):$(IMAGE_VERSION) .
push-container: container
docker push $(IMAGE_NAME):$(IMAGE_VERSION)
clean:
go clean -r -x
rm -f deploy/docker/rbdplugin
-rm -rf _output

View File

@ -110,7 +110,7 @@ The following output should be displayed:
```
NAMESPACE NAME READY STATUS RESTARTS AGE
default csi-attacher-0 1/1 Running 0 1d
default csi-nodeplugin-rbdplugin-qxqtl 2/2 Running 0 1d
default csi-rbdplugin-qxqtl 2/2 Running 0 1d
default csi-provisioner-0 1/1 Running 0 1d
```

View File

@ -12,6 +12,9 @@ apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: external-attacher-runner
rules:
- apiGroups: [""]
resources: ["events"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "update"]
@ -66,19 +69,19 @@ spec:
serviceAccount: csi-attacher
containers:
- name: csi-attacher
image: docker.io/k8scsi/csi-attacher:latest
image: quay.io/k8scsi/csi-attacher:v0.2.0
args:
- "--v=5"
- "--csi-address=$(ADDRESS)"
env:
- name: ADDRESS
value: /var/lib/kubelet/plugins/rbdplugin/csi.sock
value: /var/lib/kubelet/plugins/csi-rbdplugin/csi.sock
imagePullPolicy: "IfNotPresent"
volumeMounts:
- name: socket-dir
mountPath: /var/lib/kubelet/plugins/rbdplugin
mountPath: /var/lib/kubelet/plugins/csi-rbdplugin
volumes:
- name: socket-dir
hostPath:
path: /var/lib/kubelet/plugins/rbdplugin
path: /var/lib/kubelet/plugins/csi-rbdplugin
type: DirectoryOrCreate

View File

@ -18,6 +18,9 @@ apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: external-provisioner-runner
rules:
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get", "list"]
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "create", "delete"]
@ -75,20 +78,20 @@ spec:
serviceAccount: csi-provisioner
containers:
- name: csi-provisioner
image: quay.io/k8scsi/csi-provisioner:latest
image: quay.io/k8scsi/csi-provisioner:v0.2.0
args:
- "--provisioner=rbdplugin"
- "--provisioner=csi-rbdplugin"
- "--csi-address=$(ADDRESS)"
- "--v=5"
env:
- name: ADDRESS
value: /var/lib/kubelet/plugins/rbdplugin/csi.sock
value: /var/lib/kubelet/plugins/csi-rbdplugin/csi.sock
imagePullPolicy: "IfNotPresent"
volumeMounts:
- name: socket-dir
mountPath: /var/lib/kubelet/plugins/rbdplugin
mountPath: /var/lib/kubelet/plugins/csi-rbdplugin
volumes:
- name: socket-dir
hostPath:
path: /var/lib/kubelet/plugins/rbdplugin
path: /var/lib/kubelet/plugins/csi-rbdplugin
type: DirectoryOrCreate

View File

@ -1,19 +1,10 @@
apiVersion: v1
kind: Secret
metadata:
name: ceph-secret-admin
namespace: kube-system
type: "kubernetes.io/rbd"
name: csi-ceph-secret
namespace: default
data:
#Please note this value is base64 encoded.
key: QVFDZUhPMVpJTFBQRFJBQTd6dzNkNzZicGxrdlR3em9vc3lidkE9PQo=
type: kubernetes.io/rbd
---
apiVersion: v1
kind: Secret
metadata:
name: ceph-secret-user
type: "kubernetes.io/rbd"
data:
#Please note this value is base64 encoded.
key: QVFDZDR1MVoxSDI0QnhBQWFxdmZIRnFuMSs0RFZlK1pRZ0ZmUEE9PQo=
# Key value corresponds to a user name defined in ceph cluster
admin: QVFDZUhPMVpJTFBQRFJBQTd6dzNkNzZicGxrdlR3em9vc3lidkE9PQo=
kubernetes: QVFDZDR1MVoxSDI0QnhBQWFxdmZIRnFuMSs0RFZlK1pRZ0ZmUEE9PQo=

View File

@ -1,13 +1,11 @@
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: rbd
provisioner: rbdplugin
name: csi-rbd
provisioner: csi-rbdplugin
parameters:
monitors: 192.168.80.233:6789
pool: kubernetes
adminID: admin
adminSecret: AQAmsGBap4EoBhAAET/Hc7fBqAZj/cy7cDcoQA==
userID: kube
userSecret: AQAMgXhVwBCeDhAA9nlPaFyfUSatGD4drFWDvQ==
csiProvisionerSecretName: csi-ceph-secret
csiProvisionerSecretNamespace: default
reclaimPolicy: Delete

View File

@ -3,20 +3,17 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: csi-nodeplugin
name: csi-rbdplugin
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: csi-nodeplugin
name: csi-rbdplugin
rules:
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get", "list", "update"]
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get", "list"]
- apiGroups: [""]
resources: ["namespaces"]
verbs: ["get", "list"]
@ -30,14 +27,14 @@ rules:
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: csi-nodeplugin
name: csi-rbdplugin
subjects:
- kind: ServiceAccount
name: csi-nodeplugin
name: csi-rbdplugin
namespace: default
roleRef:
kind: ClusterRole
name: csi-nodeplugin
name: csi-rbdplugin
apiGroup: rbac.authorization.k8s.io
---
@ -47,35 +44,35 @@ roleRef:
kind: DaemonSet
apiVersion: apps/v1beta2
metadata:
name: csi-nodeplugin-rbdplugin
name: csi-rbdplugin
spec:
selector:
matchLabels:
app: csi-nodeplugin-rbdplugin
app: csi-rbdplugin
template:
metadata:
labels:
app: csi-nodeplugin-rbdplugin
app: csi-rbdplugin
spec:
serviceAccount: csi-nodeplugin
serviceAccount: csi-rbdplugin
hostNetwork: true
containers:
- name: driver-registrar
image: docker.io/k8scsi/driver-registrar:latest
image: quay.io/k8scsi/driver-registrar:v0.2.0
args:
- "--v=5"
- "--csi-address=$(ADDRESS)"
env:
- name: ADDRESS
value: /var/lib/kubelet/plugins/rbdplugin/csi.sock
value: /var/lib/kubelet/plugins/csi-rbdplugin/csi.sock
- name: KUBE_NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
volumeMounts:
- name: socket-dir
mountPath: /var/lib/kubelet/plugins/rbdplugin
- name: rbdplugin
mountPath: /var/lib/kubelet/plugins/csi-rbdplugin
- name: csi-rbdplugin
securityContext:
privileged: true
capabilities:
@ -86,18 +83,18 @@ spec:
- "--nodeid=$(NODE_ID)"
- "--endpoint=$(CSI_ENDPOINT)"
- "--v=5"
- "--drivername=rbdplugin"
- "--drivername=csi-rbdplugin"
env:
- name: NODE_ID
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: CSI_ENDPOINT
value: unix://var/lib/kubelet/plugins/rbdplugin/csi.sock
value: unix://var/lib/kubelet/plugins/csi-rbdplugin/csi.sock
imagePullPolicy: "IfNotPresent"
volumeMounts:
- name: plugin-dir
mountPath: /var/lib/kubelet/plugins/rbdplugin
mountPath: /var/lib/kubelet/plugins/csi-rbdplugin
- name: pods-mount-dir
mountPath: /var/lib/kubelet/pods
mountPropagation: "Bidirectional"
@ -111,7 +108,7 @@ spec:
volumes:
- name: plugin-dir
hostPath:
path: /var/lib/kubelet/plugins/rbdplugin
path: /var/lib/kubelet/plugins/csi-rbdplugin
type: DirectoryOrCreate
- name: pods-mount-dir
hostPath:
@ -119,7 +116,7 @@ spec:
type: Directory
- name: socket-dir
hostPath:
path: /var/lib/kubelet/plugins/rbdplugin
path: /var/lib/kubelet/plugins/csi-rbdplugin
type: DirectoryOrCreate
- name: host-dev
hostPath:
@ -129,4 +126,4 @@ spec:
path: /sys
- name: lib-modules
hostPath:
path: /lib/modules
path: /lib/modules

View File

@ -20,12 +20,13 @@ import (
"fmt"
"path"
"github.com/container-storage-interface/spec/lib/go/csi/v0"
"github.com/golang/glog"
"github.com/kubernetes-csi/drivers/pkg/csi-common"
"github.com/pborman/uuid"
"golang.org/x/net/context"
"github.com/container-storage-interface/spec/lib/go/csi"
"github.com/kubernetes-csi/drivers/pkg/csi-common"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
const (
@ -36,17 +37,42 @@ type controllerServer struct {
*csicommon.DefaultControllerServer
}
func GetVersionString(ver *csi.Version) string {
return fmt.Sprintf("%d.%d.%d", ver.Major, ver.Minor, ver.Patch)
}
func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) {
if err := cs.Driver.ValidateControllerServiceRequest(req.Version, csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME); err != nil {
if err := cs.Driver.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME); err != nil {
glog.V(3).Infof("invalid create volume req: %v", req)
return nil, err
}
// Check sanity of request Name, Volume Capabilities
if len(req.Name) == 0 {
return nil, status.Error(codes.InvalidArgument, "Volume Name cannot be empty")
}
if req.VolumeCapabilities == nil {
return nil, status.Error(codes.InvalidArgument, "Volume Capabilities cannot be empty")
}
volOptions, err := getRBDVolumeOptions(req.GetParameters())
// Need to check for already existing volume name, and if found
// check for the requested capacity and already allocated capacity
if exVol, err := getRBDVolumeByName(req.GetName()); err == nil {
// Since err is nil, it means the volume with the same name already exists
// need to check if the size of exisiting volume is the same as in new
// request
if exVol.VolSize >= int64(req.GetCapacityRange().GetRequiredBytes()) {
// exisiting volume is compatible with new request and should be reused.
// TODO (sbezverk) Do I need to make sure that RBD volume still exists?
return &csi.CreateVolumeResponse{
Volume: &csi.Volume{
Id: exVol.VolID,
CapacityBytes: int64(exVol.VolSize),
Attributes: req.GetParameters(),
},
}, nil
}
return nil, status.Error(codes.AlreadyExists, fmt.Sprintf("Volume with the same name: %s but with different size already exist", req.GetName()))
}
// TODO (sbezverk) Last check for not exceeding total storage capacity
rbdVol, err := getRBDVolumeOptions(req.GetParameters())
if err != nil {
return nil, err
}
@ -55,21 +81,23 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
volName := req.GetName()
uniqueID := uuid.NewUUID().String()
if len(volName) == 0 {
volName = volOptions.Pool + "-dynamic-pvc-" + uniqueID
volName = rbdVol.Pool + "-dynamic-pvc-" + uniqueID
}
volOptions.VolName = volName
rbdVol.VolName = volName
volumeID := "csi-rbd-" + uniqueID
rbdVol.VolID = volumeID
// Volume Size - Default is 1 GiB
volSizeBytes := int64(oneGB)
if req.GetCapacityRange() != nil {
volSizeBytes = int64(req.GetCapacityRange().GetRequiredBytes())
}
rbdVol.VolSize = volSizeBytes
volSizeGB := int(volSizeBytes / 1024 / 1024 / 1024)
// Check if there is already RBD image with requested name
found, _, _ := rbdStatus(volOptions)
found, _, _ := rbdStatus(rbdVol, req.GetControllerCreateSecrets())
if !found {
if err := createRBDImage(volOptions, volSizeGB); err != nil {
if err := createRBDImage(rbdVol, volSizeGB, req.GetControllerCreateSecrets()); err != nil {
if err != nil {
glog.Warningf("failed to create volume: %v", err)
return nil, err
@ -77,12 +105,11 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
}
glog.V(4).Infof("create volume %s", volName)
}
// Storing volInfo into a persistent file, will need info to delete rbd image
// in ControllerUnpublishVolume
if err := persistVolInfo(volumeID, path.Join(PluginFolder, "controller"), volOptions); err != nil {
// Storing volInfo into a persistent file.
if err := persistVolInfo(volumeID, path.Join(PluginFolder, "controller"), rbdVol); err != nil {
glog.Warningf("rbd: failed to store volInfo with error: %v", err)
}
rbdVolumes[volumeID] = *rbdVol
return &csi.CreateVolumeResponse{
Volume: &csi.Volume{
Id: volumeID,
@ -93,23 +120,21 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
}
func (cs *controllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) {
if err := cs.Driver.ValidateControllerServiceRequest(req.Version, csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME); err != nil {
if err := cs.Driver.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME); err != nil {
glog.Warningf("invalid delete volume req: %v", req)
return nil, err
}
// For now the image get unconditionally deleted, but here retention policy can be checked
volumeID := req.GetVolumeId()
volOptions := &rbdVolumeOptions{}
if err := loadVolInfo(volumeID, path.Join(PluginFolder, "controller"), volOptions); err != nil {
rbdVol := &rbdVolume{}
if err := loadVolInfo(volumeID, path.Join(PluginFolder, "controller"), rbdVol); err != nil {
return nil, err
}
volName := volOptions.VolName
volName := rbdVol.VolName
// Deleting rbd image
glog.V(4).Infof("deleting volume %s", volName)
if err := deleteRBDImage(volOptions); err != nil {
glog.V(3).Infof("failed to delete rbd image: %s/%s with error: %v", volOptions.Pool, volName, err)
if err := deleteRBDImage(rbdVol, req.GetControllerDeleteSecrets()); err != nil {
glog.V(3).Infof("failed to delete rbd image: %s/%s with error: %v", rbdVol.Pool, volName, err)
return nil, err
}
// Removing persistent storage file for the unmapped volume
@ -117,6 +142,7 @@ func (cs *controllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVol
return nil, err
}
delete(rbdVolumes, volumeID)
return &csi.DeleteVolumeResponse{}, nil
}

View File

@ -24,7 +24,7 @@ import (
"github.com/golang/glog"
"golang.org/x/net/context"
"github.com/container-storage-interface/spec/lib/go/csi"
"github.com/container-storage-interface/spec/lib/go/csi/v0"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
@ -67,7 +67,7 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis
}
volOptions.VolName = volName
// Mapping RBD image
devicePath, err := attachRBDImage(volOptions)
devicePath, err := attachRBDImage(volOptions, req.GetNodePublishSecrets())
if err != nil {
return nil, err
}
@ -130,3 +130,19 @@ func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu
return &csi.NodeUnpublishVolumeResponse{}, nil
}
func (ns *nodeServer) NodeStageVolume(
ctx context.Context,
req *csi.NodeStageVolumeRequest) (
*csi.NodeStageVolumeResponse, error) {
return nil, status.Error(codes.Unimplemented, "")
}
func (ns *nodeServer) NodeUnstageVolume(
ctx context.Context,
req *csi.NodeUnstageVolumeRequest) (
*csi.NodeUnstageVolumeResponse, error) {
return nil, status.Error(codes.Unimplemented, "")
}

View File

@ -17,15 +17,25 @@ limitations under the License.
package rbd
import (
"encoding/json"
"io/ioutil"
"os"
"path"
"strings"
"github.com/golang/glog"
"github.com/container-storage-interface/spec/lib/go/csi"
"github.com/container-storage-interface/spec/lib/go/csi/v0"
"github.com/kubernetes-csi/drivers/pkg/csi-common"
)
// PluginFolder defines the location of rbdplugin
const (
PluginFolder = "/var/lib/kubelet/plugins/rbdplugin"
PluginFolder = "/var/lib/kubelet/plugins/csi-rbdplugin"
// RBDUserID used as a key in credentials map to extract the key which is
// passed be the provisioner, the value od RBDUserID must match to the key used
// in Secret object.
RBDUserID = "admin"
)
type rbd struct {
@ -41,13 +51,54 @@ type rbd struct {
var (
rbdDriver *rbd
version = csi.Version{
Minor: 2,
}
version = "0.2.0"
)
func GetSupportedVersions() []*csi.Version {
return []*csi.Version{&version}
var rbdVolumes map[string]rbdVolume
// Init checks for the persistent volume file and loads all found volumes
// into a memory structure
func init() {
rbdVolumes = map[string]rbdVolume{}
if _, err := os.Stat(path.Join(PluginFolder, "controller")); os.IsNotExist(err) {
glog.Infof("rbd: folder %s not found. Creating... \n", path.Join(PluginFolder, "controller"))
if err := os.Mkdir(path.Join(PluginFolder, "controller"), 0755); err != nil {
glog.Fatalf("Failed to create a controller's volumes folder with error: %v\n", err)
}
return
}
// Since "controller" folder exists, it means the rbdplugin has already been running, it means
// there might be some volumes left, they must be re-inserted into rbdVolumes map
loadExVolumes()
}
// loadExVolumes check for any *.json files in the PluginFolder/controller folder
// and loads then into rbdVolumes map
func loadExVolumes() {
rbdVol := rbdVolume{}
files, err := ioutil.ReadDir(path.Join(PluginFolder, "controller"))
if err != nil {
glog.Infof("rbd: failed to read controller's volumes folder: %s error:%v", path.Join(PluginFolder, "controller"), err)
return
}
for _, f := range files {
if !strings.HasSuffix(f.Name(), ".json") {
continue
}
fp, err := os.Open(path.Join(PluginFolder, "controller", f.Name()))
if err != nil {
glog.Infof("rbd: open file: %s err %%v", f.Name(), err)
continue
}
decoder := json.NewDecoder(fp)
if err = decoder.Decode(&rbdVol); err != nil {
glog.Infof("rbd: decode file: %s err: %v", f.Name(), err)
fp.Close()
continue
}
rbdVolumes[rbdVol.VolID] = rbdVol
}
glog.Infof("rbd: Loaded %d volumes from %s", len(rbdVolumes), path.Join(PluginFolder, "controller"))
}
func GetRBDDriver() *rbd {
@ -73,10 +124,10 @@ func NewNodeServer(d *csicommon.CSIDriver) *nodeServer {
}
func (rbd *rbd) Run(driverName, nodeID, endpoint string) {
glog.Infof("Driver: %v version: %v", driverName, GetVersionString(&version))
glog.Infof("Driver: %v version: %v", driverName, version)
// Initialize default library driver
rbd.driver = csicommon.NewCSIDriver(driverName, &version, GetSupportedVersions(), nodeID)
rbd.driver = csicommon.NewCSIDriver(driverName, version, nodeID)
if rbd.driver == nil {
glog.Fatalln("Failed to initialize CSI Driver.")
}

View File

@ -19,15 +19,16 @@ package rbd
import (
"encoding/json"
"fmt"
"github.com/golang/glog"
"io/ioutil"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/kubernetes/pkg/util/keymutex"
"os"
"os/exec"
"path"
"strings"
"time"
"github.com/golang/glog"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/kubernetes/pkg/util/keymutex"
)
const (
@ -44,22 +45,29 @@ const (
rbdImageWatcherSteps = 10
)
type rbdVolumeOptions struct {
VolName string `json:"volName"`
Monitors string `json:"monitors"`
Pool string `json:"pool"`
AdminID string `json:"adminID"`
AdminSecret string `json:"adminSecret"`
UserID string `json:"userID"`
UserSecret string `json:"userSecret"`
ImageFormat string `json:"imageFormat"`
type rbdVolume struct {
VolName string `json:"volName"`
VolID string `json:"volID"`
Monitors string `json:"monitors"`
Pool string `json:"pool"`
ImageFormat string `json:"imageFormat"`
// TODO (sbezverk) check if it is used and how
ImageFeatures []string `json:"imageFeatures"`
VolSize int64 `json:"volSize"`
}
var attachdetachMutex = keymutex.NewKeyMutex()
func getRBDKey(id string, credentials map[string]string) (string, error) {
if key, ok := credentials[id]; ok {
return key, nil
}
return "", fmt.Errorf("RBD key for ID: %s not found", id)
}
// CreateImage creates a new ceph image with provision and volume options.
func createRBDImage(pOpts *rbdVolumeOptions, volSz int) error {
func createRBDImage(pOpts *rbdVolume, volSz int, credentials map[string]string) error {
var output []byte
var err error
@ -68,12 +76,16 @@ func createRBDImage(pOpts *rbdVolumeOptions, volSz int) error {
image := pOpts.VolName
volSzGB := fmt.Sprintf("%dG", volSz)
if pOpts.ImageFormat == rbdImageFormat2 {
glog.V(4).Infof("rbd: create %s size %s format %s (features: %s) using mon %s, pool %s id %s key %s", image, volSzGB, pOpts.ImageFormat, pOpts.ImageFeatures, mon, pOpts.Pool, pOpts.AdminID, pOpts.AdminSecret)
} else {
glog.V(4).Infof("rbd: create %s size %s format %s using mon %s, pool %s id %s key %s", image, volSzGB, pOpts.ImageFormat, mon, pOpts.Pool, pOpts.AdminID, pOpts.AdminSecret)
key, err := getRBDKey(RBDUserID, credentials)
if err != nil {
return err
}
args := []string{"create", image, "--size", volSzGB, "--pool", pOpts.Pool, "--id", pOpts.AdminID, "-m", mon, "--key=" + pOpts.AdminSecret, "--image-format", pOpts.ImageFormat}
if pOpts.ImageFormat == rbdImageFormat2 {
glog.V(4).Infof("rbd: create %s size %s format %s (features: %s) using mon %s, pool %s id %s key %s", image, volSzGB, pOpts.ImageFormat, pOpts.ImageFeatures, mon, pOpts.Pool, RBDUserID, key)
} else {
glog.V(4).Infof("rbd: create %s size %s format %s using mon %s, pool %s id %s key %s", image, volSzGB, pOpts.ImageFormat, mon, pOpts.Pool, RBDUserID, key)
}
args := []string{"create", image, "--size", volSzGB, "--pool", pOpts.Pool, "--id", RBDUserID, "-m", mon, "--key=" + key, "--image-format", pOpts.ImageFormat}
if pOpts.ImageFormat == rbdImageFormat2 {
// if no image features is provided, it results in empty string
// which disable all RBD image format 2 features as we expected
@ -91,22 +103,20 @@ func createRBDImage(pOpts *rbdVolumeOptions, volSz int) error {
// rbdStatus checks if there is watcher on the image.
// It returns true if there is a watcher onthe image, otherwise returns false.
func rbdStatus(b *rbdVolumeOptions) (bool, string, error) {
func rbdStatus(pOpts *rbdVolume, credentials map[string]string) (bool, string, error) {
var err error
var output string
var cmd []byte
image := b.VolName
image := pOpts.VolName
// If we don't have admin id/secret (e.g. attaching), fallback to user id/secret.
id := b.AdminID
secret := b.AdminSecret
if id == "" {
id = b.UserID
secret = b.UserSecret
key, err := getRBDKey(RBDUserID, credentials)
if err != nil {
return false, "", err
}
glog.V(4).Infof("rbd: status %s using mon %s, pool %s id %s key %s", image, b.Monitors, b.Pool, id, secret)
args := []string{"status", image, "--pool", b.Pool, "-m", b.Monitors, "--id", id, "--key=" + secret}
glog.V(4).Infof("rbd: status %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, RBDUserID, key)
args := []string{"status", image, "--pool", pOpts.Pool, "-m", pOpts.Monitors, "--id", RBDUserID, "--key=" + key}
cmd, err = execCommand("rbd", args)
output = string(cmd)
@ -133,10 +143,10 @@ func rbdStatus(b *rbdVolumeOptions) (bool, string, error) {
}
// DeleteImage deletes a ceph image with provision and volume options.
func deleteRBDImage(b *rbdVolumeOptions) error {
func deleteRBDImage(pOpts *rbdVolume, credentials map[string]string) error {
var output []byte
image := b.VolName
found, _, err := rbdStatus(b)
image := pOpts.VolName
found, _, err := rbdStatus(pOpts, credentials)
if err != nil {
return err
}
@ -144,15 +154,13 @@ func deleteRBDImage(b *rbdVolumeOptions) error {
glog.Info("rbd is still being used ", image)
return fmt.Errorf("rbd %s is still being used", image)
}
id := b.AdminID
secret := b.AdminSecret
if id == "" {
id = b.UserID
secret = b.UserSecret
key, err := getRBDKey(RBDUserID, credentials)
if err != nil {
return err
}
glog.V(4).Infof("rbd: rm %s using mon %s, pool %s id %s key %s", image, b.Monitors, b.Pool, id, secret)
args := []string{"rm", image, "--pool", b.Pool, "--id", id, "-m", b.Monitors, "--key=" + secret}
glog.V(4).Infof("rbd: rm %s using mon %s, pool %s id %s key %s", image, pOpts.Monitors, pOpts.Pool, RBDUserID, key)
args := []string{"rm", image, "--pool", pOpts.Pool, "--id", RBDUserID, "-m", pOpts.Monitors, "--key=" + key}
output, err = execCommand("rbd", args)
if err == nil {
return nil
@ -166,42 +174,26 @@ func execCommand(command string, args []string) ([]byte, error) {
return cmd.CombinedOutput()
}
func getRBDVolumeOptions(volOptions map[string]string) (*rbdVolumeOptions, error) {
rbdVolume := &rbdVolumeOptions{}
func getRBDVolumeOptions(volOptions map[string]string) (*rbdVolume, error) {
var ok bool
rbdVolume.AdminID, ok = volOptions["adminID"]
if !ok {
return nil, fmt.Errorf("Missing required parameter adminID")
}
rbdVolume.AdminSecret, ok = volOptions["adminSecret"]
if !ok {
return nil, fmt.Errorf("Missing required parameter adminSecret")
}
rbdVolume.Pool, ok = volOptions["pool"]
rbdVol := &rbdVolume{}
rbdVol.Pool, ok = volOptions["pool"]
if !ok {
return nil, fmt.Errorf("Missing required parameter pool")
}
rbdVolume.Monitors, ok = volOptions["monitors"]
rbdVol.Monitors, ok = volOptions["monitors"]
if !ok {
return nil, fmt.Errorf("Missing required parameter monitors")
}
rbdVolume.UserID, ok = volOptions["userID"]
rbdVol.ImageFormat, ok = volOptions["imageFormat"]
if !ok {
return nil, fmt.Errorf("Missing required parameter userID")
}
rbdVolume.UserSecret, ok = volOptions["userSecret"]
if !ok {
return nil, fmt.Errorf("Missing required parameter userSecret")
}
rbdVolume.ImageFormat, ok = volOptions["imageFormat"]
if !ok {
rbdVolume.ImageFormat = "2"
rbdVol.ImageFormat = "2"
}
return rbdVolume, nil
return rbdVol, nil
}
func attachRBDImage(volOptions *rbdVolumeOptions) (string, error) {
func attachRBDImage(volOptions *rbdVolume, credentials map[string]string) (string, error) {
var err error
var output []byte
@ -222,7 +214,7 @@ func attachRBDImage(volOptions *rbdVolumeOptions) (string, error) {
Steps: rbdImageWatcherSteps,
}
err := wait.ExponentialBackoff(backoff, func() (bool, error) {
used, rbdOutput, err := rbdStatus(volOptions)
used, rbdOutput, err := rbdStatus(volOptions, credentials)
if err != nil {
return false, fmt.Errorf("fail to check rbd image status with: (%v), rbd output: (%s)", err, rbdOutput)
}
@ -238,11 +230,13 @@ func attachRBDImage(volOptions *rbdVolumeOptions) (string, error) {
}
glog.V(1).Infof("rbd: map mon %s", volOptions.Monitors)
id := volOptions.UserID
secret := volOptions.UserSecret
key, err := getRBDKey(RBDUserID, credentials)
if err != nil {
return "", err
}
output, err = execCommand("rbd", []string{
"map", image, "--pool", volOptions.Pool, "--id", id, "-m", volOptions.Monitors, "--key=" + secret})
"map", image, "--pool", volOptions.Pool, "--id", RBDUserID, "-m", volOptions.Monitors, "--key=" + key})
if err != nil {
glog.V(1).Infof("rbd: map error %v, rbd output: %s", err, string(output))
return "", fmt.Errorf("rbd: map failed %v, rbd output: %s", err, string(output))
@ -322,23 +316,24 @@ func waitForPath(pool, image string, maxRetries int) (string, bool) {
return "", false
}
func persistVolInfo(image string, persistentStoragePath string, volInfo *rbdVolumeOptions) error {
func persistVolInfo(image string, persistentStoragePath string, volInfo *rbdVolume) error {
file := path.Join(persistentStoragePath, image+".json")
fp, err := os.Create(file)
if err != nil {
glog.Errorf("rbd: failed to create persistent storage file %s with error: %v\n", file, err)
return fmt.Errorf("rbd: create err %s/%s", file, err)
}
defer fp.Close()
encoder := json.NewEncoder(fp)
if err = encoder.Encode(volInfo); err != nil {
return fmt.Errorf("rbd: encode err: %v.", err)
glog.Errorf("rbd: failed to encode volInfo: %+v for file: %s with error: %v\n", volInfo, file, err)
return fmt.Errorf("rbd: encode err: %v", err)
}
glog.Infof("rbd: successfully saved volInfo: %+v into file: %s\n", volInfo, file)
return nil
}
func loadVolInfo(image string, persistentStoragePath string, volInfo *rbdVolumeOptions) error {
func loadVolInfo(image string, persistentStoragePath string, volInfo *rbdVolume) error {
file := path.Join(persistentStoragePath, image+".json")
fp, err := os.Open(file)
if err != nil {
@ -356,11 +351,28 @@ func loadVolInfo(image string, persistentStoragePath string, volInfo *rbdVolumeO
func deleteVolInfo(image string, persistentStoragePath string) error {
file := path.Join(persistentStoragePath, image+".json")
glog.Infof("rbd: Deleting file for Volume: %s at: %s resulting path: %+v\n", image, persistentStoragePath, file)
err := os.Remove(file)
if err != nil {
if err != os.ErrNotExist {
return fmt.Errorf("rbd: open err %s/%s", file, err)
return fmt.Errorf("rbd: error removing file: %s/%s", file, err)
}
}
return nil
}
func getRBDVolumeByID(volumeID string) (rbdVolume, error) {
if rbdVol, ok := rbdVolumes[volumeID]; ok {
return rbdVol, nil
}
return rbdVolume{}, fmt.Errorf("volume id %s does not exit in the volumes list", volumeID)
}
func getRBDVolumeByName(volName string) (rbdVolume, error) {
for _, rbdVol := range rbdVolumes {
if rbdVol.VolName == volName {
return rbdVol, nil
}
}
return rbdVolume{}, fmt.Errorf("volume name %s does not exit in the volumes list", volName)
}

View File

@ -31,7 +31,7 @@ func init() {
var (
endpoint = flag.String("endpoint", "unix://tmp/csi.sock", "CSI endpoint")
driverName = flag.String("drivername", "rbdplugin", "name of the driver")
driverName = flag.String("drivername", "csi-rbdplugin", "name of the driver")
nodeID = flag.String("nodeid", "", "node id")
)