mirror of
https://github.com/ceph/ceph-csi.git
synced 2024-12-21 04:20:23 +00:00
Merge pull request #9 from rootfs/rbd-refactoring
rbd plugin refactoring: remove k8s clientSet
This commit is contained in:
commit
f7f8e13909
52
README.md
52
README.md
@ -1,8 +1,60 @@
|
|||||||
# Ceph CSI
|
# Ceph CSI
|
||||||
[Container Storage Interface (CSI)](https://github.com/container-storage-interface/) driver, provisioner, and attacher for Ceph RBD and CephFS
|
[Container Storage Interface (CSI)](https://github.com/container-storage-interface/) driver, provisioner, and attacher for Ceph RBD and CephFS
|
||||||
|
|
||||||
|
# Prerequisite
|
||||||
|
|
||||||
|
## Enable Mount Propagation in Docker
|
||||||
|
|
||||||
|
Comment out `MountFlags=slave` in docker systemd service then restart docker service.
|
||||||
|
```bash
|
||||||
|
# systemctl daemon-reload
|
||||||
|
# systemctl restart docker
|
||||||
|
```
|
||||||
|
|
||||||
|
## Enable Kubernetes Feature Gates
|
||||||
|
|
||||||
|
Enable features `MountPropagation=true,CSIPersistentVolume=true` and runtime config `storage.k8s.io/v1alpha1=true`
|
||||||
|
|
||||||
# Build
|
# Build
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# make container
|
||||||
|
```
|
||||||
|
|
||||||
# Test
|
# Test
|
||||||
|
|
||||||
|
## Start rbdplugin and driver registrar
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# kubectl create -f deploy/kubernetes/rbdplugin.yaml
|
||||||
|
```
|
||||||
|
|
||||||
|
### Start CSI external volume provisioner
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# kubectl create -f deploy/kubernetes/csi-provisioner.yaml
|
||||||
|
```
|
||||||
|
|
||||||
|
### Start CSI external volume attacher
|
||||||
|
|
||||||
|
```
|
||||||
|
# kubectl create -f deploy/kubernetes/csi-attacher.yaml
|
||||||
|
```
|
||||||
|
|
||||||
|
### Verify all componets are ready
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# kubectl get pod
|
||||||
|
NAME READY STATUS RESTARTS AGE
|
||||||
|
csi-attacher-0 1/1 Running 0 6s
|
||||||
|
csi-nodeplugin-rbdplugin-kwhhc 2/2 Running 0 6m
|
||||||
|
csi-provisioner-0 1/1 Running 0 1m
|
||||||
|
```
|
||||||
|
|
||||||
|
### Create a CSI storage class
|
||||||
|
|
||||||
|
|
||||||
|
### Create a PVC
|
||||||
|
|
||||||
|
### Create a Pod
|
||||||
|
|
||||||
|
@ -75,7 +75,7 @@ spec:
|
|||||||
serviceAccount: csi-provisioner
|
serviceAccount: csi-provisioner
|
||||||
containers:
|
containers:
|
||||||
- name: csi-provisioner
|
- name: csi-provisioner
|
||||||
image: docker.io/k8scsi/csi-provisioner:latest
|
image: quay.io/k8scsi/csi-provisioner:latest
|
||||||
args:
|
args:
|
||||||
- "--provisioner=rbdplugin"
|
- "--provisioner=rbdplugin"
|
||||||
- "--csi-address=$(ADDRESS)"
|
- "--csi-address=$(ADDRESS)"
|
||||||
|
@ -11,6 +11,9 @@ apiVersion: rbac.authorization.k8s.io/v1
|
|||||||
metadata:
|
metadata:
|
||||||
name: csi-nodeplugin
|
name: csi-nodeplugin
|
||||||
rules:
|
rules:
|
||||||
|
- apiGroups: [""]
|
||||||
|
resources: ["nodes"]
|
||||||
|
verbs: ["get", "list", "update"]
|
||||||
- apiGroups: [""]
|
- apiGroups: [""]
|
||||||
resources: ["secrets"]
|
resources: ["secrets"]
|
||||||
verbs: ["get", "list"]
|
verbs: ["get", "list"]
|
||||||
@ -58,7 +61,7 @@ spec:
|
|||||||
hostNetwork: true
|
hostNetwork: true
|
||||||
containers:
|
containers:
|
||||||
- name: driver-registrar
|
- name: driver-registrar
|
||||||
image: csi_images/driver-registrar:latest
|
image: docker.io/k8scsi/driver-registrar:latest
|
||||||
args:
|
args:
|
||||||
- "--v=5"
|
- "--v=5"
|
||||||
- "--csi-address=$(ADDRESS)"
|
- "--csi-address=$(ADDRESS)"
|
||||||
|
17
examples/pod.yaml
Normal file
17
examples/pod.yaml
Normal file
@ -0,0 +1,17 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: Pod
|
||||||
|
metadata:
|
||||||
|
name: web-server
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: web-server
|
||||||
|
image: nginx
|
||||||
|
volumeMounts:
|
||||||
|
- mountPath: /var/lib/www/html
|
||||||
|
name: mypvc
|
||||||
|
volumes:
|
||||||
|
- name: mypvc
|
||||||
|
persistentVolumeClaim:
|
||||||
|
claimName: rbd-pv
|
||||||
|
readOnly: false
|
||||||
|
|
11
examples/pvc.yaml
Normal file
11
examples/pvc.yaml
Normal file
@ -0,0 +1,11 @@
|
|||||||
|
kind: PersistentVolumeClaim
|
||||||
|
apiVersion: v1
|
||||||
|
metadata:
|
||||||
|
name: rbd-pv
|
||||||
|
spec:
|
||||||
|
accessModes:
|
||||||
|
- ReadWriteOnce
|
||||||
|
storageClassName: rbd-csi-provisioner
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
storage: 1Gi
|
12
examples/sc.yaml
Normal file
12
examples/sc.yaml
Normal file
@ -0,0 +1,12 @@
|
|||||||
|
apiVersion: storage.k8s.io/v1beta1
|
||||||
|
kind: StorageClass
|
||||||
|
metadata:
|
||||||
|
name: rbd-csi-provisioner
|
||||||
|
provisioner: csi-rbd
|
||||||
|
parameters:
|
||||||
|
monitors: 172.24.0.6:6789
|
||||||
|
pool: rbd
|
||||||
|
adminID: admin
|
||||||
|
adminSecret: AQAmsGBap4EoBhAAET/Hc7fBqAZj/cy7cDcoQA==
|
||||||
|
userID: kube
|
||||||
|
userSecret: AQAMgXhVwBCeDhAA9nlPaFyfUSatGD4drFWDvQ==
|
@ -24,8 +24,6 @@ import (
|
|||||||
"github.com/pborman/uuid"
|
"github.com/pborman/uuid"
|
||||||
"golang.org/x/net/context"
|
"golang.org/x/net/context"
|
||||||
|
|
||||||
"k8s.io/client-go/kubernetes"
|
|
||||||
|
|
||||||
"github.com/container-storage-interface/spec/lib/go/csi"
|
"github.com/container-storage-interface/spec/lib/go/csi"
|
||||||
"github.com/kubernetes-csi/drivers/pkg/csi-common"
|
"github.com/kubernetes-csi/drivers/pkg/csi-common"
|
||||||
)
|
)
|
||||||
@ -36,7 +34,6 @@ const (
|
|||||||
|
|
||||||
type controllerServer struct {
|
type controllerServer struct {
|
||||||
*csicommon.DefaultControllerServer
|
*csicommon.DefaultControllerServer
|
||||||
clientSet *kubernetes.Clientset
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func GetVersionString(ver *csi.Version) string {
|
func GetVersionString(ver *csi.Version) string {
|
||||||
@ -49,7 +46,7 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
volOptions, err := getRBDVolumeOptions(req.Parameters, cs.clientSet)
|
volOptions, err := getRBDVolumeOptions(req.GetParameters())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@ -101,6 +98,25 @@ func (cs *controllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVol
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// For now the image get unconditionally deleted, but here retention policy can be checked
|
||||||
|
volumeID := req.GetVolumeId()
|
||||||
|
volOptions := &rbdVolumeOptions{}
|
||||||
|
if err := loadVolInfo(volumeID, path.Join(PluginFolder, "controller"), volOptions); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
volName := volOptions.VolName
|
||||||
|
// Deleting rbd image
|
||||||
|
glog.V(4).Infof("deleting volume %s", volName)
|
||||||
|
if err := deleteRBDImage(volOptions); err != nil {
|
||||||
|
glog.V(3).Infof("failed to delete rbd image: %s/%s with error: %v", volOptions.Pool, volName, err)
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
// Removing persistent storage file for the unmapped volume
|
||||||
|
if err := deleteVolInfo(volumeID, path.Join(PluginFolder, "controller")); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
return &csi.DeleteVolumeResponse{}, nil
|
return &csi.DeleteVolumeResponse{}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -114,44 +130,9 @@ func (cs *controllerServer) ValidateVolumeCapabilities(ctx context.Context, req
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (cs *controllerServer) ControllerUnpublishVolume(ctx context.Context, req *csi.ControllerUnpublishVolumeRequest) (*csi.ControllerUnpublishVolumeResponse, error) {
|
func (cs *controllerServer) ControllerUnpublishVolume(ctx context.Context, req *csi.ControllerUnpublishVolumeRequest) (*csi.ControllerUnpublishVolumeResponse, error) {
|
||||||
|
|
||||||
// For now the image get unconditionally deleted, but here retention policy can be checked
|
|
||||||
volumeID := req.GetVolumeId()
|
|
||||||
volOptions := &rbdVolumeOptions{}
|
|
||||||
if err := loadVolInfo(volumeID, path.Join(PluginFolder, "controller"), volOptions); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
volName := volOptions.VolName
|
|
||||||
// Recover rbd secret key value, for now by k8s specific call
|
|
||||||
id := volOptions.AdminID
|
|
||||||
secretName := volOptions.AdminSecretName
|
|
||||||
secretNamespace := volOptions.AdminSecretNamespace
|
|
||||||
if id == "" {
|
|
||||||
secretName = volOptions.UserSecretName
|
|
||||||
secretNamespace = volOptions.UserSecretNamespace
|
|
||||||
}
|
|
||||||
if key, err := parseStorageClassSecret(secretName, secretNamespace, cs.clientSet); err != nil {
|
|
||||||
return nil, err
|
|
||||||
} else {
|
|
||||||
volOptions.adminSecret = key
|
|
||||||
}
|
|
||||||
|
|
||||||
// Deleting rbd image
|
|
||||||
glog.V(4).Infof("deleting volume %s", volName)
|
|
||||||
if err := deleteRBDImage(volOptions); err != nil {
|
|
||||||
glog.V(3).Infof("failed to delete rbd image: %s/%s with error: %v", volOptions.Pool, volName, err)
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
// Removing persistent storage file for the unmapped volume
|
|
||||||
if err := deleteVolInfo(volumeID, path.Join(PluginFolder, "controller")); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return &csi.ControllerUnpublishVolumeResponse{}, nil
|
return &csi.ControllerUnpublishVolumeResponse{}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (cs *controllerServer) ControllerPublishVolume(ctx context.Context, req *csi.ControllerPublishVolumeRequest) (*csi.ControllerPublishVolumeResponse, error) {
|
func (cs *controllerServer) ControllerPublishVolume(ctx context.Context, req *csi.ControllerPublishVolumeRequest) (*csi.ControllerPublishVolumeResponse, error) {
|
||||||
|
|
||||||
return &csi.ControllerPublishVolumeResponse{}, nil
|
return &csi.ControllerPublishVolumeResponse{}, nil
|
||||||
}
|
}
|
||||||
|
@ -28,7 +28,7 @@ import (
|
|||||||
"github.com/container-storage-interface/spec/lib/go/csi"
|
"github.com/container-storage-interface/spec/lib/go/csi"
|
||||||
"google.golang.org/grpc/codes"
|
"google.golang.org/grpc/codes"
|
||||||
"google.golang.org/grpc/status"
|
"google.golang.org/grpc/status"
|
||||||
"k8s.io/client-go/kubernetes"
|
|
||||||
"k8s.io/kubernetes/pkg/util/mount"
|
"k8s.io/kubernetes/pkg/util/mount"
|
||||||
|
|
||||||
"github.com/kubernetes-csi/drivers/pkg/csi-common"
|
"github.com/kubernetes-csi/drivers/pkg/csi-common"
|
||||||
@ -36,7 +36,6 @@ import (
|
|||||||
|
|
||||||
type nodeServer struct {
|
type nodeServer struct {
|
||||||
*csicommon.DefaultNodeServer
|
*csicommon.DefaultNodeServer
|
||||||
clientSet *kubernetes.Clientset
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) {
|
func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) {
|
||||||
@ -63,7 +62,7 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis
|
|||||||
if !notMnt {
|
if !notMnt {
|
||||||
return &csi.NodePublishVolumeResponse{}, nil
|
return &csi.NodePublishVolumeResponse{}, nil
|
||||||
}
|
}
|
||||||
volOptions, err := getRBDVolumeOptions(req.VolumeAttributes, ns.clientSet)
|
volOptions, err := getRBDVolumeOptions(req.VolumeAttributes)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@ -92,9 +91,6 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis
|
|||||||
if err := diskMounter.FormatAndMount(devicePath, targetPath, fsType, options); err != nil {
|
if err := diskMounter.FormatAndMount(devicePath, targetPath, fsType, options); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
// Storing rbd device path
|
|
||||||
|
|
||||||
volOptions.ImageMapping = map[string]string{volOptions.VolName: devicePath}
|
|
||||||
// Storing volInfo into a persistent file
|
// Storing volInfo into a persistent file
|
||||||
if err := persistVolInfo(req.GetVolumeId(), path.Join(PluginFolder, "node"), volOptions); err != nil {
|
if err := persistVolInfo(req.GetVolumeId(), path.Join(PluginFolder, "node"), volOptions); err != nil {
|
||||||
glog.Warningf("rbd: failed to store volInfo with error: %v", err)
|
glog.Warningf("rbd: failed to store volInfo with error: %v", err)
|
||||||
@ -112,20 +108,6 @@ func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu
|
|||||||
}
|
}
|
||||||
volName := volOptions.VolName
|
volName := volOptions.VolName
|
||||||
|
|
||||||
// Recover rbd secret key value, for now by k8s specific call
|
|
||||||
id := volOptions.AdminID
|
|
||||||
secretName := volOptions.AdminSecretName
|
|
||||||
secretNamespace := volOptions.AdminSecretNamespace
|
|
||||||
if id == "" {
|
|
||||||
secretName = volOptions.UserSecretName
|
|
||||||
secretNamespace = volOptions.UserSecretNamespace
|
|
||||||
}
|
|
||||||
if key, err := parseStorageClassSecret(secretName, secretNamespace, ns.clientSet); err != nil {
|
|
||||||
return nil, err
|
|
||||||
} else {
|
|
||||||
volOptions.adminSecret = key
|
|
||||||
}
|
|
||||||
|
|
||||||
notMnt, err := mount.New("").IsLikelyNotMountPoint(targetPath)
|
notMnt, err := mount.New("").IsLikelyNotMountPoint(targetPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, status.Error(codes.Internal, err.Error())
|
return nil, status.Error(codes.Internal, err.Error())
|
||||||
@ -141,7 +123,7 @@ func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu
|
|||||||
// Unmapping rbd device
|
// Unmapping rbd device
|
||||||
glog.V(4).Infof("deleting volume %s", volName)
|
glog.V(4).Infof("deleting volume %s", volName)
|
||||||
if err := detachRBDImage(volOptions); err != nil {
|
if err := detachRBDImage(volOptions); err != nil {
|
||||||
glog.V(3).Infof("failed to unmap rbd device: %s with error: %v", volOptions.ImageMapping[volName], err)
|
glog.V(3).Infof("failed to unmap rbd device: %s with error: %v", volOptions.VolName, err)
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
// Removing persistent storage file for the unmapped volume
|
// Removing persistent storage file for the unmapped volume
|
||||||
|
@ -17,11 +17,10 @@ limitations under the License.
|
|||||||
package rbd
|
package rbd
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"github.com/container-storage-interface/spec/lib/go/csi"
|
|
||||||
"github.com/golang/glog"
|
"github.com/golang/glog"
|
||||||
|
|
||||||
|
"github.com/container-storage-interface/spec/lib/go/csi"
|
||||||
"github.com/kubernetes-csi/drivers/pkg/csi-common"
|
"github.com/kubernetes-csi/drivers/pkg/csi-common"
|
||||||
"k8s.io/client-go/kubernetes"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// PluginFolder defines the location of rbdplugin
|
// PluginFolder defines the location of rbdplugin
|
||||||
@ -61,21 +60,19 @@ func NewIdentityServer(d *csicommon.CSIDriver) *identityServer {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewControllerServer(d *csicommon.CSIDriver, clientSet *kubernetes.Clientset) *controllerServer {
|
func NewControllerServer(d *csicommon.CSIDriver) *controllerServer {
|
||||||
return &controllerServer{
|
return &controllerServer{
|
||||||
DefaultControllerServer: csicommon.NewDefaultControllerServer(d),
|
DefaultControllerServer: csicommon.NewDefaultControllerServer(d),
|
||||||
clientSet: clientSet,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewNodeServer(d *csicommon.CSIDriver, clientSet *kubernetes.Clientset) *nodeServer {
|
func NewNodeServer(d *csicommon.CSIDriver) *nodeServer {
|
||||||
return &nodeServer{
|
return &nodeServer{
|
||||||
DefaultNodeServer: csicommon.NewDefaultNodeServer(d),
|
DefaultNodeServer: csicommon.NewDefaultNodeServer(d),
|
||||||
clientSet: clientSet,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (rbd *rbd) Run(driverName, nodeID, endpoint string, clientSet *kubernetes.Clientset) {
|
func (rbd *rbd) Run(driverName, nodeID, endpoint string) {
|
||||||
glog.Infof("Driver: %v version: %v", driverName, GetVersionString(&version))
|
glog.Infof("Driver: %v version: %v", driverName, GetVersionString(&version))
|
||||||
|
|
||||||
// Initialize default library driver
|
// Initialize default library driver
|
||||||
@ -91,8 +88,8 @@ func (rbd *rbd) Run(driverName, nodeID, endpoint string, clientSet *kubernetes.C
|
|||||||
|
|
||||||
// Create GRPC servers
|
// Create GRPC servers
|
||||||
rbd.ids = NewIdentityServer(rbd.driver)
|
rbd.ids = NewIdentityServer(rbd.driver)
|
||||||
rbd.ns = NewNodeServer(rbd.driver, clientSet)
|
rbd.ns = NewNodeServer(rbd.driver)
|
||||||
rbd.cs = NewControllerServer(rbd.driver, clientSet)
|
rbd.cs = NewControllerServer(rbd.driver)
|
||||||
s := csicommon.NewNonBlockingGRPCServer()
|
s := csicommon.NewNonBlockingGRPCServer()
|
||||||
s.Start(endpoint, rbd.ids, rbd.cs, rbd.ns)
|
s.Start(endpoint, rbd.ids, rbd.cs, rbd.ns)
|
||||||
s.Wait()
|
s.Wait()
|
||||||
|
@ -21,9 +21,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"github.com/golang/glog"
|
"github.com/golang/glog"
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
||||||
"k8s.io/apimachinery/pkg/util/wait"
|
"k8s.io/apimachinery/pkg/util/wait"
|
||||||
"k8s.io/client-go/kubernetes"
|
|
||||||
"k8s.io/kubernetes/pkg/util/keymutex"
|
"k8s.io/kubernetes/pkg/util/keymutex"
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
@ -50,17 +48,12 @@ type rbdVolumeOptions struct {
|
|||||||
VolName string `json:"volName"`
|
VolName string `json:"volName"`
|
||||||
Monitors string `json:"monitors"`
|
Monitors string `json:"monitors"`
|
||||||
Pool string `json:"pool"`
|
Pool string `json:"pool"`
|
||||||
AdminSecretName string `json:"adminSecret"`
|
|
||||||
AdminSecretNamespace string `json:"adminSecretNamespace"`
|
|
||||||
AdminID string `json:"adminID"`
|
AdminID string `json:"adminID"`
|
||||||
|
AdminSecret string `json:"adminSecret"`
|
||||||
UserID string `json:"userID"`
|
UserID string `json:"userID"`
|
||||||
UserSecretName string `json:"userSecret"`
|
UserSecret string `json:"userSecret"`
|
||||||
UserSecretNamespace string `json:"userSecretNamespace"`
|
|
||||||
ImageFormat string `json:"imageFormat"`
|
ImageFormat string `json:"imageFormat"`
|
||||||
ImageFeatures []string `json:"imageFeatures"`
|
ImageFeatures []string `json:"imageFeatures"`
|
||||||
ImageMapping map[string]string `json:"imageMapping"`
|
|
||||||
adminSecret string
|
|
||||||
userSecret string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
var attachdetachMutex = keymutex.NewKeyMutex()
|
var attachdetachMutex = keymutex.NewKeyMutex()
|
||||||
@ -76,11 +69,11 @@ func createRBDImage(pOpts *rbdVolumeOptions, volSz int) error {
|
|||||||
volSzGB := fmt.Sprintf("%dG", volSz)
|
volSzGB := fmt.Sprintf("%dG", volSz)
|
||||||
|
|
||||||
if pOpts.ImageFormat == rbdImageFormat2 {
|
if pOpts.ImageFormat == rbdImageFormat2 {
|
||||||
glog.V(4).Infof("rbd: create %s size %s format %s (features: %s) using mon %s, pool %s id %s key %s", image, volSzGB, pOpts.ImageFormat, pOpts.ImageFeatures, mon, pOpts.Pool, pOpts.AdminID, pOpts.adminSecret)
|
glog.V(4).Infof("rbd: create %s size %s format %s (features: %s) using mon %s, pool %s id %s key %s", image, volSzGB, pOpts.ImageFormat, pOpts.ImageFeatures, mon, pOpts.Pool, pOpts.AdminID, pOpts.AdminSecret)
|
||||||
} else {
|
} else {
|
||||||
glog.V(4).Infof("rbd: create %s size %s format %s using mon %s, pool %s id %s key %s", image, volSzGB, pOpts.ImageFormat, mon, pOpts.Pool, pOpts.AdminID, pOpts.adminSecret)
|
glog.V(4).Infof("rbd: create %s size %s format %s using mon %s, pool %s id %s key %s", image, volSzGB, pOpts.ImageFormat, mon, pOpts.Pool, pOpts.AdminID, pOpts.AdminSecret)
|
||||||
}
|
}
|
||||||
args := []string{"create", image, "--size", volSzGB, "--pool", pOpts.Pool, "--id", pOpts.AdminID, "-m", mon, "--key=" + pOpts.adminSecret, "--image-format", pOpts.ImageFormat}
|
args := []string{"create", image, "--size", volSzGB, "--pool", pOpts.Pool, "--id", pOpts.AdminID, "-m", mon, "--key=" + pOpts.AdminSecret, "--image-format", pOpts.ImageFormat}
|
||||||
if pOpts.ImageFormat == rbdImageFormat2 {
|
if pOpts.ImageFormat == rbdImageFormat2 {
|
||||||
// if no image features is provided, it results in empty string
|
// if no image features is provided, it results in empty string
|
||||||
// which disable all RBD image format 2 features as we expected
|
// which disable all RBD image format 2 features as we expected
|
||||||
@ -106,10 +99,10 @@ func rbdStatus(b *rbdVolumeOptions) (bool, string, error) {
|
|||||||
image := b.VolName
|
image := b.VolName
|
||||||
// If we don't have admin id/secret (e.g. attaching), fallback to user id/secret.
|
// If we don't have admin id/secret (e.g. attaching), fallback to user id/secret.
|
||||||
id := b.AdminID
|
id := b.AdminID
|
||||||
secret := b.adminSecret
|
secret := b.AdminSecret
|
||||||
if id == "" {
|
if id == "" {
|
||||||
id = b.UserID
|
id = b.UserID
|
||||||
secret = b.userSecret
|
secret = b.UserSecret
|
||||||
}
|
}
|
||||||
|
|
||||||
glog.V(4).Infof("rbd: status %s using mon %s, pool %s id %s key %s", image, b.Monitors, b.Pool, id, secret)
|
glog.V(4).Infof("rbd: status %s using mon %s, pool %s id %s key %s", image, b.Monitors, b.Pool, id, secret)
|
||||||
@ -152,10 +145,10 @@ func deleteRBDImage(b *rbdVolumeOptions) error {
|
|||||||
return fmt.Errorf("rbd %s is still being used", image)
|
return fmt.Errorf("rbd %s is still being used", image)
|
||||||
}
|
}
|
||||||
id := b.AdminID
|
id := b.AdminID
|
||||||
secret := b.adminSecret
|
secret := b.AdminSecret
|
||||||
if id == "" {
|
if id == "" {
|
||||||
id = b.UserID
|
id = b.UserID
|
||||||
secret = b.userSecret
|
secret = b.UserSecret
|
||||||
}
|
}
|
||||||
|
|
||||||
glog.V(4).Infof("rbd: rm %s using mon %s, pool %s id %s key %s", image, b.Monitors, b.Pool, id, secret)
|
glog.V(4).Infof("rbd: rm %s using mon %s, pool %s id %s key %s", image, b.Monitors, b.Pool, id, secret)
|
||||||
@ -173,25 +166,16 @@ func execCommand(command string, args []string) ([]byte, error) {
|
|||||||
return cmd.CombinedOutput()
|
return cmd.CombinedOutput()
|
||||||
}
|
}
|
||||||
|
|
||||||
func getRBDVolumeOptions(volOptions map[string]string, client *kubernetes.Clientset) (*rbdVolumeOptions, error) {
|
func getRBDVolumeOptions(volOptions map[string]string) (*rbdVolumeOptions, error) {
|
||||||
rbdVolume := &rbdVolumeOptions{}
|
rbdVolume := &rbdVolumeOptions{}
|
||||||
var ok bool
|
var ok bool
|
||||||
var err error
|
rbdVolume.AdminID, ok = volOptions["adminID"]
|
||||||
rbdVolume.AdminID, ok = volOptions["adminId"]
|
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil, fmt.Errorf("Missing required parameter adminId")
|
return nil, fmt.Errorf("Missing required parameter adminID")
|
||||||
}
|
}
|
||||||
rbdVolume.AdminSecretName, ok = volOptions["adminSecretName"]
|
rbdVolume.AdminSecret, ok = volOptions["adminSecret"]
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil, fmt.Errorf("Missing required parameter adminSecretName")
|
return nil, fmt.Errorf("Missing required parameter adminSecret")
|
||||||
}
|
|
||||||
rbdVolume.AdminSecretNamespace, ok = volOptions["adminSecretNamespace"]
|
|
||||||
if !ok {
|
|
||||||
rbdVolume.AdminSecretNamespace = "default"
|
|
||||||
}
|
|
||||||
rbdVolume.adminSecret, err = parseStorageClassSecret(rbdVolume.AdminSecretName, rbdVolume.AdminSecretNamespace, client)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("Failed to retrieve Admin secret %v", err)
|
|
||||||
}
|
}
|
||||||
rbdVolume.Pool, ok = volOptions["pool"]
|
rbdVolume.Pool, ok = volOptions["pool"]
|
||||||
if !ok {
|
if !ok {
|
||||||
@ -201,23 +185,13 @@ func getRBDVolumeOptions(volOptions map[string]string, client *kubernetes.Client
|
|||||||
if !ok {
|
if !ok {
|
||||||
return nil, fmt.Errorf("Missing required parameter monitors")
|
return nil, fmt.Errorf("Missing required parameter monitors")
|
||||||
}
|
}
|
||||||
if err != nil {
|
rbdVolume.UserID, ok = volOptions["userID"]
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
rbdVolume.UserID, ok = volOptions["userId"]
|
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil, fmt.Errorf("Missing required parameter userId")
|
return nil, fmt.Errorf("Missing required parameter userID")
|
||||||
}
|
}
|
||||||
rbdVolume.UserSecretName, ok = volOptions["userSecretName"]
|
rbdVolume.UserSecret, ok = volOptions["userSecret"]
|
||||||
if ok {
|
|
||||||
rbdVolume.UserSecretNamespace, ok = volOptions["userSecretNamespace"]
|
|
||||||
if !ok {
|
if !ok {
|
||||||
rbdVolume.UserSecretNamespace = "default"
|
return nil, fmt.Errorf("Missing required parameter userSecret")
|
||||||
}
|
|
||||||
rbdVolume.userSecret, err = parseStorageClassSecret(rbdVolume.UserSecretName, rbdVolume.UserSecretNamespace, client)
|
|
||||||
if err != nil {
|
|
||||||
glog.Errorf("failed to retrieve user's secret: %s/%s (%v)", rbdVolume.UserSecretName, rbdVolume.UserSecretNamespace, err)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
rbdVolume.ImageFormat, ok = volOptions["imageFormat"]
|
rbdVolume.ImageFormat, ok = volOptions["imageFormat"]
|
||||||
if !ok {
|
if !ok {
|
||||||
@ -227,25 +201,6 @@ func getRBDVolumeOptions(volOptions map[string]string, client *kubernetes.Client
|
|||||||
return rbdVolume, nil
|
return rbdVolume, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func parseStorageClassSecret(secretName string, namespace string, client *kubernetes.Clientset) (string, error) {
|
|
||||||
if client == nil {
|
|
||||||
return "", fmt.Errorf("Cannot get kube client")
|
|
||||||
}
|
|
||||||
secrets, err := client.CoreV1().Secrets(namespace).Get(secretName, metav1.GetOptions{})
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
secret := ""
|
|
||||||
for k, v := range secrets.Data {
|
|
||||||
if k == secretName {
|
|
||||||
return string(v), nil
|
|
||||||
}
|
|
||||||
secret = string(v)
|
|
||||||
}
|
|
||||||
|
|
||||||
return secret, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func attachRBDImage(volOptions *rbdVolumeOptions) (string, error) {
|
func attachRBDImage(volOptions *rbdVolumeOptions) (string, error) {
|
||||||
var err error
|
var err error
|
||||||
var output []byte
|
var output []byte
|
||||||
@ -283,13 +238,8 @@ func attachRBDImage(volOptions *rbdVolumeOptions) (string, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
glog.V(1).Infof("rbd: map mon %s", volOptions.Monitors)
|
glog.V(1).Infof("rbd: map mon %s", volOptions.Monitors)
|
||||||
// If we don't have admin id/secret (e.g. attaching), fallback to user id/secret.
|
id := volOptions.UserID
|
||||||
id := volOptions.AdminID
|
secret := volOptions.UserSecret
|
||||||
secret := volOptions.adminSecret
|
|
||||||
if id == "" {
|
|
||||||
id = volOptions.UserID
|
|
||||||
secret = volOptions.userSecret
|
|
||||||
}
|
|
||||||
|
|
||||||
output, err = execCommand("rbd", []string{
|
output, err = execCommand("rbd", []string{
|
||||||
"map", image, "--pool", volOptions.Pool, "--id", id, "-m", volOptions.Monitors, "--key=" + secret})
|
"map", image, "--pool", volOptions.Pool, "--id", id, "-m", volOptions.Monitors, "--key=" + secret})
|
||||||
@ -311,17 +261,12 @@ func detachRBDImage(volOptions *rbdVolumeOptions) error {
|
|||||||
var output []byte
|
var output []byte
|
||||||
|
|
||||||
image := volOptions.VolName
|
image := volOptions.VolName
|
||||||
glog.V(1).Infof("rbd: unmap device %s", volOptions.ImageMapping[image])
|
glog.V(1).Infof("rbd: unmap device %s", image)
|
||||||
// If we don't have admin id/secret (e.g. attaching), fallback to user id/secret.
|
id := volOptions.UserID
|
||||||
id := volOptions.AdminID
|
secret := volOptions.UserSecret
|
||||||
secret := volOptions.adminSecret
|
|
||||||
if id == "" {
|
|
||||||
id = volOptions.UserID
|
|
||||||
secret = volOptions.userSecret
|
|
||||||
}
|
|
||||||
|
|
||||||
output, err = execCommand("rbd", []string{
|
output, err = execCommand("rbd", []string{
|
||||||
"unmap", volOptions.ImageMapping[image], "--id", id, "--key=" + secret})
|
"unmap", image, "--id", id, "--key=" + secret})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
glog.V(1).Infof("rbd: unmap error %v, rbd output: %s", err, string(output))
|
glog.V(1).Infof("rbd: unmap error %v, rbd output: %s", err, string(output))
|
||||||
return fmt.Errorf("rbd: unmap failed %v, rbd output: %s", err, string(output))
|
return fmt.Errorf("rbd: unmap failed %v, rbd output: %s", err, string(output))
|
||||||
|
21
rbd/main.go
21
rbd/main.go
@ -18,13 +18,11 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"flag"
|
"flag"
|
||||||
"github.com/golang/glog"
|
|
||||||
"os"
|
"os"
|
||||||
"path"
|
"path"
|
||||||
|
|
||||||
"github.com/ceph/ceph-csi/pkg/rbd"
|
"github.com/ceph/ceph-csi/pkg/rbd"
|
||||||
"k8s.io/client-go/kubernetes"
|
"github.com/golang/glog"
|
||||||
"k8s.io/client-go/rest"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
@ -40,17 +38,6 @@ var (
|
|||||||
func main() {
|
func main() {
|
||||||
flag.Parse()
|
flag.Parse()
|
||||||
|
|
||||||
// creates the in-cluster config
|
|
||||||
config, err := rest.InClusterConfig()
|
|
||||||
if err != nil {
|
|
||||||
panic(err.Error())
|
|
||||||
}
|
|
||||||
// creates the clientset
|
|
||||||
clientSet, err := kubernetes.NewForConfig(config)
|
|
||||||
if err != nil {
|
|
||||||
panic(err.Error())
|
|
||||||
}
|
|
||||||
|
|
||||||
if err := createPersistentStorage(path.Join(rbd.PluginFolder, "controller")); err != nil {
|
if err := createPersistentStorage(path.Join(rbd.PluginFolder, "controller")); err != nil {
|
||||||
glog.Errorf("failed to create persistent storage for controller %v", err)
|
glog.Errorf("failed to create persistent storage for controller %v", err)
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
@ -60,13 +47,13 @@ func main() {
|
|||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
|
|
||||||
handle(clientSet)
|
handle()
|
||||||
os.Exit(0)
|
os.Exit(0)
|
||||||
}
|
}
|
||||||
|
|
||||||
func handle(clientSet *kubernetes.Clientset) {
|
func handle() {
|
||||||
driver := rbd.GetRBDDriver()
|
driver := rbd.GetRBDDriver()
|
||||||
driver.Run(*driverName, *nodeID, *endpoint, clientSet)
|
driver.Run(*driverName, *nodeID, *endpoint)
|
||||||
}
|
}
|
||||||
|
|
||||||
func createPersistentStorage(persistentStoragePath string) error {
|
func createPersistentStorage(persistentStoragePath string) error {
|
||||||
|
Loading…
Reference in New Issue
Block a user