mirror of
https://github.com/ceph/ceph-csi.git
synced 2025-06-13 02:33:34 +00:00
just code files
This commit is contained in:
Serguei Bezverkhi
152
pkg/rbd/controllerserver.go
Normal file
152
pkg/rbd/controllerserver.go
Normal file
@ -0,0 +1,152 @@
|
||||
/*
|
||||
Copyright 2018 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package rbd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"path"
|
||||
|
||||
"github.com/golang/glog"
|
||||
"github.com/pborman/uuid"
|
||||
"golang.org/x/net/context"
|
||||
|
||||
"k8s.io/client-go/kubernetes"
|
||||
|
||||
"github.com/container-storage-interface/spec/lib/go/csi"
|
||||
"github.com/kubernetes-csi/drivers/pkg/csi-common"
|
||||
)
|
||||
|
||||
const (
|
||||
oneGB = 1073741824
|
||||
)
|
||||
|
||||
type controllerServer struct {
|
||||
*csicommon.DefaultControllerServer
|
||||
clientSet *kubernetes.Clientset
|
||||
}
|
||||
|
||||
func GetVersionString(ver *csi.Version) string {
|
||||
return fmt.Sprintf("%d.%d.%d", ver.Major, ver.Minor, ver.Patch)
|
||||
}
|
||||
|
||||
func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) {
|
||||
if err := cs.Driver.ValidateControllerServiceRequest(req.Version, csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME); err != nil {
|
||||
glog.V(3).Infof("invalid create volume req: %v", req)
|
||||
return nil, err
|
||||
}
|
||||
// Volume Name
|
||||
volName := req.GetName()
|
||||
if len(volName) == 0 {
|
||||
volName = uuid.NewUUID().String()
|
||||
}
|
||||
|
||||
// Volume Size - Default is 1 GiB
|
||||
volSizeBytes := int64(oneGB)
|
||||
if req.GetCapacityRange() != nil {
|
||||
volSizeBytes = int64(req.GetCapacityRange().GetRequiredBytes())
|
||||
}
|
||||
volSizeGB := int(volSizeBytes / 1024 / 1024 / 1024)
|
||||
|
||||
volOptions, err := getRBDVolumeOptions(req.Parameters, cs.clientSet)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Check if there is already RBD image with requested name
|
||||
found, _, _ := rbdStatus(volName, volOptions)
|
||||
if !found {
|
||||
if err := createRBDImage(volName, volSizeGB, volOptions); err != nil {
|
||||
if err != nil {
|
||||
glog.Warningf("failed to create volume: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
glog.V(4).Infof("create volume %s", volName)
|
||||
}
|
||||
// Storing volInfo into a persistent file, will need info to delete rbd image
|
||||
// in ControllerUnpublishVolume
|
||||
if err := persistVolInfo(volName, path.Join(PluginFolder, "controller"), volOptions); err != nil {
|
||||
glog.Warningf("rbd: failed to store volInfo with error: %v", err)
|
||||
}
|
||||
|
||||
return &csi.CreateVolumeResponse{
|
||||
VolumeInfo: &csi.VolumeInfo{
|
||||
Id: volName,
|
||||
CapacityBytes: uint64(volSizeBytes),
|
||||
Attributes: req.GetParameters(),
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (cs *controllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) {
|
||||
if err := cs.Driver.ValidateControllerServiceRequest(req.Version, csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME); err != nil {
|
||||
glog.Warningf("invalid delete volume req: %v", req)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &csi.DeleteVolumeResponse{}, nil
|
||||
}
|
||||
|
||||
func (cs *controllerServer) ValidateVolumeCapabilities(ctx context.Context, req *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error) {
|
||||
for _, cap := range req.VolumeCapabilities {
|
||||
if cap.GetAccessMode().GetMode() != csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER {
|
||||
return &csi.ValidateVolumeCapabilitiesResponse{Supported: false, Message: ""}, nil
|
||||
}
|
||||
}
|
||||
return &csi.ValidateVolumeCapabilitiesResponse{Supported: true, Message: ""}, nil
|
||||
}
|
||||
|
||||
func (cs *controllerServer) ControllerUnpublishVolume(ctx context.Context, req *csi.ControllerUnpublishVolumeRequest) (*csi.ControllerUnpublishVolumeResponse, error) {
|
||||
|
||||
// For now the image get unconditionally deleted, but here retention policy can be checked
|
||||
volName := req.GetVolumeId()
|
||||
volOptions := &rbdVolumeOptions{}
|
||||
if err := loadVolInfo(volName, path.Join(PluginFolder, "controller"), volOptions); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Recover rbd secret key value, for now by k8s specific call
|
||||
id := volOptions.AdminID
|
||||
secretName := volOptions.AdminSecretName
|
||||
secretNamespace := volOptions.AdminSecretNamespace
|
||||
if id == "" {
|
||||
secretName = volOptions.UserSecretName
|
||||
secretNamespace = volOptions.UserSecretNamespace
|
||||
}
|
||||
if key, err := parseStorageClassSecret(secretName, secretNamespace, cs.clientSet); err != nil {
|
||||
return nil, err
|
||||
} else {
|
||||
volOptions.adminSecret = key
|
||||
}
|
||||
|
||||
// Deleting rbd image
|
||||
glog.V(4).Infof("deleting volume %s", volName)
|
||||
if err := deleteRBDImage(volName, volOptions); err != nil {
|
||||
glog.V(3).Infof("failed to delete rbd image: %s/%s with error: %v", volOptions.Pool, volName, err)
|
||||
return nil, err
|
||||
}
|
||||
// Removing persistent storage file for the unmapped volume
|
||||
if err := deleteVolInfo(volName, path.Join(PluginFolder, "controller")); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &csi.ControllerUnpublishVolumeResponse{}, nil
|
||||
}
|
||||
|
||||
func (cs *controllerServer) ControllerPublishVolume(ctx context.Context, req *csi.ControllerPublishVolumeRequest) (*csi.ControllerPublishVolumeResponse, error) {
|
||||
|
||||
return &csi.ControllerPublishVolumeResponse{}, nil
|
||||
}
|
||||
25
pkg/rbd/identityserver.go
Normal file
25
pkg/rbd/identityserver.go
Normal file
@ -0,0 +1,25 @@
|
||||
/*
|
||||
Copyright 2018 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package rbd
|
||||
|
||||
import (
|
||||
"github.com/kubernetes-csi/drivers/pkg/csi-common"
|
||||
)
|
||||
|
||||
type identityServer struct {
|
||||
*csicommon.DefaultIdentityServer
|
||||
}
|
||||
144
pkg/rbd/nodeserver.go
Normal file
144
pkg/rbd/nodeserver.go
Normal file
@ -0,0 +1,144 @@
|
||||
/*
|
||||
Copyright 2018 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package rbd
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path"
|
||||
|
||||
"github.com/golang/glog"
|
||||
"golang.org/x/net/context"
|
||||
|
||||
"github.com/container-storage-interface/spec/lib/go/csi"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/kubernetes/pkg/util/mount"
|
||||
|
||||
"github.com/kubernetes-csi/drivers/pkg/csi-common"
|
||||
)
|
||||
|
||||
type nodeServer struct {
|
||||
*csicommon.DefaultNodeServer
|
||||
clientSet *kubernetes.Clientset
|
||||
}
|
||||
|
||||
func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) {
|
||||
targetPath := req.GetTargetPath()
|
||||
|
||||
notMnt, err := mount.New("").IsLikelyNotMountPoint(targetPath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
if err = os.MkdirAll(targetPath, 0750); err != nil {
|
||||
return nil, status.Error(codes.Internal, err.Error())
|
||||
}
|
||||
notMnt = true
|
||||
} else {
|
||||
return nil, status.Error(codes.Internal, err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
if !notMnt {
|
||||
return &csi.NodePublishVolumeResponse{}, nil
|
||||
}
|
||||
volOptions, err := getRBDVolumeOptions(req.VolumeAttributes, ns.clientSet)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Mapping RBD image
|
||||
devicePath, err := attachRBDImage(req.GetVolumeId(), volOptions)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
glog.V(4).Infof("rbd image: %s/%s was succesfully mapped at %s\n", req.GetVolumeId(), volOptions.Pool, devicePath)
|
||||
fsType := req.GetVolumeCapability().GetMount().GetFsType()
|
||||
|
||||
readOnly := req.GetReadonly()
|
||||
attrib := req.GetVolumeAttributes()
|
||||
mountFlags := req.GetVolumeCapability().GetMount().GetMountFlags()
|
||||
|
||||
glog.V(4).Infof("target %v\nfstype %v\ndevice %v\nreadonly %v\nattributes %v\n mountflags %v\n",
|
||||
targetPath, fsType, devicePath, readOnly, attrib, mountFlags)
|
||||
|
||||
options := []string{}
|
||||
if readOnly {
|
||||
options = append(options, "ro")
|
||||
}
|
||||
|
||||
diskMounter := &mount.SafeFormatAndMount{Interface: mount.New(""), Exec: mount.NewOsExec()}
|
||||
if err := diskMounter.FormatAndMount(devicePath, targetPath, fsType, options); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Storing rbd device path
|
||||
|
||||
volOptions.ImageMapping = map[string]string{req.GetVolumeId(): devicePath}
|
||||
// Storing volInfo into a persistent file
|
||||
if err := persistVolInfo(req.GetVolumeId(), path.Join(PluginFolder, "node"), volOptions); err != nil {
|
||||
glog.Warningf("rbd: failed to store volInfo with error: %v", err)
|
||||
}
|
||||
|
||||
return &csi.NodePublishVolumeResponse{}, nil
|
||||
}
|
||||
|
||||
func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) {
|
||||
targetPath := req.GetTargetPath()
|
||||
volName := req.GetVolumeId()
|
||||
volOptions := &rbdVolumeOptions{}
|
||||
if err := loadVolInfo(volName, path.Join(PluginFolder, "node"), volOptions); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Recover rbd secret key value, for now by k8s specific call
|
||||
id := volOptions.AdminID
|
||||
secretName := volOptions.AdminSecretName
|
||||
secretNamespace := volOptions.AdminSecretNamespace
|
||||
if id == "" {
|
||||
secretName = volOptions.UserSecretName
|
||||
secretNamespace = volOptions.UserSecretNamespace
|
||||
}
|
||||
if key, err := parseStorageClassSecret(secretName, secretNamespace, ns.clientSet); err != nil {
|
||||
return nil, err
|
||||
} else {
|
||||
volOptions.adminSecret = key
|
||||
}
|
||||
|
||||
notMnt, err := mount.New("").IsLikelyNotMountPoint(targetPath)
|
||||
if err != nil {
|
||||
return nil, status.Error(codes.Internal, err.Error())
|
||||
}
|
||||
if notMnt {
|
||||
return nil, status.Error(codes.NotFound, "Volume not mounted")
|
||||
}
|
||||
// Unmounting the image
|
||||
err = mount.New("").Unmount(req.GetTargetPath())
|
||||
if err != nil {
|
||||
return nil, status.Error(codes.Internal, err.Error())
|
||||
}
|
||||
// Unmapping rbd device
|
||||
glog.V(4).Infof("deleting volume %s", volName)
|
||||
if err := detachRBDImage(volName, volOptions); err != nil {
|
||||
glog.V(3).Infof("failed to unmap rbd device: %s with error: %v", volOptions.ImageMapping[volName], err)
|
||||
return nil, err
|
||||
}
|
||||
// Removing persistent storage file for the unmapped volume
|
||||
if err := deleteVolInfo(volName, path.Join(PluginFolder, "node")); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &csi.NodeUnpublishVolumeResponse{}, nil
|
||||
}
|
||||
99
pkg/rbd/rbd.go
Normal file
99
pkg/rbd/rbd.go
Normal file
@ -0,0 +1,99 @@
|
||||
/*
|
||||
Copyright 2018 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package rbd
|
||||
|
||||
import (
|
||||
"github.com/container-storage-interface/spec/lib/go/csi"
|
||||
"github.com/golang/glog"
|
||||
|
||||
"github.com/kubernetes-csi/drivers/pkg/csi-common"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
)
|
||||
|
||||
// PluginFolder defines the location of rbdplugin
|
||||
const (
|
||||
PluginFolder = "/var/lib/kubelet/plugins/rbdplugin"
|
||||
)
|
||||
|
||||
type rbd struct {
|
||||
driver *csicommon.CSIDriver
|
||||
|
||||
ids *identityServer
|
||||
ns *nodeServer
|
||||
cs *controllerServer
|
||||
|
||||
cap []*csi.VolumeCapability_AccessMode
|
||||
cscap []*csi.ControllerServiceCapability
|
||||
}
|
||||
|
||||
var (
|
||||
rbdDriver *rbd
|
||||
version = csi.Version{
|
||||
Minor: 1,
|
||||
}
|
||||
)
|
||||
|
||||
func GetSupportedVersions() []*csi.Version {
|
||||
return []*csi.Version{&version}
|
||||
}
|
||||
|
||||
func GetRBDDriver() *rbd {
|
||||
return &rbd{}
|
||||
}
|
||||
|
||||
func NewIdentityServer(d *csicommon.CSIDriver) *identityServer {
|
||||
return &identityServer{
|
||||
DefaultIdentityServer: csicommon.NewDefaultIdentityServer(d),
|
||||
}
|
||||
}
|
||||
|
||||
func NewControllerServer(d *csicommon.CSIDriver, clientSet *kubernetes.Clientset) *controllerServer {
|
||||
return &controllerServer{
|
||||
DefaultControllerServer: csicommon.NewDefaultControllerServer(d),
|
||||
clientSet: clientSet,
|
||||
}
|
||||
}
|
||||
|
||||
func NewNodeServer(d *csicommon.CSIDriver, clientSet *kubernetes.Clientset) *nodeServer {
|
||||
return &nodeServer{
|
||||
DefaultNodeServer: csicommon.NewDefaultNodeServer(d),
|
||||
clientSet: clientSet,
|
||||
}
|
||||
}
|
||||
|
||||
func (rbd *rbd) Run(driverName, nodeID, endpoint string, clientSet *kubernetes.Clientset) {
|
||||
glog.Infof("Driver: %v version: %v", driverName, GetVersionString(&version))
|
||||
|
||||
// Initialize default library driver
|
||||
rbd.driver = csicommon.NewCSIDriver(driverName, &version, GetSupportedVersions(), nodeID)
|
||||
if rbd.driver == nil {
|
||||
glog.Fatalln("Failed to initialize CSI Driver.")
|
||||
}
|
||||
rbd.driver.AddControllerServiceCapabilities([]csi.ControllerServiceCapability_RPC_Type{
|
||||
csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME,
|
||||
csi.ControllerServiceCapability_RPC_PUBLISH_UNPUBLISH_VOLUME,
|
||||
})
|
||||
rbd.driver.AddVolumeCapabilityAccessModes([]csi.VolumeCapability_AccessMode_Mode{csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER})
|
||||
|
||||
// Create GRPC servers
|
||||
rbd.ids = NewIdentityServer(rbd.driver)
|
||||
rbd.ns = NewNodeServer(rbd.driver, clientSet)
|
||||
rbd.cs = NewControllerServer(rbd.driver, clientSet)
|
||||
s := csicommon.NewNonBlockingGRPCServer()
|
||||
s.Start(endpoint, rbd.ids, rbd.cs, rbd.ns)
|
||||
s.Wait()
|
||||
}
|
||||
420
pkg/rbd/rbd_util.go
Normal file
420
pkg/rbd/rbd_util.go
Normal file
@ -0,0 +1,420 @@
|
||||
/*
|
||||
Copyright 2018 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package rbd
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/golang/glog"
|
||||
"io/ioutil"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/kubernetes/pkg/util/keymutex"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
imageWatcherStr = "watcher="
|
||||
rbdImageFormat1 = "1"
|
||||
rbdImageFormat2 = "2"
|
||||
imageSizeStr = "size "
|
||||
sizeDivStr = " MB in"
|
||||
kubeLockMagic = "kubelet_lock_magic_"
|
||||
// The following three values are used for 30 seconds timeout
|
||||
// while waiting for RBD Watcher to expire.
|
||||
rbdImageWatcherInitDelay = 1 * time.Second
|
||||
rbdImageWatcherFactor = 1.4
|
||||
rbdImageWatcherSteps = 10
|
||||
)
|
||||
|
||||
type rbdVolumeOptions struct {
|
||||
Monitors string `json:"monitors"`
|
||||
Pool string `json:"pool"`
|
||||
AdminSecretName string `json:"adminSecret"`
|
||||
AdminSecretNamespace string `json:"adminSecretNamespace"`
|
||||
AdminID string `json:"adminID"`
|
||||
UserID string `json:"userID"`
|
||||
UserSecretName string `json:"userSecret"`
|
||||
UserSecretNamespace string `json:"userSecretNamespace"`
|
||||
ImageFormat string `json:"imageFormat"`
|
||||
ImageFeatures []string `json:"imageFeatures"`
|
||||
ImageMapping map[string]string `json:"imageMapping"`
|
||||
adminSecret string
|
||||
userSecret string
|
||||
}
|
||||
|
||||
var attachdetachMutex = keymutex.NewKeyMutex()
|
||||
|
||||
// CreateImage creates a new ceph image with provision and volume options.
|
||||
func createRBDImage(image string, volSz int, pOpts *rbdVolumeOptions) error {
|
||||
var output []byte
|
||||
var err error
|
||||
|
||||
// rbd create
|
||||
mon := pOpts.Monitors
|
||||
volSzGB := fmt.Sprintf("%dG", volSz)
|
||||
|
||||
if pOpts.ImageFormat == rbdImageFormat2 {
|
||||
glog.V(4).Infof("rbd: create %s size %s format %s (features: %s) using mon %s, pool %s id %s key %s", image, volSzGB, pOpts.ImageFormat, pOpts.ImageFeatures, mon, pOpts.Pool, pOpts.AdminID, pOpts.adminSecret)
|
||||
} else {
|
||||
glog.V(4).Infof("rbd: create %s size %s format %s using mon %s, pool %s id %s key %s", image, volSzGB, pOpts.ImageFormat, mon, pOpts.Pool, pOpts.AdminID, pOpts.adminSecret)
|
||||
}
|
||||
args := []string{"create", image, "--size", volSzGB, "--pool", pOpts.Pool, "--id", pOpts.AdminID, "-m", mon, "--key=" + pOpts.adminSecret, "--image-format", pOpts.ImageFormat}
|
||||
if pOpts.ImageFormat == rbdImageFormat2 {
|
||||
// if no image features is provided, it results in empty string
|
||||
// which disable all RBD image format 2 features as we expected
|
||||
features := strings.Join(pOpts.ImageFeatures, ",")
|
||||
args = append(args, "--image-feature", features)
|
||||
}
|
||||
output, err = execCommand("rbd", args)
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create rbd image: %v, command output: %s", err, string(output))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// rbdStatus checks if there is watcher on the image.
|
||||
// It returns true if there is a watcher onthe image, otherwise returns false.
|
||||
func rbdStatus(image string, b *rbdVolumeOptions) (bool, string, error) {
|
||||
var err error
|
||||
var output string
|
||||
var cmd []byte
|
||||
|
||||
// If we don't have admin id/secret (e.g. attaching), fallback to user id/secret.
|
||||
id := b.AdminID
|
||||
secret := b.adminSecret
|
||||
if id == "" {
|
||||
id = b.UserID
|
||||
secret = b.userSecret
|
||||
}
|
||||
|
||||
glog.V(4).Infof("rbd: status %s using mon %s, pool %s id %s key %s", image, b.Monitors, b.Pool, id, secret)
|
||||
args := []string{"status", image, "--pool", b.Pool, "-m", b.Monitors, "--id", id, "--key=" + secret}
|
||||
cmd, err = execCommand("rbd", args)
|
||||
output = string(cmd)
|
||||
|
||||
if err, ok := err.(*exec.Error); ok {
|
||||
if err.Err == exec.ErrNotFound {
|
||||
glog.Errorf("rbd cmd not found")
|
||||
// fail fast if command not found
|
||||
return false, output, err
|
||||
}
|
||||
}
|
||||
|
||||
// If command never succeed, returns its last error.
|
||||
if err != nil {
|
||||
return false, output, err
|
||||
}
|
||||
|
||||
if strings.Contains(output, imageWatcherStr) {
|
||||
glog.V(4).Infof("rbd: watchers on %s: %s", image, output)
|
||||
return true, output, nil
|
||||
} else {
|
||||
glog.Warningf("rbd: no watchers on %s", image)
|
||||
return false, output, nil
|
||||
}
|
||||
}
|
||||
|
||||
// DeleteImage deletes a ceph image with provision and volume options.
|
||||
func deleteRBDImage(image string, b *rbdVolumeOptions) error {
|
||||
var output []byte
|
||||
found, _, err := rbdStatus(image, b)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if found {
|
||||
glog.Info("rbd is still being used ", image)
|
||||
return fmt.Errorf("rbd %s is still being used", image)
|
||||
}
|
||||
id := b.AdminID
|
||||
secret := b.adminSecret
|
||||
if id == "" {
|
||||
id = b.UserID
|
||||
secret = b.userSecret
|
||||
}
|
||||
|
||||
glog.V(4).Infof("rbd: rm %s using mon %s, pool %s id %s key %s", image, b.Monitors, b.Pool, id, secret)
|
||||
args := []string{"rm", image, "--pool", b.Pool, "--id", id, "-m", b.Monitors, "--key=" + secret}
|
||||
output, err = execCommand("rbd", args)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
glog.Errorf("failed to delete rbd image: %v, command output: %s", err, string(output))
|
||||
return err
|
||||
}
|
||||
|
||||
func execCommand(command string, args []string) ([]byte, error) {
|
||||
cmd := exec.Command(command, args...)
|
||||
return cmd.CombinedOutput()
|
||||
}
|
||||
|
||||
func getRBDVolumeOptions(volOptions map[string]string, client *kubernetes.Clientset) (*rbdVolumeOptions, error) {
|
||||
rbdVolume := &rbdVolumeOptions{}
|
||||
var ok bool
|
||||
var err error
|
||||
rbdVolume.AdminID, ok = volOptions["adminId"]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("Missing required parameter adminId")
|
||||
}
|
||||
rbdVolume.AdminSecretName, ok = volOptions["adminSecretName"]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("Missing required parameter adminSecretName")
|
||||
}
|
||||
rbdVolume.AdminSecretNamespace, ok = volOptions["adminSecretNamespace"]
|
||||
if !ok {
|
||||
rbdVolume.AdminSecretNamespace = "default"
|
||||
}
|
||||
rbdVolume.adminSecret, err = parseStorageClassSecret(rbdVolume.AdminSecretName, rbdVolume.AdminSecretNamespace, client)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Failed to retrieve Admin secret %v", err)
|
||||
}
|
||||
rbdVolume.Pool, ok = volOptions["pool"]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("Missing required parameter pool")
|
||||
}
|
||||
rbdVolume.Monitors, ok = volOptions["monitors"]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("Missing required parameter monitors")
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rbdVolume.UserID, ok = volOptions["userId"]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("Missing required parameter userId")
|
||||
}
|
||||
rbdVolume.UserSecretName, ok = volOptions["userSecretName"]
|
||||
if ok {
|
||||
rbdVolume.UserSecretNamespace, ok = volOptions["userSecretNamespace"]
|
||||
if !ok {
|
||||
rbdVolume.UserSecretNamespace = "default"
|
||||
}
|
||||
rbdVolume.userSecret, err = parseStorageClassSecret(rbdVolume.UserSecretName, rbdVolume.UserSecretNamespace, client)
|
||||
if err != nil {
|
||||
glog.Errorf("failed to retrieve user's secret: %s/%s (%v)", rbdVolume.UserSecretName, rbdVolume.UserSecretNamespace, err)
|
||||
}
|
||||
}
|
||||
rbdVolume.ImageFormat, ok = volOptions["imageFormat"]
|
||||
if !ok {
|
||||
rbdVolume.ImageFormat = "2"
|
||||
}
|
||||
|
||||
return rbdVolume, nil
|
||||
}
|
||||
|
||||
func parseStorageClassSecret(secretName string, namespace string, client *kubernetes.Clientset) (string, error) {
|
||||
if client == nil {
|
||||
return "", fmt.Errorf("Cannot get kube client")
|
||||
}
|
||||
secrets, err := client.CoreV1().Secrets(namespace).Get(secretName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
secret := ""
|
||||
for k, v := range secrets.Data {
|
||||
if k == secretName {
|
||||
return string(v), nil
|
||||
}
|
||||
secret = string(v)
|
||||
}
|
||||
|
||||
return secret, nil
|
||||
}
|
||||
|
||||
func attachRBDImage(image string, volOptions *rbdVolumeOptions) (string, error) {
|
||||
var err error
|
||||
var output []byte
|
||||
|
||||
devicePath, found := waitForPath(volOptions.Pool, image, 1)
|
||||
if !found {
|
||||
attachdetachMutex.LockKey(string(volOptions.Pool + image))
|
||||
defer attachdetachMutex.UnlockKey(string(volOptions.Pool + image))
|
||||
|
||||
_, err = execCommand("modprobe", []string{"rbd"})
|
||||
if err != nil {
|
||||
glog.Warningf("rbd: failed to load rbd kernel module:%v", err)
|
||||
}
|
||||
|
||||
backoff := wait.Backoff{
|
||||
Duration: rbdImageWatcherInitDelay,
|
||||
Factor: rbdImageWatcherFactor,
|
||||
Steps: rbdImageWatcherSteps,
|
||||
}
|
||||
err := wait.ExponentialBackoff(backoff, func() (bool, error) {
|
||||
used, rbdOutput, err := rbdStatus(image, volOptions)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("fail to check rbd image status with: (%v), rbd output: (%s)", err, rbdOutput)
|
||||
}
|
||||
return !used, nil
|
||||
})
|
||||
// return error if rbd image has not become available for the specified timeout
|
||||
if err == wait.ErrWaitTimeout {
|
||||
return "", fmt.Errorf("rbd image %s/%s is still being used", volOptions.Pool, image)
|
||||
}
|
||||
// return error if any other errors were encountered during wating for the image to becme avialble
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
glog.V(1).Infof("rbd: map mon %s", volOptions.Monitors)
|
||||
// If we don't have admin id/secret (e.g. attaching), fallback to user id/secret.
|
||||
id := volOptions.AdminID
|
||||
secret := volOptions.adminSecret
|
||||
if id == "" {
|
||||
id = volOptions.UserID
|
||||
secret = volOptions.userSecret
|
||||
}
|
||||
|
||||
output, err = execCommand("rbd", []string{
|
||||
"map", image, "--pool", volOptions.Pool, "--id", id, "-m", volOptions.Monitors, "--key=" + secret})
|
||||
if err != nil {
|
||||
glog.V(1).Infof("rbd: map error %v, rbd output: %s", err, string(output))
|
||||
return "", fmt.Errorf("rbd: map failed %v, rbd output: %s", err, string(output))
|
||||
}
|
||||
devicePath, found = waitForPath(volOptions.Pool, image, 10)
|
||||
if !found {
|
||||
return "", fmt.Errorf("Could not map image %s/%s, Timeout after 10s", volOptions.Pool, image)
|
||||
}
|
||||
}
|
||||
|
||||
return devicePath, nil
|
||||
}
|
||||
|
||||
func detachRBDImage(image string, volOptions *rbdVolumeOptions) error {
|
||||
var err error
|
||||
var output []byte
|
||||
|
||||
glog.V(1).Infof("rbd: unmap device %s", volOptions.ImageMapping[image])
|
||||
// If we don't have admin id/secret (e.g. attaching), fallback to user id/secret.
|
||||
id := volOptions.AdminID
|
||||
secret := volOptions.adminSecret
|
||||
if id == "" {
|
||||
id = volOptions.UserID
|
||||
secret = volOptions.userSecret
|
||||
}
|
||||
|
||||
output, err = execCommand("rbd", []string{
|
||||
"unmap", volOptions.ImageMapping[image], "--id", id, "--key=" + secret})
|
||||
if err != nil {
|
||||
glog.V(1).Infof("rbd: unmap error %v, rbd output: %s", err, string(output))
|
||||
return fmt.Errorf("rbd: unmap failed %v, rbd output: %s", err, string(output))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func getDevFromImageAndPool(pool, image string) (string, bool) {
|
||||
// /sys/bus/rbd/devices/X/name and /sys/bus/rbd/devices/X/pool
|
||||
sys_path := "/sys/bus/rbd/devices"
|
||||
if dirs, err := ioutil.ReadDir(sys_path); err == nil {
|
||||
for _, f := range dirs {
|
||||
name := f.Name()
|
||||
// first match pool, then match name
|
||||
poolFile := path.Join(sys_path, name, "pool")
|
||||
poolBytes, err := ioutil.ReadFile(poolFile)
|
||||
if err != nil {
|
||||
glog.V(4).Infof("Error reading %s: %v", poolFile, err)
|
||||
continue
|
||||
}
|
||||
if strings.TrimSpace(string(poolBytes)) != pool {
|
||||
glog.V(4).Infof("Device %s is not %q: %q", name, pool, string(poolBytes))
|
||||
continue
|
||||
}
|
||||
imgFile := path.Join(sys_path, name, "name")
|
||||
imgBytes, err := ioutil.ReadFile(imgFile)
|
||||
if err != nil {
|
||||
glog.V(4).Infof("Error reading %s: %v", imgFile, err)
|
||||
continue
|
||||
}
|
||||
if strings.TrimSpace(string(imgBytes)) != image {
|
||||
glog.V(4).Infof("Device %s is not %q: %q", name, image, string(imgBytes))
|
||||
continue
|
||||
}
|
||||
// found a match, check if device exists
|
||||
devicePath := "/dev/rbd" + name
|
||||
if _, err := os.Lstat(devicePath); err == nil {
|
||||
return devicePath, true
|
||||
}
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// stat a path, if not exists, retry maxRetries times
|
||||
func waitForPath(pool, image string, maxRetries int) (string, bool) {
|
||||
for i := 0; i < maxRetries; i++ {
|
||||
devicePath, found := getDevFromImageAndPool(pool, image)
|
||||
if found {
|
||||
return devicePath, true
|
||||
}
|
||||
if i == maxRetries-1 {
|
||||
break
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
func persistVolInfo(image string, persistentStoragePath string, volInfo *rbdVolumeOptions) error {
|
||||
file := path.Join(persistentStoragePath, image+".json")
|
||||
fp, err := os.Create(file)
|
||||
if err != nil {
|
||||
return fmt.Errorf("rbd: create err %s/%s", file, err)
|
||||
}
|
||||
defer fp.Close()
|
||||
|
||||
encoder := json.NewEncoder(fp)
|
||||
if err = encoder.Encode(volInfo); err != nil {
|
||||
return fmt.Errorf("rbd: encode err: %v.", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func loadVolInfo(image string, persistentStoragePath string, volInfo *rbdVolumeOptions) error {
|
||||
file := path.Join(persistentStoragePath, image+".json")
|
||||
fp, err := os.Open(file)
|
||||
if err != nil {
|
||||
return fmt.Errorf("rbd: open err %s/%s", file, err)
|
||||
}
|
||||
defer fp.Close()
|
||||
|
||||
decoder := json.NewDecoder(fp)
|
||||
if err = decoder.Decode(volInfo); err != nil {
|
||||
return fmt.Errorf("rbd: decode err: %v.", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func deleteVolInfo(image string, persistentStoragePath string) error {
|
||||
file := path.Join(persistentStoragePath, image+".json")
|
||||
err := os.Remove(file)
|
||||
if err != nil {
|
||||
if err != os.ErrNotExist {
|
||||
return fmt.Errorf("rbd: open err %s/%s", file, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user