provide option to set pluginpath for cephfs

Signed-off-by: Madhu Rajanna <madhupr007@gmail.com>
This commit is contained in:
Madhu Rajanna 2019-07-04 15:19:57 +05:30
parent da111de65d
commit 778cfb3090
6 changed files with 39 additions and 49 deletions

View File

@ -20,6 +20,7 @@ import (
"flag"
"os"
"path"
"path/filepath"
"strings"
"github.com/ceph/ceph-csi/pkg/cephfs"
@ -45,6 +46,7 @@ var (
instanceID = flag.String("instanceid", "", "Unique ID distinguishing this instance of Ceph CSI among other"+
" instances, when sharing Ceph clusters across CSI instances for provisioning")
metadataStorage = flag.String("metadatastorage", "", "metadata persistence method [node|k8s_configmap]")
pluginPath = flag.String("pluginpath", "/var/lib/kubelet/plugins/", "the location of cephcsi plugin")
// rbd related flags
containerized = flag.Bool("containerized", true, "whether run as containerized")
@ -106,31 +108,24 @@ func main() {
if err != nil {
klog.Fatalln(err) // calls exit
}
klog.Infof("Starting driver type: %v with name: %v", driverType, dname)
switch driverType {
case rbdType:
rbd.PluginFolder += dname
csipluginPath := filepath.Join(*pluginPath, dname)
if *metadataStorage != "" {
cp, err = util.CreatePersistanceStorage(
rbd.PluginFolder, *metadataStorage, dname)
csipluginPath, *metadataStorage, *pluginPath)
if err != nil {
os.Exit(1)
}
}
klog.Infof("Starting driver type: %v with name: %v", driverType, dname)
switch driverType {
case rbdType:
driver := rbd.NewDriver()
driver.Run(dname, *nodeID, *endpoint, *instanceID, *containerized, cp)
case cephfsType:
cephfs.PluginFolder += dname
if *metadataStorage != "" {
cp, err = util.CreatePersistanceStorage(
cephfs.PluginFolder, *metadataStorage, dname)
if err != nil {
os.Exit(1)
}
}
driver := cephfs.NewDriver()
driver.Run(dname, *nodeID, *endpoint, *volumeMounter, *mountCacheDir, *instanceID, cp)
driver.Run(dname, *nodeID, *endpoint, *volumeMounter, *mountCacheDir, *instanceID, csipluginPath, cp)
default:
klog.Fatalln("invalid volume type", vtype) // calls exit

View File

@ -28,16 +28,18 @@ make image-cephcsi
**Available command line arguments:**
Option | Default value | Description
--------------------|-----------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
`--endpoint` | `unix://tmp/csi.sock` | CSI endpoint, must be a UNIX socket
`--drivername` | `cephfs.csi.ceph.com` | Name of the driver (Kubernetes: `provisioner` field in StorageClass must correspond to this value)
`--nodeid` | _empty_ | This node's ID
| `--type` | _empty_ | Driver type `[rbd | cephfs]` If the driver type is set to `rbd` it will act as a `rbd plugin` or if it's set to `cephfs` will act as a `cephfs plugin`
`--volumemounter` | _empty_ | Default volume mounter. Available options are `kernel` and `fuse`. This is the mount method used if volume parameters don't specify otherwise. If left unspecified, the driver will first probe for `ceph-fuse` in system's path and will choose Ceph kernel client if probing failed.
`--mountcachedir` | _empty_ | Volume mount cache info save dir. If left unspecified, the dirver will not record mount info, or it will save mount info and when driver restart it will remount volume it cached.
`--instanceid` | "default" | Unique ID distinguishing this instance of Ceph CSI among other instances, when sharing Ceph clusters across CSI instances for provisioning
`--metadatastorage` | _empty_ | Points to where older (1.0.0 or older plugin versions) metadata about provisioned volumes are kept, as file or in as k8s configmap (`node` or `k8s_configmap` respectively)
| Option | Default value | Description |
| ----------------- | --------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `--endpoint` | `unix://tmp/csi.sock` | CSI endpoint, must be a UNIX socket |
| `--drivername` | `cephfs.csi.ceph.com` | Name of the driver (Kubernetes: `provisioner` field in StorageClass must correspond to this value) |
| `--nodeid` | _empty_ | This node's ID |
| `--type` | _empty_ | Driver type `[rbd | cephfs]` If the driver type is set to `rbd` it will act as a `rbd plugin` or if it's set to `cephfs` will act as a `cephfs plugin` |
| `--volumemounter` | _empty_ | Default volume mounter. Available options are `kernel` and `fuse`. This is the mount method used if volume parameters don't specify otherwise. If left unspecified, the driver will first probe for `ceph-fuse` in system's path and will choose Ceph kernel client if probing failed. |
| `--mountcachedir` | _empty_ | Volume mount cache info save dir. If left unspecified, the dirver will not record mount info, or it will save mount info and when driver restart it will remount volume it cached. |
| `--instanceid` | "default" | Unique ID distinguishing this instance of Ceph CSI among other instances, when sharing Ceph clusters across CSI instances for provisioning |
| `--pluginpath` | "/var/lib/kubelet/plugins/" | The location of cephcsi plugin on host |
| `--metadatastorage` | _empty_ | Points to where older (1.0.0 or older plugin versions) metadata about provisioned volumes are kept, as file or in as k8s configmap (`node` or `k8s_configmap` respectively) |
**Available environmental variables:**
@ -50,14 +52,14 @@ is used to define in which namespace you want the configmaps to be stored
**Available volume parameters:**
Parameter | Required | Description
----------------------------------------------------------------------------------------------------|--------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
`clusterID` | yes | String representing a Ceph cluster, must be unique across all Ceph clusters in use for provisioning, cannot be greater than 36 bytes in length, and should remain immutable for the lifetime of the Ceph cluster in use
`fsName` | yes | CephFS filesystem name into which the volume shall be created
`mounter` | no | Mount method to be used for this volume. Available options are `kernel` for Ceph kernel client and `fuse` for Ceph FUSE driver. Defaults to "default mounter", see command line arguments.
`pool` | yes | Ceph pool into which the volume shall be created
`csi.storage.k8s.io/provisioner-secret-name`, `csi.storage.k8s.io/node-stage-secret-name` | for Kubernetes | Name of the Kubernetes Secret object containing Ceph client credentials. Both parameters should have the same value
`csi.storage.k8s.io/provisioner-secret-namespace`, `csi.storage.k8s.io/node-stage-secret-namespace` | for Kubernetes | Namespaces of the above Secret objects
| Parameter | Required | Description |
| --------------------------------------------------------------------------------------------------- | -------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `clusterID` | yes | String representing a Ceph cluster, must be unique across all Ceph clusters in use for provisioning, cannot be greater than 36 bytes in length, and should remain immutable for the lifetime of the Ceph cluster in use |
| `fsName` | yes | CephFS filesystem name into which the volume shall be created |
| `mounter` | no | Mount method to be used for this volume. Available options are `kernel` for Ceph kernel client and `fuse` for Ceph FUSE driver. Defaults to "default mounter", see command line arguments. |
| `pool` | yes | Ceph pool into which the volume shall be created |
| `csi.storage.k8s.io/provisioner-secret-name`, `csi.storage.k8s.io/node-stage-secret-name` | for Kubernetes | Name of the Kubernetes Secret object containing Ceph client credentials. Both parameters should have the same value |
| `csi.storage.k8s.io/provisioner-secret-namespace`, `csi.storage.k8s.io/node-stage-secret-namespace` | for Kubernetes | Namespaces of the above Secret objects |
**NOTE:** An accompanying CSI configuration file, needs to be provided to the
running pods. Refer to [Creating CSI configuration](../examples/README.md#creating-csi-configuration)

View File

@ -38,7 +38,7 @@ const (
)
// PluginFolder defines the location of ceph plugin
var PluginFolder = "/var/lib/kubelet/plugins/"
var PluginFolder = ""
// Driver contains the default identity,node and controller struct
type Driver struct {
@ -91,9 +91,10 @@ func NewNodeServer(d *csicommon.CSIDriver) *NodeServer {
// Run start a non-blocking grpc controller,node and identityserver for
// ceph CSI driver which can serve multiple parallel requests
func (fs *Driver) Run(driverName, nodeID, endpoint, volumeMounter, mountCacheDir, instanceID string, cachePersister util.CachePersister) {
func (fs *Driver) Run(driverName, nodeID, endpoint, volumeMounter, mountCacheDir, instanceID, pluginPath string, cachePersister util.CachePersister) {
// Configuration
PluginFolder = pluginPath
if err := loadAvailableMounters(); err != nil {
klog.Fatalf("cephfs: failed to load ceph mounters: %v", err)

View File

@ -47,9 +47,6 @@ type Driver struct {
var (
// PluginFolder defines the location of ceph plugin
PluginFolder = "/var/lib/kubelet/plugins/"
// CSIInstanceID is the instance ID that is unique to an instance of CSI, used when sharing
// ceph clusters across CSI instances, to differentiate omap names per CSI instance
CSIInstanceID = "default"

View File

@ -22,11 +22,6 @@ import (
"k8s.io/klog"
)
const (
// PluginFolder defines location of plugins
PluginFolder = "/var/lib/kubelet/plugins"
)
// ForAllFunc is a unary predicate for visiting all cache entries
// matching the `pattern' in CachePersister's ForAll function.
type ForAllFunc func(identifier string) error
@ -45,7 +40,7 @@ type CachePersister interface {
}
// NewCachePersister returns CachePersister based on store
func NewCachePersister(metadataStore, driverName string) (CachePersister, error) {
func NewCachePersister(metadataStore, pluginPath string) (CachePersister, error) {
if metadataStore == "k8s_configmap" {
klog.Infof("cache-perister: using kubernetes configmap as metadata cache persister")
k8scm := &K8sCMCache{}
@ -55,7 +50,7 @@ func NewCachePersister(metadataStore, driverName string) (CachePersister, error)
} else if metadataStore == "node" {
klog.Infof("cache-persister: using node as metadata cache persister")
nc := &NodeCache{}
nc.BasePath = PluginFolder + "/" + driverName
nc.BasePath = pluginPath
nc.CacheDir = "controller"
return nc, nil
}

View File

@ -59,7 +59,7 @@ func roundUpSize(volumeSizeBytes, allocationUnitBytes int64) int64 {
}
// CreatePersistanceStorage creates storage path and initializes new cache
func CreatePersistanceStorage(sPath, metaDataStore, driverName string) (CachePersister, error) {
func CreatePersistanceStorage(sPath, metaDataStore, pluginPath string) (CachePersister, error) {
var err error
if err = CreateMountPoint(path.Join(sPath, "controller")); err != nil {
klog.Errorf("failed to create persistent storage for controller: %v", err)
@ -71,7 +71,7 @@ func CreatePersistanceStorage(sPath, metaDataStore, driverName string) (CachePer
return nil, err
}
cp, err := NewCachePersister(metaDataStore, driverName)
cp, err := NewCachePersister(metaDataStore, pluginPath)
if err != nil {
klog.Errorf("failed to define cache persistence method: %v", err)
return nil, err