2018-03-05 11:59:47 +00:00
|
|
|
/*
|
2019-04-03 08:46:15 +00:00
|
|
|
Copyright 2018 The Ceph-CSI Authors.
|
2018-03-05 11:59:47 +00:00
|
|
|
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
you may not use this file except in compliance with the License.
|
|
|
|
You may obtain a copy of the License at
|
|
|
|
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
See the License for the specific language governing permissions and
|
|
|
|
limitations under the License.
|
|
|
|
*/
|
|
|
|
|
|
|
|
package cephfs
|
|
|
|
|
|
|
|
import (
|
2019-02-04 13:04:11 +00:00
|
|
|
"k8s.io/klog"
|
2018-03-05 11:59:47 +00:00
|
|
|
|
2019-03-13 05:09:58 +00:00
|
|
|
csicommon "github.com/ceph/ceph-csi/pkg/csi-common"
|
2018-12-19 14:26:16 +00:00
|
|
|
"github.com/ceph/ceph-csi/pkg/util"
|
2019-02-18 11:30:28 +00:00
|
|
|
|
|
|
|
"github.com/container-storage-interface/spec/lib/go/csi"
|
2018-03-05 11:59:47 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
const (
|
2019-05-28 19:03:18 +00:00
|
|
|
|
|
|
|
// volIDVersion is the version number of volume ID encoding scheme
|
|
|
|
volIDVersion uint16 = 1
|
|
|
|
|
|
|
|
// csiConfigFile is the location of the CSI config file
|
|
|
|
csiConfigFile = "/etc/ceph-csi-config/config.json"
|
|
|
|
|
|
|
|
// RADOS namespace to store CSI specific objects and keys
|
|
|
|
radosNamespace = "csi"
|
2018-03-05 11:59:47 +00:00
|
|
|
)
|
|
|
|
|
2019-03-13 05:09:58 +00:00
|
|
|
// PluginFolder defines the location of ceph plugin
|
2019-07-04 09:49:57 +00:00
|
|
|
var PluginFolder = ""
|
2019-03-13 05:09:58 +00:00
|
|
|
|
2019-01-28 11:47:06 +00:00
|
|
|
// Driver contains the default identity,node and controller struct
|
2019-01-17 07:51:06 +00:00
|
|
|
type Driver struct {
|
2019-01-17 06:18:18 +00:00
|
|
|
cd *csicommon.CSIDriver
|
2018-03-05 11:59:47 +00:00
|
|
|
|
2019-01-17 07:51:06 +00:00
|
|
|
is *IdentityServer
|
|
|
|
ns *NodeServer
|
|
|
|
cs *ControllerServer
|
2018-03-05 11:59:47 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
var (
|
2019-05-28 19:03:18 +00:00
|
|
|
// CSIInstanceID is the instance ID that is unique to an instance of CSI, used when sharing
|
|
|
|
// ceph clusters across CSI instances, to differentiate omap names per CSI instance
|
|
|
|
CSIInstanceID = "default"
|
|
|
|
|
|
|
|
// volJournal is used to maintain RADOS based journals for CO generated
|
|
|
|
// VolumeName to backing CephFS subvolumes
|
|
|
|
volJournal *util.CSIJournal
|
2018-03-05 11:59:47 +00:00
|
|
|
)
|
|
|
|
|
2019-01-28 11:47:06 +00:00
|
|
|
// NewDriver returns new ceph driver
|
2019-01-17 07:51:06 +00:00
|
|
|
func NewDriver() *Driver {
|
|
|
|
return &Driver{}
|
2018-03-05 11:59:47 +00:00
|
|
|
}
|
|
|
|
|
2019-01-28 11:47:06 +00:00
|
|
|
// NewIdentityServer initialize a identity server for ceph CSI driver
|
2019-01-17 07:51:06 +00:00
|
|
|
func NewIdentityServer(d *csicommon.CSIDriver) *IdentityServer {
|
|
|
|
return &IdentityServer{
|
2018-03-05 11:59:47 +00:00
|
|
|
DefaultIdentityServer: csicommon.NewDefaultIdentityServer(d),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-01-28 11:47:06 +00:00
|
|
|
// NewControllerServer initialize a controller server for ceph CSI driver
|
2019-01-17 07:51:06 +00:00
|
|
|
func NewControllerServer(d *csicommon.CSIDriver, cachePersister util.CachePersister) *ControllerServer {
|
|
|
|
return &ControllerServer{
|
2018-03-05 11:59:47 +00:00
|
|
|
DefaultControllerServer: csicommon.NewDefaultControllerServer(d),
|
2018-12-19 14:26:16 +00:00
|
|
|
MetadataStore: cachePersister,
|
2018-03-05 11:59:47 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-01-28 11:47:06 +00:00
|
|
|
// NewNodeServer initialize a node server for ceph CSI driver.
|
2019-07-30 06:20:28 +00:00
|
|
|
func NewNodeServer(d *csicommon.CSIDriver, t string) *NodeServer {
|
2019-01-17 07:51:06 +00:00
|
|
|
return &NodeServer{
|
2019-07-30 06:20:28 +00:00
|
|
|
DefaultNodeServer: csicommon.NewDefaultNodeServer(d, t),
|
2018-03-05 11:59:47 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-01-28 11:47:06 +00:00
|
|
|
// Run start a non-blocking grpc controller,node and identityserver for
|
|
|
|
// ceph CSI driver which can serve multiple parallel requests
|
2019-08-14 05:57:45 +00:00
|
|
|
func (fs *Driver) Run(conf *util.Config, cachePersister util.CachePersister) {
|
2018-04-13 12:53:17 +00:00
|
|
|
|
|
|
|
// Configuration
|
2019-08-14 05:57:45 +00:00
|
|
|
PluginFolder = conf.PluginPath
|
2018-04-13 12:53:17 +00:00
|
|
|
|
2018-08-14 14:48:30 +00:00
|
|
|
if err := loadAvailableMounters(); err != nil {
|
2019-02-04 13:04:11 +00:00
|
|
|
klog.Fatalf("cephfs: failed to load ceph mounters: %v", err)
|
2018-08-14 14:48:30 +00:00
|
|
|
}
|
|
|
|
|
2019-04-22 21:35:39 +00:00
|
|
|
if err := util.WriteCephConfig(); err != nil {
|
2019-02-13 12:57:16 +00:00
|
|
|
klog.Fatalf("failed to write ceph configuration file: %v", err)
|
|
|
|
}
|
|
|
|
|
2019-05-28 19:03:18 +00:00
|
|
|
// Use passed in instance ID, if provided for omap suffix naming
|
2019-08-14 05:57:45 +00:00
|
|
|
if conf.InstanceID != "" {
|
|
|
|
CSIInstanceID = conf.InstanceID
|
2019-05-28 19:03:18 +00:00
|
|
|
}
|
|
|
|
// Get an instance of the volume journal
|
|
|
|
volJournal = util.NewCSIVolumeJournal()
|
|
|
|
|
|
|
|
// Update keys with CSI instance suffix
|
|
|
|
volJournal.SetCSIDirectorySuffix(CSIInstanceID)
|
|
|
|
|
|
|
|
// Update namespace for storing keys into a specific namespace on RADOS, in the CephFS
|
|
|
|
// metadata pool
|
|
|
|
volJournal.SetNamespace(radosNamespace)
|
|
|
|
|
2019-08-14 05:57:45 +00:00
|
|
|
initVolumeMountCache(conf.DriverName, conf.MountCacheDir)
|
|
|
|
if conf.MountCacheDir != "" {
|
2019-03-29 02:18:59 +00:00
|
|
|
if err := remountCachedVolumes(); err != nil {
|
|
|
|
klog.Warningf("failed to remount cached volumes: %v", err)
|
2019-06-10 06:48:41 +00:00
|
|
|
// ignore remount fail
|
2019-03-29 02:18:59 +00:00
|
|
|
}
|
2019-03-25 14:47:39 +00:00
|
|
|
}
|
2018-03-05 11:59:47 +00:00
|
|
|
// Initialize default library driver
|
|
|
|
|
2019-08-14 05:57:45 +00:00
|
|
|
fs.cd = csicommon.NewCSIDriver(conf.DriverName, util.DriverVersion, conf.NodeID)
|
2019-01-17 06:18:18 +00:00
|
|
|
if fs.cd == nil {
|
2019-02-13 12:57:16 +00:00
|
|
|
klog.Fatalln("failed to initialize CSI driver")
|
2018-03-05 11:59:47 +00:00
|
|
|
}
|
|
|
|
|
2019-08-14 06:42:17 +00:00
|
|
|
if conf.IsControllerServer || !conf.IsNodeServer {
|
|
|
|
fs.cd.AddControllerServiceCapabilities([]csi.ControllerServiceCapability_RPC_Type{
|
|
|
|
csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME,
|
|
|
|
})
|
|
|
|
|
|
|
|
fs.cd.AddVolumeCapabilityAccessModes([]csi.VolumeCapability_AccessMode_Mode{
|
|
|
|
csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER,
|
|
|
|
})
|
|
|
|
}
|
2018-03-05 11:59:47 +00:00
|
|
|
// Create gRPC servers
|
|
|
|
|
2019-01-17 06:18:18 +00:00
|
|
|
fs.is = NewIdentityServer(fs.cd)
|
2018-12-19 14:26:16 +00:00
|
|
|
|
2019-08-14 06:42:17 +00:00
|
|
|
if conf.IsNodeServer {
|
|
|
|
fs.ns = NewNodeServer(fs.cd, conf.Vtype)
|
|
|
|
}
|
|
|
|
|
|
|
|
if conf.IsControllerServer {
|
|
|
|
fs.cs = NewControllerServer(fs.cd, cachePersister)
|
|
|
|
}
|
|
|
|
if !conf.IsControllerServer && !conf.IsNodeServer {
|
|
|
|
fs.ns = NewNodeServer(fs.cd, conf.Vtype)
|
|
|
|
fs.cs = NewControllerServer(fs.cd, cachePersister)
|
|
|
|
}
|
2018-03-05 11:59:47 +00:00
|
|
|
|
|
|
|
server := csicommon.NewNonBlockingGRPCServer()
|
2019-08-21 09:28:02 +00:00
|
|
|
server.Start(conf.Endpoint, conf.HistogramOption, fs.is, fs.cs, fs.ns, conf.EnableGRPCMetrics)
|
|
|
|
if conf.EnableGRPCMetrics {
|
|
|
|
go util.StartMetricsServer(conf)
|
|
|
|
}
|
2018-03-05 11:59:47 +00:00
|
|
|
server.Wait()
|
|
|
|
}
|