mirrors/ceph-csi
1
0
Fork 0
mirror of https://github.com/ceph/ceph-csi.git synced 2025-06-13 02:33:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

Removed config maps and replaced with rados omaps

Browse Source 
Existing config maps are now replaced with rados omaps that help
store information regarding the requested volume names and the rbd
image names backing the same.

Further to detect cluster, pool and which image a volume ID refers
to, changes to volume ID encoding has been done as per provided
design specification in the stateless ceph-csi proposal.

Additional changes and updates,
- Updated documentation
- Updated manifests
- Updated Helm chart
- Addressed a few csi-test failures

Signed-off-by: ShyamsundarR <srangana@redhat.com>
This commit is contained in:
ShyamsundarR
2019-04-22 17:35:39 -04:00
committed by mergify[bot]
parent f60a07ae82
commit d02e50aa9b
40 changed files with 2379 additions and 1373 deletions
Download Patch File Download Diff File Expand all files Collapse all files

256
pkg/util/cephcmds.go Normal file
View File

@ -0,0 +1,256 @@
/*
Copyright 2019 The Ceph-CSI Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package util
import (
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"k8s.io/klog"
"os"
"os/exec"
"strings"
)
// ExecCommand executes passed in program with args and returns seperate stdout and stderr streams
func ExecCommand(program string, args ...string) (stdout, stderr []byte, err error) {
var (
cmd = exec.Command(program, args...) // nolint: gosec
stdoutBuf bytes.Buffer
stderrBuf bytes.Buffer
)
cmd.Stdout = &stdoutBuf
cmd.Stderr = &stderrBuf
if err := cmd.Run(); err != nil {
return stdoutBuf.Bytes(), stderrBuf.Bytes(), fmt.Errorf("an error (%v)"+
" occurred while running %s", err, program)
}
return stdoutBuf.Bytes(), nil, nil
}
// cephStoragePoolSummary strongly typed JSON spec for osd ls pools output
type cephStoragePoolSummary struct {
Name string `json:"poolname"`
Number int64 `json:"poolnum"`
}
// GetPoolID searches a list of pools in a cluster and returns the ID of the pool that matches
// the passed in poolName parameter
func GetPoolID(monitors string, adminID string, key string, poolName string) (int64, error) {
// ceph <options> -f json osd lspools
// JSON out: [{"poolnum":<int64>,"poolname":<string>}]
stdout, _, err := ExecCommand(
"ceph",
"-m", monitors,
"--id", adminID,
"--key="+key,
"-c", CephConfigPath,
"-f", "json",
"osd", "lspools")
if err != nil {
klog.Errorf("failed getting pool list from cluster (%s)", err)
return 0, err
}
var pools []cephStoragePoolSummary
err = json.Unmarshal(stdout, &pools)
if err != nil {
klog.Errorf("failed to parse JSON output of pool list from cluster (%s)", err)
return 0, fmt.Errorf("unmarshal failed: %+v. raw buffer response: %s", err, string(stdout))
}
for _, p := range pools {
if poolName == p.Name {
return p.Number, nil
}
}
return 0, fmt.Errorf("pool (%s) not found in Ceph cluster", poolName)
}
// GetPoolName lists all pools in a ceph cluster, and matches the pool whose pool ID is equal to
// the requested poolID parameter
func GetPoolName(monitors string, adminID string, key string, poolID int64) (string, error) {
// ceph <options> -f json osd lspools
// [{"poolnum":1,"poolname":"replicapool"}]
stdout, _, err := ExecCommand(
"ceph",
"-m", monitors,
"--id", adminID,
"--key="+key,
"-c", CephConfigPath,
"-f", "json",
"osd", "lspools")
if err != nil {
klog.Errorf("failed getting pool list from cluster (%s)", err)
return "", err
}
var pools []cephStoragePoolSummary
err = json.Unmarshal(stdout, &pools)
if err != nil {
klog.Errorf("failed to parse JSON output of pool list from cluster (%s)", err)
return "", fmt.Errorf("unmarshal failed: %+v. raw buffer response: %s", err, string(stdout))
}
for _, p := range pools {
if poolID == p.Number {
return p.Name, nil
}
}
return "", fmt.Errorf("pool ID (%d) not found in Ceph cluster", poolID)
}
// SetOMapKeyValue sets the given key and value into the provided Ceph omap name
func SetOMapKeyValue(monitors, adminID, key, poolName, oMapName, oMapKey, keyValue string) error {
// Command: "rados <options> setomapval oMapName oMapKey keyValue"
_, _, err := ExecCommand(
"rados",
"-m", monitors,
"--id", adminID,
"--key="+key,
"-c", CephConfigPath,
"-p", poolName,
"setomapval", oMapName, oMapKey, keyValue)
if err != nil {
klog.Errorf("failed adding key (%s with value %s), to omap (%s) in "+
"pool (%s): (%v)", oMapKey, keyValue, oMapName, poolName, err)
return err
}
return nil
}
// GetOMapValue gets the value for the given key from the named omap
func GetOMapValue(monitors, adminID, key, poolName, oMapName, oMapKey string) (string, error) {
// Command: "rados <options> getomapval oMapName oMapKey <outfile>"
// No such key: replicapool/csi.volumes.directory.default/csi.volname
tmpFile, err := ioutil.TempFile("", "omap-get-")
if err != nil {
klog.Errorf("failed creating a temporary file for key contents")
return "", err
}
defer tmpFile.Close()
defer os.Remove(tmpFile.Name())
stdout, stderr, err := ExecCommand(
"rados",
"-m", monitors,
"--id", adminID,
"--key="+key,
"-c", CephConfigPath,
"-p", poolName,
"getomapval", oMapName, oMapKey, tmpFile.Name())
if err != nil {
// no logs, as attempting to check for key/value is done even on regular call sequences
stdoutanderr := strings.Join([]string{string(stdout), string(stderr)}, " ")
if strings.Contains(stdoutanderr, "No such key: "+poolName+"/"+oMapName+"/"+oMapKey) {
return "", ErrKeyNotFound{poolName + "/" + oMapName + "/" + oMapKey, err}
}
if strings.Contains(stdoutanderr, "error getting omap value "+
poolName+"/"+oMapName+"/"+oMapKey+": (2) No such file or directory") {
return "", ErrKeyNotFound{poolName + "/" + oMapName + "/" + oMapKey, err}
}
return "", fmt.Errorf("error (%v) occured, command output streams is (%s)",
err.Error(), stdoutanderr)
}
keyValue, err := ioutil.ReadAll(tmpFile)
return string(keyValue), err
}
// RemoveOMapKey removes the omap key from the given omap name
func RemoveOMapKey(monitors, adminID, key, poolName, oMapName, oMapKey string) error {
// Command: "rados <options> rmomapkey oMapName oMapKey"
_, _, err := ExecCommand(
"rados",
"-m", monitors,
"--id", adminID,
"--key="+key,
"-c", CephConfigPath,
"-p", poolName,
"rmomapkey", oMapName, oMapKey)
if err != nil {
// NOTE: Missing omap key removal does not return an error
klog.Errorf("failed removing key (%s), from omap (%s) in "+
"pool (%s): (%v)", oMapKey, oMapName, poolName, err)
return err
}
return nil
}
// CreateObject creates the object name passed in and returns ErrObjectExists if the provided object
// is already present in rados
func CreateObject(monitors, adminID, key, poolName, objectName string) error {
// Command: "rados <options> create objectName"
stdout, _, err := ExecCommand(
"rados",
"-m", monitors,
"--id", adminID,
"--key="+key,
"-c", CephConfigPath,
"-p", poolName,
"create", objectName)
if err != nil {
klog.Errorf("failed creating omap (%s) in pool (%s): (%v)", objectName, poolName, err)
if strings.Contains(string(stdout), "error creating "+poolName+"/"+objectName+
": (17) File exists") {
return ErrObjectExists{objectName, err}
}
return err
}
return nil
}
// RemoveObject removes the entire omap name passed in and returns ErrObjectNotFound is provided omap
// is not found in rados
func RemoveObject(monitors, adminID, key, poolName, oMapName string) error {
// Command: "rados <options> rm oMapName"
stdout, _, err := ExecCommand(
"rados",
"-m", monitors,
"--id", adminID,
"--key="+key,
"-c", CephConfigPath,
"-p", poolName,
"rm", oMapName)
if err != nil {
klog.Errorf("failed removing omap (%s) in pool (%s): (%v)", oMapName, poolName, err)
if strings.Contains(string(stdout), "error removing "+poolName+">"+oMapName+
": (2) No such file or directory") {
return ErrObjectNotFound{oMapName, err}
}
return err
}
return nil
}

50
pkg/util/cephconf.go Normal file
View File

@ -0,0 +1,50 @@
/*
Copyright 2018 The Ceph-CSI Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package util
import (
"io/ioutil"
"os"
)
var cephConfig = []byte(`[global]
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx
# Workaround for http://tracker.ceph.com/issues/23446
fuse_set_user_groups = false
`)
const (
cephConfigRoot = "/etc/ceph"
CephConfigPath = "/etc/ceph/ceph.conf"
)
func createCephConfigRoot() error {
return os.MkdirAll(cephConfigRoot, 0755) // #nosec
}
// WriteCephConfig writes out a basic ceph.conf file, making it easy to use
// ceph related CLIs
func WriteCephConfig() error {
if err := createCephConfigRoot(); err != nil {
return err
}
return ioutil.WriteFile(CephConfigPath, cephConfig, 0640)
}

137
pkg/util/configstore.go
View File

@ -1,137 +0,0 @@
/*
Copyright 2019 The Ceph-CSI Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package util
import (
"errors"
"fmt"
"k8s.io/klog"
"path"
"strings"
)
// StoreReader interface enables plugging different stores, that contain the
// keys and data. (e.g k8s secrets or local files)
type StoreReader interface {
DataForKey(clusterID string, key string) (string, error)
}
/* ConfigKeys contents and format,
- csMonitors: MON list, comma separated
- csAdminID: adminID, used for provisioning
- csUserID: userID, used for publishing
- csAdminKey: key, for adminID in csProvisionerUser
- csUserKey: key, for userID in csPublisherUser
- csPools: Pool list, comma separated
*/
// Constants for various ConfigKeys
const (
csMonitors = "monitors"
csAdminID = "adminid"
csUserID = "userid"
csAdminKey = "adminkey"
csUserKey = "userkey"
csPools = "pools"
)
// ConfigStore provides various gettors for ConfigKeys
type ConfigStore struct {
StoreReader
}
// dataForKey returns data from the config store for the provided key
func (dc *ConfigStore) dataForKey(clusterID, key string) (string, error) {
if dc.StoreReader != nil {
return dc.StoreReader.DataForKey(clusterID, key)
}
return "", errors.New("config store location uninitialized")
}
// Mons returns a comma separated MON list from the cluster config represented by clusterID
func (dc *ConfigStore) Mons(clusterID string) (string, error) {
return dc.dataForKey(clusterID, csMonitors)
}
// Pools returns a list of pool names from the cluster config represented by clusterID
func (dc *ConfigStore) Pools(clusterID string) ([]string, error) {
content, err := dc.dataForKey(clusterID, csPools)
if err != nil {
return nil, err
}
return strings.Split(content, ","), nil
}
// AdminID returns the admin ID from the cluster config represented by clusterID
func (dc *ConfigStore) AdminID(clusterID string) (string, error) {
return dc.dataForKey(clusterID, csAdminID)
}
// UserID returns the user ID from the cluster config represented by clusterID
func (dc *ConfigStore) UserID(clusterID string) (string, error) {
return dc.dataForKey(clusterID, csUserID)
}
// KeyForUser returns the key for the requested user ID from the cluster config
// represented by clusterID
func (dc *ConfigStore) KeyForUser(clusterID, userID string) (data string, err error) {
var fetchKey string
user, err := dc.AdminID(clusterID)
if err != nil {
return
}
if user == userID {
fetchKey = csAdminKey
} else {
user, err = dc.UserID(clusterID)
if err != nil {
return
}
if user != userID {
err = fmt.Errorf("requested user (%s) not found in cluster configuration of (%s)", userID, clusterID)
return
}
fetchKey = csUserKey
}
return dc.dataForKey(clusterID, fetchKey)
}
// NewConfigStore returns a config store based on value of configRoot. If
// configRoot is not "k8s_objects" then it is assumed to be a path to a
// directory, under which the configuration files can be found
func NewConfigStore(configRoot string) (*ConfigStore, error) {
if configRoot != "k8s_objects" {
klog.Infof("cache-store: using files in path (%s) as config store", configRoot)
fc := &FileConfig{}
fc.BasePath = path.Clean(configRoot)
dc := &ConfigStore{fc}
return dc, nil
}
klog.Infof("cache-store: using k8s objects as config store")
kc := &K8sConfig{}
kc.Client = NewK8sClient()
kc.Namespace = GetK8sNamespace()
dc := &ConfigStore{kc}
return dc, nil
}

160
pkg/util/configstore_test.go
View File

@ -1,160 +0,0 @@
/*
Copyright 2019 ceph-csi authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package util
import (
"io/ioutil"
"os"
"strings"
"testing"
)
var basePath = "./test_artifacts"
var clusterID = "testclusterid"
var cs *ConfigStore
func cleanupTestData() {
os.RemoveAll(basePath)
}
// nolint: gocyclo
func TestConfigStore(t *testing.T) {
var err error
var data string
var content string
var testDir string
defer cleanupTestData()
cs, err = NewConfigStore(basePath)
if err != nil {
t.Errorf("Fatal, failed to get a new config store")
}
err = os.MkdirAll(basePath, 0700)
if err != nil {
t.Errorf("Test setup error %s", err)
}
// TEST: Should fail as clusterid directory is missing
_, err = cs.Mons(clusterID)
if err == nil {
t.Errorf("Failed: expected error due to missing parent directory")
}
testDir = basePath + "/" + "ceph-cluster-" + clusterID
err = os.MkdirAll(testDir, 0700)
if err != nil {
t.Errorf("Test setup error %s", err)
}
// TEST: Should fail as mons file is missing
_, err = cs.Mons(clusterID)
if err == nil {
t.Errorf("Failed: expected error due to missing mons file")
}
data = ""
err = ioutil.WriteFile(testDir+"/"+csMonitors, []byte(data), 0644)
if err != nil {
t.Errorf("Test setup error %s", err)
}
// TEST: Should fail as MONs is an empty string
content, err = cs.Mons(clusterID)
if err == nil {
t.Errorf("Failed: want (%s), got (%s)", data, content)
}
data = "mon1,mon2,mon3"
err = ioutil.WriteFile(testDir+"/"+csMonitors, []byte(data), 0644)
if err != nil {
t.Errorf("Test setup error %s", err)
}
// TEST: Fetching MONs should succeed
content, err = cs.Mons(clusterID)
if err != nil || content != data {
t.Errorf("Failed: want (%s), got (%s), err (%s)", data, content, err)
}
data = "pool1,pool2"
err = ioutil.WriteFile(testDir+"/"+csPools, []byte(data), 0644)
if err != nil {
t.Errorf("Test setup error %s", err)
}
// TEST: Fetching MONs should succeed
listContent, err := cs.Pools(clusterID)
if err != nil || strings.Join(listContent, ",") != data {
t.Errorf("Failed: want (%s), got (%s), err (%s)", data, content, err)
}
data = "provuser"
err = ioutil.WriteFile(testDir+"/"+csAdminID, []byte(data), 0644)
if err != nil {
t.Errorf("Test setup error %s", err)
}
// TEST: Fetching provuser should succeed
content, err = cs.AdminID(clusterID)
if err != nil || content != data {
t.Errorf("Failed: want (%s), got (%s), err (%s)", data, content, err)
}
data = "pubuser"
err = ioutil.WriteFile(testDir+"/"+csUserID, []byte(data), 0644)
if err != nil {
t.Errorf("Test setup error %s", err)
}
// TEST: Fetching pubuser should succeed
content, err = cs.UserID(clusterID)
if err != nil || content != data {
t.Errorf("Failed: want (%s), got (%s), err (%s)", data, content, err)
}
data = "provkey"
err = ioutil.WriteFile(testDir+"/"+csAdminKey, []byte(data), 0644)
if err != nil {
t.Errorf("Test setup error %s", err)
}
// TEST: Fetching provkey should succeed
content, err = cs.KeyForUser(clusterID, "provuser")
if err != nil || content != data {
t.Errorf("Failed: want (%s), got (%s), err (%s)", data, content, err)
}
data = "pubkey"
err = ioutil.WriteFile(testDir+"/"+csUserKey, []byte(data), 0644)
if err != nil {
t.Errorf("Test setup error %s", err)
}
// TEST: Fetching pubkey should succeed
content, err = cs.KeyForUser(clusterID, "pubuser")
if err != nil || content != data {
t.Errorf("Failed: want (%s), got (%s), err (%s)", data, content, err)
}
// TEST: Fetching random user key should fail
_, err = cs.KeyForUser(clusterID, "random")
if err == nil {
t.Errorf("Failed: Expected to fail fetching random user key")
}
}

74
pkg/util/csiconfig.go Normal file
View File

@ -0,0 +1,74 @@
/*
Copyright 2019 The Ceph-CSI Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package util
import (
"encoding/json"
"fmt"
"io/ioutil"
"strings"
)
/*
Mons returns a comma separated MON list from the csi config for the given clusterID
Expected JSON structure in the passed in config file is,
[
{
"clusterID": "<cluster-id>",
"monitors":
[
"<monitor-value>",
"<monitor-value>",
...
]
},
...
]
*/
// clusterInfo strongly typed JSON spec for the above JSON structure
type clusterInfo struct {
ClusterID string `json:"clusterID"`
Monitors []string `json:"monitors"`
}
func Mons(pathToConfig, clusterID string) (string, error) {
var config []clusterInfo
// #nosec
content, err := ioutil.ReadFile(pathToConfig)
if err != nil {
err = fmt.Errorf("error fetching configuration for cluster ID (%s). (%s)", clusterID, err)
return "", err
}
err = json.Unmarshal(content, &config)
if err != nil {
return "", fmt.Errorf("unmarshal failed: %v. raw buffer response: %s",
err, string(content))
}
for _, cluster := range config {
if cluster.ClusterID == clusterID {
if len(cluster.Monitors) == 0 {
return "", fmt.Errorf("empty monitor list for cluster ID (%s) in config", clusterID)
}
return strings.Join(cluster.Monitors, ","), nil
}
}
return "", fmt.Errorf("missing configuration for cluster ID (%s)", clusterID)
}

132
pkg/util/csiconfig_test.go Normal file
View File

@ -0,0 +1,132 @@
/*
Copyright 2019 ceph-csi authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package util
import (
"io/ioutil"
"os"
"testing"
)
var basePath = "./test_artifacts"
var csiClusters = "csi-clusters.json"
var pathToConfig = basePath + "/" + csiClusters
var clusterID1 = "test1"
var clusterID2 = "test2"
func cleanupTestData() {
os.RemoveAll(basePath)
}
// nolint: gocyclo
func TestCSIConfig(t *testing.T) {
var err error
var data string
var content string
defer cleanupTestData()
err = os.MkdirAll(basePath, 0700)
if err != nil {
t.Errorf("Test setup error %s", err)
}
// TEST: Should fail as clusterid file is missing
_, err = Mons(pathToConfig, clusterID1)
if err == nil {
t.Errorf("Failed: expected error due to missing config")
}
data = ""
err = ioutil.WriteFile(basePath+"/"+csiClusters, []byte(data), 0644)
if err != nil {
t.Errorf("Test setup error %s", err)
}
// TEST: Should fail as file is empty
content, err = Mons(pathToConfig, clusterID1)
if err == nil {
t.Errorf("Failed: want (%s), got (%s)", data, content)
}
data = "[{\"clusterIDBad\":\"" + clusterID2 + "\",\"monitors\":[\"mon1\",\"mon2\",\"mon3\"]}]"
err = ioutil.WriteFile(basePath+"/"+csiClusters, []byte(data), 0644)
if err != nil {
t.Errorf("Test setup error %s", err)
}
// TEST: Should fail as clusterID data is malformed
content, err = Mons(pathToConfig, clusterID2)
if err == nil {
t.Errorf("Failed: want (%s), got (%s)", data, content)
}
data = "[{\"clusterID\":\"" + clusterID2 + "\",\"monitorsBad\":[\"mon1\",\"mon2\",\"mon3\"]}]"
err = ioutil.WriteFile(basePath+"/"+csiClusters, []byte(data), 0644)
if err != nil {
t.Errorf("Test setup error %s", err)
}
// TEST: Should fail as monitors key is incorrect/missing
content, err = Mons(pathToConfig, clusterID2)
if err == nil {
t.Errorf("Failed: want (%s), got (%s)", data, content)
}
data = "[{\"clusterID\":\"" + clusterID2 + "\",\"monitors\":[\"mon1\",2,\"mon3\"]}]"
err = ioutil.WriteFile(basePath+"/"+csiClusters, []byte(data), 0644)
if err != nil {
t.Errorf("Test setup error %s", err)
}
// TEST: Should fail as monitor data is malformed
content, err = Mons(pathToConfig, clusterID2)
if err == nil {
t.Errorf("Failed: want (%s), got (%s)", data, content)
}
data = "[{\"clusterID\":\"" + clusterID2 + "\",\"monitors\":[\"mon1\",\"mon2\",\"mon3\"]}]"
err = ioutil.WriteFile(basePath+"/"+csiClusters, []byte(data), 0644)
if err != nil {
t.Errorf("Test setup error %s", err)
}
// TEST: Should fail as clusterID is not present in config
content, err = Mons(pathToConfig, clusterID1)
if err == nil {
t.Errorf("Failed: want (%s), got (%s)", data, content)
}
// TEST: Should pass as clusterID is present in config
content, err = Mons(pathToConfig, clusterID2)
if err != nil || content != "mon1,mon2,mon3" {
t.Errorf("Failed: want (%s), got (%s) (%v)", "mon1,mon2,mon3", content, err)
}
data = "[{\"clusterID\":\"" + clusterID2 + "\",\"monitors\":[\"mon1\",\"mon2\",\"mon3\"]}," +
"{\"clusterID\":\"" + clusterID1 + "\",\"monitors\":[\"mon4\",\"mon5\",\"mon6\"]}]"
err = ioutil.WriteFile(basePath+"/"+csiClusters, []byte(data), 0644)
if err != nil {
t.Errorf("Test setup error %s", err)
}
// TEST: Should pass as clusterID is present in config
content, err = Mons(pathToConfig, clusterID1)
if err != nil || content != "mon4,mon5,mon6" {
t.Errorf("Failed: want (%s), got (%s) (%v)", "mon4,mon5,mon6", content, err)
}
}

47
pkg/util/errors.go Normal file
View File

@ -0,0 +1,47 @@
/*
Copyright 2019 The Ceph-CSI Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package util
// ErrKeyNotFound is returned when requested key in omap is not found
type ErrKeyNotFound struct {
keyName string
err error
}
func (e ErrKeyNotFound) Error() string {
return e.err.Error()
}
// ErrObjectExists is returned when named omap is already present in rados
type ErrObjectExists struct {
objectName string
err error
}
func (e ErrObjectExists) Error() string {
return e.err.Error()
}
// ErrObjectNotFound is returned when named omap is not found in rados
type ErrObjectNotFound struct {
oMapName string
err error
}
func (e ErrObjectNotFound) Error() string {
return e.err.Error()
}

57
pkg/util/fileconfig.go
View File

@ -1,57 +0,0 @@
/*
Copyright 2019 ceph-csi authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package util
import (
"fmt"
"io/ioutil"
"path"
)
/*
FileConfig is a ConfigStore interface implementation that reads configuration
information from files.
BasePath defines the directory under which FileConfig will attempt to open and
read contents of various Ceph cluster configurations.
Each Ceph cluster configuration is stored under a directory named,
BasePath/ceph-cluster-<clusterid>, where <clusterid> uniquely identifies and
separates the each Ceph cluster configuration.
Under each Ceph cluster configuration directory, individual files named as per
the ConfigKeys constants in the ConfigStore interface, store the required
configuration information.
*/
type FileConfig struct {
BasePath string
}
// DataForKey reads the appropriate config file, named using key, and returns
// the contents of the file to the caller
func (fc *FileConfig) DataForKey(clusterid, key string) (data string, err error) {
pathToKey := path.Join(fc.BasePath, "ceph-cluster-"+clusterid, key)
// #nosec
content, err := ioutil.ReadFile(pathToKey)
if err != nil || string(content) == "" {
err = fmt.Errorf("error fetching configuration for cluster ID (%s). (%s)", clusterid, err)
return
}
data = string(content)
return
}

58
pkg/util/k8sconfig.go
View File

@ -1,58 +0,0 @@
/*
Copyright 2019 ceph-csi authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package util
import (
"fmt"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
k8s "k8s.io/client-go/kubernetes"
)
/*
K8sConfig is a ConfigStore interface implementation that reads configuration
information from k8s secrets.
Each Ceph cluster configuration secret is expected to be named,
ceph-cluster-<clusterid>, where <clusterid> uniquely identifies and
separates the each Ceph cluster configuration.
The secret is expected to contain keys, as defined by the ConfigKeys constants
in the ConfigStore interface.
*/
type K8sConfig struct {
Client *k8s.Clientset
Namespace string
}
// DataForKey reads the appropriate k8s secret, named using clusterid, and
// returns the contents of key within the secret
func (kc *K8sConfig) DataForKey(clusterid, key string) (data string, err error) {
secret, err := kc.Client.CoreV1().Secrets(kc.Namespace).Get("ceph-cluster-"+clusterid, metav1.GetOptions{})
if err != nil {
err = fmt.Errorf("error fetching configuration for cluster ID (%s). (%s)", clusterid, err)
return
}
content, ok := secret.Data[key]
if !ok {
err = fmt.Errorf("missing data for key (%s) in cluster configuration of (%s)", key, clusterid)
return
}
data = string(content)
return
}

151
pkg/util/volid.go Normal file
View File

@ -0,0 +1,151 @@
/*
Copyright 2019 Ceph-CSI authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package util
import (
"encoding/binary"
"encoding/hex"
"errors"
"strings"
)
/*
CSIIdentifier contains the elements that form a CSI ID to be returned by the CSI plugin, and
contains enough information to decompose and extract required cluster and pool information to locate
the volume that relates to the CSI ID.
The CSI identifier is composed as elaborated in the comment against ComposeCSIID and thus,
DecomposeCSIID is the inverse of the same function.
The CSIIdentifier structure carries the following fields,
- PoolID: 64 bit integer of the pool that the volume belongs to, where the ID comes from Ceph pool
identifier for the corresponding pool name.
- EncodingVersion: Carries the version number of the encoding scheme used to encode the CSI ID,
and is preserved for any future proofing w.r.t changes in the encoding scheme, and to retain
ability to parse backward compatible encodings.
- ClusterID: Is a unique ID per cluster that the CSI instance is serving and is restricted to
lengths that can be accommodated in the encoding scheme.
- ObjectUUID: Is the on-disk uuid of the object (image/snapshot) name, for the CSI volume that
corresponds to this CSI ID.
*/
type CSIIdentifier struct {
PoolID int64 // TODO: Name appropriately when reused for CephFS
EncodingVersion uint16
ClusterID string
ObjectUUID string
}
// This maximum comes from the CSI spec on max bytes allowed in the various CSI ID fields
const maxVolIDLen = 128
/*
ComposeCSIID composes a CSI ID from passed in parameters.
Version 1 of the encoding scheme is as follows,
[csi_id_version=1:4byte] + [-:1byte]
[length of clusterID=1:4byte] + [-:1byte]
[clusterID:36bytes (MAX)] + [-:1byte]
[poolID:16bytes] + [-:1byte]
[ObjectUUID:36bytes]
Total of constant field lengths, including '-' field separators would hence be,
4+1+4+1+1+16+1+36 = 64
*/
const (
knownFieldSize = 64
uuidSize = 36
)
func (ci CSIIdentifier) ComposeCSIID() (string, error) {
buf16 := make([]byte, 2)
buf64 := make([]byte, 8)
if (knownFieldSize + len(ci.ClusterID)) > maxVolIDLen {
return "", errors.New("CSI ID encoding length overflow")
}
if len(ci.ObjectUUID) != uuidSize {
return "", errors.New("CSI ID invalid object uuid")
}
binary.BigEndian.PutUint16(buf16, ci.EncodingVersion)
versionEncodedHex := hex.EncodeToString(buf16)
binary.BigEndian.PutUint16(buf16, uint16(len(ci.ClusterID)))
clusterIDLength := hex.EncodeToString(buf16)
binary.BigEndian.PutUint64(buf64, uint64(ci.PoolID))
poolIDEncodedHex := hex.EncodeToString(buf64)
return strings.Join([]string{versionEncodedHex, clusterIDLength, ci.ClusterID,
poolIDEncodedHex, ci.ObjectUUID}, "-"), nil
}
/*
DecomposeCSIID composes a CSIIdentifier from passed in string
*/
func (ci *CSIIdentifier) DecomposeCSIID(composedCSIID string) (err error) {
bytesToProcess := uint16(len(composedCSIID))
// if length is less that expected constant elements, then bail out!
if bytesToProcess < knownFieldSize {
return errors.New("failed to decode CSI identifier, string underflow")
}
buf16, err := hex.DecodeString(composedCSIID[0:4])
if err != nil {
return err
}
ci.EncodingVersion = binary.BigEndian.Uint16(buf16)
// 4 for version encoding and 1 for '-' separator
bytesToProcess -= 5
buf16, err = hex.DecodeString(composedCSIID[5:9])
if err != nil {
return err
}
clusterIDLength := binary.BigEndian.Uint16(buf16)
// 4 for length encoding and 1 for '-' separator
bytesToProcess -= 5
if bytesToProcess < (clusterIDLength + 1) {
return errors.New("failed to decode CSI identifier, string underflow")
}
ci.ClusterID = composedCSIID[10 : 10+clusterIDLength]
// additional 1 for '-' separator
bytesToProcess -= (clusterIDLength + 1)
nextFieldStartIdx := 10 + clusterIDLength + 1
if bytesToProcess < 17 {
return errors.New("failed to decode CSI identifier, string underflow")
}
buf64, err := hex.DecodeString(composedCSIID[nextFieldStartIdx : nextFieldStartIdx+16])
if err != nil {
return err
}
ci.PoolID = int64(binary.BigEndian.Uint64(buf64))
// 16 for poolID encoding and 1 for '-' separator
bytesToProcess -= 17
nextFieldStartIdx = nextFieldStartIdx + 17
// has to be an exact match
if bytesToProcess != uuidSize {
return errors.New("failed to decode CSI identifier, string size mismatch")
}
ci.ObjectUUID = composedCSIID[nextFieldStartIdx : nextFieldStartIdx+uuidSize]
return err
}

95
pkg/util/volid_test.go Normal file
View File

@ -0,0 +1,95 @@
/*
Copyright 2019 Ceph-CSI authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package util
import (
"testing"
)
type testTuple struct {
vID CSIIdentifier
composedVolID string
wantEnc bool
wantEncError bool
wantDec bool
wantDecError bool
}
// TODO: Add more test tuples to test out other edge conditions
var testData = []testTuple{
{
vID: CSIIdentifier{
PoolID: 0xffff,
EncodingVersion: 0xffff,
ClusterID: "01616094-9d93-4178-bf45-c7eac19e8b15",
ObjectUUID: "00000000-1111-2222-bbbb-cacacacacaca",
},
composedVolID: "ffff-0024-01616094-9d93-4178-bf45-c7eac19e8b15-000000000000ffff-00000000-1111-2222-bbbb-cacacacacaca",
wantEnc: true,
wantEncError: false,
wantDec: true,
wantDecError: false,
},
}
func TestComposeDecomposeID(t *testing.T) {
var (
err error
viDecompose CSIIdentifier
composedVolID string
)
for _, test := range testData {
if test.wantEnc {
composedVolID, err = test.vID.ComposeCSIID()
if err != nil && !test.wantEncError {
t.Errorf("Composing failed: want (%#v), got (%#v %#v)",
test, composedVolID, err)
}
if err == nil && test.wantEncError {
t.Errorf("Composing failed: want (%#v), got (%#v %#v)",
test, composedVolID, err)
}
if !test.wantEncError && err == nil && composedVolID != test.composedVolID {
t.Errorf("Composing failed: want (%#v), got (%#v %#v)",
test, composedVolID, err)
}
}
if test.wantDec {
err = viDecompose.DecomposeCSIID(test.composedVolID)
if err != nil && !test.wantDecError {
t.Errorf("Decomposing failed: want (%#v), got (%#v %#v)",
test, viDecompose, err)
}
if err == nil && test.wantDecError {
t.Errorf("Decomposing failed: want (%#v), got (%#v %#v)",
test, viDecompose, err)
}
if !test.wantDecError && err == nil && viDecompose != test.vID {
t.Errorf("Decomposing failed: want (%#v), got (%#v %#v)",
test, viDecompose, err)
}
}
}
}