diff --git a/.mergify.yml b/.mergify.yml new file mode 100644 index 000000000..d74b5199e --- /dev/null +++ b/.mergify.yml @@ -0,0 +1,14 @@ +--- +pull_request_rules: + - name: automatic merge + conditions: + - label!=DNM + - '#approved-reviews-by>=1' + - 'status-success=continuous-integration/travis-ci/pr' + actions: + merge: + method: rebase + rebase_fallback: merge + strict: smart + dismiss_reviews: {} + delete_head_branch: {} diff --git a/.travis.yml b/.travis.yml index 9c4d7f76a..d8fe27f78 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,14 +11,15 @@ addons: language: go branches: only: + - csi-v0.3 - master - - csi-v1.0 + - csi-v1.0 # remove this once csi-v1.0 becomes master go: 1.11.x env: global: - - GO_METALINTER_VERSION="v3.0.0" + - GOLANGCI_VERSION="v1.15.0" - TEST_COVERAGE=stdout - GO_METALINTER_THREADS=1 - GO_COVER_DIR=_output @@ -30,10 +31,10 @@ jobs: - gem install mdl - pip install --user --upgrade pip - pip install --user yamllint - # install gometalinter - - curl -L - "https://raw.githubusercontent.com/alecthomas/gometalinter/"${GO_METALINTER_VERSION}"/scripts/install.sh" - | bash -s -- -b $GOPATH/bin "${GO_METALINTER_VERSION}" + # install golangci-lint + - curl -sf + "https://install.goreleaser.com/github.com/golangci/golangci-lint.sh" + | bash -s -- -b $GOPATH/bin "${GOLANGCI_VERSION}" script: - scripts/lint-text.sh --require-all - scripts/lint-go.sh diff --git a/Gopkg.lock b/Gopkg.lock index f6322c559..d3270adf6 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -20,14 +20,6 @@ revision = "4cbf7e384e768b4e01799441fdf2a706a5635ae7" version = "v1.2.0" -[[projects]] - branch = "master" - digest = "1:e2b86e41f3d669fc36b50d31d32d22c8ac656c75aa5ea89717ce7177e134ff2a" - name = "github.com/golang/glog" - packages = ["."] - pruneopts = "NUT" - revision = "23def4e6c14b4da8ac2ed8007337bc5eb5007998" - [[projects]] digest = "1:bff0ce7c8e3d6357fa5a8549bbe4bdb620bddc13c11ae569aa7248ea92e2139f" name = "github.com/golang/protobuf" @@ -116,14 +108,6 @@ revision = "5853414e1d4771302e0df10d1870c444c2135799" version = "v0.2.0" -[[projects]] - branch = "master" - digest = "1:0bde3fb932a1aa4e12bc43ef91157fcda27dd0fc5d9f309647544ceaec075f48" - name = "github.com/kubernetes-csi/drivers" - packages = ["pkg/csi-common"] - pruneopts = "NUT" - revision = "05e1ea84df03b90296869812fa42f4244bd5ab53" - [[projects]] digest = "1:2f42fa12d6911c7b7659738758631bec870b7e9b4c6be5444f963cdcfccc191f" name = "github.com/modern-go/concurrent" @@ -376,11 +360,11 @@ "storage/v1beta1", ] pruneopts = "NUT" - revision = "67edc246be36579e46a89e29a2f165d47e012109" - version = "kubernetes-1.13.2" + revision = "74b699b93c15473932b89e3d1818ba8282f3b5ab" + version = "kubernetes-1.13.3" [[projects]] - digest = "1:a2da0cbc8dfda27eeffa54b53195e607497c6cac737d17f45a667963aeae5f02" + digest = "1:09dee8b7c6cb2fc9c6bee525de3b95199a82a8647a189e153d072a1dfce17de7" name = "k8s.io/apimachinery" packages = [ "pkg/api/errors", @@ -421,8 +405,8 @@ "third_party/forked/golang/reflect", ] pruneopts = "NUT" - revision = "2b1284ed4c93a43499e781493253e2ac5959c4fd" - version = "kubernetes-1.13.2" + revision = "572dfc7bdfcb4531361a17d27b92851f59acf0dc" + version = "kubernetes-1.13.3" [[projects]] digest = "1:638623327cb201b425a328d0bddb3379b05eb05ef4cab589380f0be07ac1dc17" @@ -485,8 +469,8 @@ "util/integer", ] pruneopts = "NUT" - revision = "6bf63545bd0257ed9e701ad95307ffa51b4407c0" - version = "kubernetes-1.13.2" + revision = "6e4752048fde21176ab35eb54ec1117359830d8a" + version = "kubernetes-1.13.3" [[projects]] digest = "1:9cc257b3c9ff6a0158c9c661ab6eebda1fe8a4a4453cd5c4044dc9a2ebfb992b" @@ -507,8 +491,8 @@ "pkg/util/nsenter", ] pruneopts = "NUT" - revision = "cff46ab41ff0bb44d8584413b598ad8360ec1def" - version = "v1.13.2" + revision = "721bfa751924da8d1680787490c54b9179b1fed0" + version = "v1.13.3" [[projects]] branch = "master" @@ -533,10 +517,11 @@ "github.com/container-storage-interface/spec/lib/go/csi", "github.com/golang/protobuf/ptypes", "github.com/golang/protobuf/ptypes/timestamp", - "github.com/kubernetes-csi/drivers/pkg/csi-common", + "github.com/kubernetes-csi/csi-lib-utils/protosanitizer", "github.com/pborman/uuid", "github.com/pkg/errors", "golang.org/x/net/context", + "google.golang.org/grpc", "google.golang.org/grpc/codes", "google.golang.org/grpc/status", "k8s.io/api/core/v1", diff --git a/Gopkg.toml b/Gopkg.toml index 6a0a2d8c5..04deea00b 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -2,10 +2,6 @@ name = "github.com/container-storage-interface/spec" version = "~1.0.0" -[[constraint]] - branch = "master" - name = "github.com/kubernetes-csi/drivers" - [[override]] revision = "5db89f0ca68677abc5eefce8f2a0a772c98ba52d" name = "github.com/docker/distribution" @@ -15,15 +11,15 @@ version = "1.10.0" [[constraint]] - version = "kubernetes-1.13.2" + version = "kubernetes-1.13.3" name = "k8s.io/apimachinery" [[constraint]] name = "k8s.io/kubernetes" - version = "v1.13.2" + version = "v1.13.3" [[override]] - version = "kubernetes-1.13.2" + version = "kubernetes-1.13.3" name = "k8s.io/api" [[override]] @@ -32,7 +28,7 @@ [[constraint]] name = "k8s.io/client-go" - version = "kubernetes-1.13.2" + version = "kubernetes-1.13.3" [prune] go-tests = true diff --git a/Makefile b/Makefile index 734a761b8..8451b380a 100644 --- a/Makefile +++ b/Makefile @@ -14,12 +14,17 @@ .PHONY: all rbdplugin cephfsplugin +CONTAINER_CMD?=docker + RBD_IMAGE_NAME=$(if $(ENV_RBD_IMAGE_NAME),$(ENV_RBD_IMAGE_NAME),quay.io/cephcsi/rbdplugin) RBD_IMAGE_VERSION=$(if $(ENV_RBD_IMAGE_VERSION),$(ENV_RBD_IMAGE_VERSION),v1.0.0) CEPHFS_IMAGE_NAME=$(if $(ENV_CEPHFS_IMAGE_NAME),$(ENV_CEPHFS_IMAGE_NAME),quay.io/cephcsi/cephfsplugin) CEPHFS_IMAGE_VERSION=$(if $(ENV_CEPHFS_IMAGE_VERSION),$(ENV_CEPHFS_IMAGE_VERSION),v1.0.0) +CSI_IMAGE_NAME?=quay.io/cephcsi/cephcsi +CSI_IMAGE_VERSION?=v1.0.0 + $(info rbd image settings: $(RBD_IMAGE_NAME) version $(RBD_IMAGE_VERSION)) $(info cephfs image settings: $(CEPHFS_IMAGE_NAME) version $(CEPHFS_IMAGE_VERSION)) @@ -31,30 +36,31 @@ go-test: ./scripts/test-go.sh static-check: - ./scripts/lint-go.sh + ./scripts/lint-go.sh ./scripts/lint-text.sh -rbdplugin: +.PHONY: cephcsi +cephcsi: if [ ! -d ./vendor ]; then dep ensure -vendor-only; fi - CGO_ENABLED=0 GOOS=linux go build -a -ldflags '-extldflags "-static"' -o _output/rbdplugin ./cmd/rbd + CGO_ENABLED=0 GOOS=linux go build -a -ldflags '-extldflags "-static"' -o _output/cephcsi ./cmd/ -image-rbdplugin: rbdplugin - cp _output/rbdplugin deploy/rbd/docker - docker build -t $(RBD_IMAGE_NAME):$(RBD_IMAGE_VERSION) deploy/rbd/docker +image-cephcsi: cephcsi + cp deploy/cephcsi/image/Dockerfile _output + $(CONTAINER_CMD) build -t $(CSI_IMAGE_NAME):$(CSI_IMAGE_VERSION) _output -cephfsplugin: - if [ ! -d ./vendor ]; then dep ensure -vendor-only; fi - CGO_ENABLED=0 GOOS=linux go build -a -ldflags '-extldflags "-static"' -o _output/cephfsplugin ./cmd/cephfs +image-rbdplugin: cephcsi + cp _output/cephcsi deploy/rbd/docker/rbdplugin + $(CONTAINER_CMD) build -t $(RBD_IMAGE_NAME):$(RBD_IMAGE_VERSION) deploy/rbd/docker -image-cephfsplugin: cephfsplugin - cp _output/cephfsplugin deploy/cephfs/docker - docker build -t $(CEPHFS_IMAGE_NAME):$(CEPHFS_IMAGE_VERSION) deploy/cephfs/docker +image-cephfsplugin: cephcsi + cp _output/cephsci deploy/cephfs/docker/cephfsplugin + $(CONTAINER_CMD) build -t $(CEPHFS_IMAGE_NAME):$(CEPHFS_IMAGE_VERSION) deploy/cephfs/docker push-image-rbdplugin: image-rbdplugin - docker push $(RBD_IMAGE_NAME):$(RBD_IMAGE_VERSION) + $(CONTAINER_CMD) push $(RBD_IMAGE_NAME):$(RBD_IMAGE_VERSION) push-image-cephfsplugin: image-cephfsplugin - docker push $(CEPHFS_IMAGE_NAME):$(CEPHFS_IMAGE_VERSION) + $(CONTAINER_CMD) push $(CEPHFS_IMAGE_NAME):$(CEPHFS_IMAGE_VERSION) clean: go clean -r -x diff --git a/cmd/cephcsi.go b/cmd/cephcsi.go new file mode 100644 index 000000000..315b4fa9c --- /dev/null +++ b/cmd/cephcsi.go @@ -0,0 +1,133 @@ +/* +Copyright 2019 The Ceph-CSI Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "os" + "path" + "strings" + + "github.com/ceph/ceph-csi/pkg/cephfs" + "github.com/ceph/ceph-csi/pkg/rbd" + "github.com/ceph/ceph-csi/pkg/util" + "k8s.io/klog" +) + +const ( + rbdType = "rbd" + cephfsType = "cephfs" + + rbdDefaultName = "rbd.csi.ceph.com" + cephfsDefaultName = "cephfs.csi.ceph.com" +) + +var ( + // common flags + vtype = flag.String("type", "", "driver type [rbd|cephfs]") + endpoint = flag.String("endpoint", "unix://tmp/csi.sock", "CSI endpoint") + driverName = flag.String("drivername", "", "name of the driver") + nodeID = flag.String("nodeid", "", "node id") + metadataStorage = flag.String("metadatastorage", "", "metadata persistence method [node|k8s_configmap]") + + // rbd related flags + containerized = flag.Bool("containerized", true, "whether run as containerized") + configRoot = flag.String("configroot", "/etc/csi-config", "directory in which CSI specific Ceph"+ + " cluster configurations are present, OR the value \"k8s_objects\" if present as kubernetes secrets") + + // cephfs related flags + volumeMounter = flag.String("volumemounter", "", "default volume mounter (possible options are 'kernel', 'fuse')") + mountCacheDir = flag.String("mountcachedir", "", "mount info cache save dir") +) + +func init() { + klog.InitFlags(nil) + if err := flag.Set("logtostderr", "true"); err != nil { + klog.Exitf("failed to set logtostderr flag: %v", err) + } + flag.Parse() +} + +func getType() string { + if vtype == nil || len(*vtype) == 0 { + a0 := path.Base(os.Args[0]) + if strings.Contains(a0, rbdType) { + return rbdType + } + if strings.Contains(a0, cephfsType) { + return cephfsType + } + return "" + } + return *vtype +} + +func getDriverName() string { + // was explicitly passed a driver name + if driverName != nil && len(*driverName) != 0 { + return *driverName + } + // select driver name based on volume type + switch getType() { + case rbdType: + return rbdDefaultName + case cephfsType: + return cephfsDefaultName + default: + return "" + } +} + +func main() { + driverType := getType() + if len(driverType) == 0 { + klog.Fatalln("driver type not specified") + } + + dname := getDriverName() + err := util.ValidateDriverName(dname) + if err != nil { + klog.Fatalln(err) // calls exit + } + + switch driverType { + case rbdType: + rbd.PluginFolder = rbd.PluginFolder + dname + cp, err := util.CreatePersistanceStorage( + rbd.PluginFolder, *metadataStorage, dname) + if err != nil { + os.Exit(1) + } + driver := rbd.NewDriver() + driver.Run(dname, *nodeID, *endpoint, *configRoot, *containerized, cp) + + case cephfsType: + cephfs.PluginFolder = cephfs.PluginFolder + dname + cp, err := util.CreatePersistanceStorage( + cephfs.PluginFolder, *metadataStorage, dname) + if err != nil { + os.Exit(1) + } + driver := cephfs.NewDriver() + driver.Run(dname, *nodeID, *endpoint, *volumeMounter, *mountCacheDir, cp) + + default: + klog.Fatalln("invalid volume type", vtype) // calls exit + } + + os.Exit(0) +} diff --git a/cmd/cephfs/main.go b/cmd/cephfs/main.go index 907246f8e..a4d01a5bc 100644 --- a/cmd/cephfs/main.go +++ b/cmd/cephfs/main.go @@ -19,7 +19,6 @@ package main import ( "flag" "os" - "path" "github.com/ceph/ceph-csi/pkg/cephfs" "github.com/ceph/ceph-csi/pkg/util" @@ -28,37 +27,37 @@ import ( var ( endpoint = flag.String("endpoint", "unix://tmp/csi.sock", "CSI endpoint") - driverName = flag.String("drivername", "csi-cephfsplugin", "name of the driver") + driverName = flag.String("drivername", "cephfs.csi.ceph.com", "name of the driver") nodeID = flag.String("nodeid", "", "node id") volumeMounter = flag.String("volumemounter", "", "default volume mounter (possible options are 'kernel', 'fuse')") metadataStorage = flag.String("metadatastorage", "", "metadata persistence method [node|k8s_configmap]") + mountCacheDir = flag.String("mountcachedir", "", "mount info cache save dir") ) +func init() { + klog.InitFlags(nil) + if err := flag.Set("logtostderr", "true"); err != nil { + klog.Exitf("failed to set logtostderr flag: %v", err) + } + flag.Parse() +} + func main() { - util.InitLogging() - if err := createPersistentStorage(path.Join(cephfs.PluginFolder, "controller")); err != nil { - klog.Errorf("failed to create persistent storage for controller: %v", err) - os.Exit(1) - } - - if err := createPersistentStorage(path.Join(cephfs.PluginFolder, "node")); err != nil { - klog.Errorf("failed to create persistent storage for node: %v", err) - os.Exit(1) - } - - cp, err := util.NewCachePersister(*metadataStorage, *driverName) + err := util.ValidateDriverName(*driverName) + if err != nil { + klog.Fatalln(err) + } + //update plugin name + cephfs.PluginFolder = cephfs.PluginFolder + *driverName + + cp, err := util.CreatePersistanceStorage(cephfs.PluginFolder, *metadataStorage, *driverName) if err != nil { - klog.Errorf("failed to define cache persistence method: %v", err) os.Exit(1) } driver := cephfs.NewDriver() - driver.Run(*driverName, *nodeID, *endpoint, *volumeMounter, cp) + driver.Run(*driverName, *nodeID, *endpoint, *volumeMounter, *mountCacheDir, cp) os.Exit(0) } - -func createPersistentStorage(persistentStoragePath string) error { - return os.MkdirAll(persistentStoragePath, os.FileMode(0755)) -} diff --git a/cmd/rbd/main.go b/cmd/rbd/main.go index cd30bef68..bc4ddf53e 100644 --- a/cmd/rbd/main.go +++ b/cmd/rbd/main.go @@ -19,7 +19,6 @@ package main import ( "flag" "os" - "path" "github.com/ceph/ceph-csi/pkg/rbd" "github.com/ceph/ceph-csi/pkg/util" @@ -28,43 +27,38 @@ import ( var ( endpoint = flag.String("endpoint", "unix://tmp/csi.sock", "CSI endpoint") - driverName = flag.String("drivername", "csi-rbdplugin", "name of the driver") + driverName = flag.String("drivername", "rbd.csi.ceph.com", "name of the driver") nodeID = flag.String("nodeid", "", "node id") containerized = flag.Bool("containerized", true, "whether run as containerized") metadataStorage = flag.String("metadatastorage", "", "metadata persistence method [node|k8s_configmap]") + configRoot = flag.String("configroot", "/etc/csi-config", "directory in which CSI specific Ceph"+ + " cluster configurations are present, OR the value \"k8s_objects\" if present as kubernetes secrets") ) +func init() { + klog.InitFlags(nil) + if err := flag.Set("logtostderr", "true"); err != nil { + klog.Exitf("failed to set logtostderr flag: %v", err) + } + flag.Parse() +} + func main() { - util.InitLogging() - if err := createPersistentStorage(path.Join(rbd.PluginFolder, "controller")); err != nil { - klog.Errorf("failed to create persistent storage for controller %v", err) - os.Exit(1) - } - if err := createPersistentStorage(path.Join(rbd.PluginFolder, "node")); err != nil { - klog.Errorf("failed to create persistent storage for node %v", err) - os.Exit(1) - } - - cp, err := util.NewCachePersister(*metadataStorage, *driverName) + err := util.ValidateDriverName(*driverName) + if err != nil { + klog.Fatalln(err) + } + //update plugin name + rbd.PluginFolder = rbd.PluginFolder + *driverName + + cp, err := util.CreatePersistanceStorage(rbd.PluginFolder, *metadataStorage, *driverName) if err != nil { - klog.Errorf("failed to define cache persistence method: %v", err) os.Exit(1) } driver := rbd.NewDriver() - driver.Run(*driverName, *nodeID, *endpoint, *containerized, cp) + driver.Run(*driverName, *nodeID, *endpoint, *configRoot, *containerized, cp) os.Exit(0) } - -func createPersistentStorage(persistentStoragePath string) error { - if _, err := os.Stat(persistentStoragePath); os.IsNotExist(err) { - if err = os.MkdirAll(persistentStoragePath, os.FileMode(0755)); err != nil { - return err - } - } else { - return err - } - return nil -} diff --git a/deploy.sh b/deploy.sh index 9ce90e442..9eb02b2a8 100755 --- a/deploy.sh +++ b/deploy.sh @@ -1,6 +1,30 @@ #!/bin/bash -if [ "${TRAVIS_BRANCH}" == 'master' ]; then +push_helm_chats() { + PACKAGE=$1 + CHANGED=0 + VERSION=$(grep 'version:' deploy/"$PACKAGE"/helm/Chart.yaml | awk '{print $2}') + + if [ ! -f "tmp/csi-charts/docs/$PACKAGE/ceph-csi-$PACKAGE-$VERSION.tgz" ]; then + CHANGED=1 + ln -s helm deploy/"$PACKAGE"/ceph-csi-"$PACKAGE" + mkdir -p tmp/csi-charts/docs/"$PACKAGE" + pushd tmp/csi-charts/docs/"$PACKAGE" >/dev/null + helm init --client-only + helm package ../../../../deploy/"$PACKAGE"/ceph-csi-"$PACKAGE" + popd >/dev/null + fi + + if [ $CHANGED -eq 1 ]; then + pushd tmp/csi-charts/docs >/dev/null + helm repo index . + git add --all :/ && git commit -m "Update repo" + git push https://"$GITHUB_TOKEN"@github.com/ceph/csi-charts + popd >/dev/null + fi +} + +if [ "${TRAVIS_BRANCH}" == 'csi-v0.3' ]; then export RBD_IMAGE_VERSION='v0.3.0' export CEPHFS_IMAGE_VERSION='v0.3.0' elif [ "${TRAVIS_BRANCH}" == 'csi-v1.0' ]; then @@ -12,7 +36,7 @@ else fi if [ "${TRAVIS_PULL_REQUEST}" == "false" ]; then - docker login -u "${QUAY_IO_USERNAME}" -p "${QUAY_IO_PASSWORD}" quay.io + "${CONTAINER_CMD:-docker}" login -u "${QUAY_IO_USERNAME}" -p "${QUAY_IO_PASSWORD}" quay.io make push-image-rbdplugin push-image-cephfsplugin set -xe @@ -29,25 +53,6 @@ if [ "${TRAVIS_PULL_REQUEST}" == "false" ]; then mkdir -p csi-charts/docs popd >/dev/null - CHANGED=0 - VERSION=$(grep 'version:' deploy/rbd/helm/Chart.yaml | awk '{print $2}') - - if [ ! -f "tmp/csi-charts/docs/rbd/ceph-csi-rbd-$VERSION.tgz" ]; then - CHANGED=1 - ln -s helm deploy/rbd/ceph-csi-rbd - mkdir -p tmp/csi-charts/docs/rbd - pushd tmp/csi-charts/docs/rbd >/dev/null - helm init --client-only - helm package ../../../../deploy/rbd/ceph-csi-rbd - popd >/dev/null - fi - - if [ $CHANGED -eq 1 ]; then - pushd tmp/csi-charts/docs >/dev/null - helm repo index . - git add --all :/ && git commit -m "Update repo" - git push https://"$GITHUB_TOKEN"@github.com/ceph/csi-charts - popd >/dev/null - fi - + push_helm_chats rbd + push_helm_chats cephfs fi diff --git a/deploy/cephcsi/image/Dockerfile b/deploy/cephcsi/image/Dockerfile new file mode 100644 index 000000000..3a11c225d --- /dev/null +++ b/deploy/cephcsi/image/Dockerfile @@ -0,0 +1,14 @@ + +FROM ceph/ceph:v14.2 +LABEL maintainers="Ceph-CSI Authors" +LABEL description="Ceph-CSI Plugin" + +ENV CSIBIN=/usr/local/bin/cephcsi + +COPY cephcsi $CSIBIN + +RUN chmod +x $CSIBIN && \ + ln -sf $CSIBIN /usr/local/bin/cephcsi-rbd && \ + ln -sf $CSIBIN /usr/local/bin/cephcsi-cephfs + +ENTRYPOINT ["/usr/local/bin/cephcsi"] diff --git a/deploy/cephfs/helm/.helmignore b/deploy/cephfs/helm/.helmignore new file mode 100644 index 000000000..f0c131944 --- /dev/null +++ b/deploy/cephfs/helm/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/deploy/cephfs/helm/Chart.yaml b/deploy/cephfs/helm/Chart.yaml new file mode 100644 index 000000000..f324e7f57 --- /dev/null +++ b/deploy/cephfs/helm/Chart.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: v1 +appVersion: "1.0.0" +description: "Container Storage Interface (CSI) driver, +provisioner, and attacher for Ceph cephfs" +name: ceph-csi-cephfs +version: 0.5.1 +keywords: + - ceph + - cephfs + - ceph-csi +home: https://github.com/ceph/ceph-csi +sources: + - https://github.com/ceph/ceph-csi/tree/csi-v1.0/deploy/cephfs/helm diff --git a/deploy/cephfs/helm/README.md b/deploy/cephfs/helm/README.md new file mode 100644 index 000000000..fcd4a98a8 --- /dev/null +++ b/deploy/cephfs/helm/README.md @@ -0,0 +1,29 @@ +# ceph-csi-cephfs + +The ceph-csi-cephfs chart adds cephfs volume support to your cluster. + +## Install Chart + +To install the Chart into your Kubernetes cluster + +```bash +helm install --namespace "ceph-csi-cephfs" --name "ceph-csi-cephfs" ceph-csi/ceph-csi-cephfs +``` + +After installation succeeds, you can get a status of Chart + +```bash +helm status "ceph-csi-cephfs" +``` + +If you want to delete your Chart, use this command + +```bash +helm delete --purge "ceph-csi-cephfs" +``` + +If you want to delete the namespace, use this command + +```bash +kubectl delete namespace ceph-csi-rbd +``` diff --git a/deploy/cephfs/helm/templates/NOTES.txt b/deploy/cephfs/helm/templates/NOTES.txt new file mode 100644 index 000000000..3af9f2b57 --- /dev/null +++ b/deploy/cephfs/helm/templates/NOTES.txt @@ -0,0 +1,2 @@ +Examples on how to configure a storage class and start using the driver are here: +https://github.com/ceph/ceph-csi/tree/csi-v1.0/examples/cephfs diff --git a/deploy/cephfs/helm/templates/_helpers.tpl b/deploy/cephfs/helm/templates/_helpers.tpl new file mode 100644 index 000000000..e604150ae --- /dev/null +++ b/deploy/cephfs/helm/templates/_helpers.tpl @@ -0,0 +1,119 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "ceph-csi-cephfs.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "ceph-csi-cephfs.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "ceph-csi-cephfs.attacher.fullname" -}} +{{- if .Values.attacher.fullnameOverride -}} +{{- .Values.attacher.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- printf "%s-%s" .Release.Name .Values.attacher.name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s-%s" .Release.Name $name .Values.attacher.name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "ceph-csi-cephfs.nodeplugin.fullname" -}} +{{- if .Values.nodeplugin.fullnameOverride -}} +{{- .Values.nodeplugin.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- printf "%s-%s" .Release.Name .Values.nodeplugin.name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s-%s" .Release.Name $name .Values.nodeplugin.name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "ceph-csi-cephfs.provisioner.fullname" -}} +{{- if .Values.provisioner.fullnameOverride -}} +{{- .Values.provisioner.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- printf "%s-%s" .Release.Name .Values.provisioner.name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s-%s" .Release.Name $name .Values.provisioner.name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "ceph-csi-cephfs.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "ceph-csi-cephfs.serviceAccountName.attacher" -}} +{{- if .Values.serviceAccounts.attacher.create -}} + {{ default (include "ceph-csi-cephfs.attacher.fullname" .) .Values.serviceAccounts.attacher.name }} +{{- else -}} + {{ default "default" .Values.serviceAccounts.attacher.name }} +{{- end -}} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "ceph-csi-cephfs.serviceAccountName.nodeplugin" -}} +{{- if .Values.serviceAccounts.nodeplugin.create -}} + {{ default (include "ceph-csi-cephfs.nodeplugin.fullname" .) .Values.serviceAccounts.nodeplugin.name }} +{{- else -}} + {{ default "default" .Values.serviceAccounts.nodeplugin.name }} +{{- end -}} +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "ceph-csi-cephfs.serviceAccountName.provisioner" -}} +{{- if .Values.serviceAccounts.provisioner.create -}} + {{ default (include "ceph-csi-cephfs.provisioner.fullname" .) .Values.serviceAccounts.provisioner.name }} +{{- else -}} + {{ default "default" .Values.serviceAccounts.provisioner.name }} +{{- end -}} +{{- end -}} diff --git a/deploy/cephfs/helm/templates/attacher-clusterrole.yaml b/deploy/cephfs/helm/templates/attacher-clusterrole.yaml new file mode 100644 index 000000000..a66256500 --- /dev/null +++ b/deploy/cephfs/helm/templates/attacher-clusterrole.yaml @@ -0,0 +1,28 @@ +{{- if .Values.rbac.create -}} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "ceph-csi-cephfs.attacher.fullname" . }} + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.attacher.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csinodeinfos"] + verbs: ["get", "list", "watch"] +{{- end -}} diff --git a/deploy/cephfs/helm/templates/attacher-clusterrolebinding.yaml b/deploy/cephfs/helm/templates/attacher-clusterrolebinding.yaml new file mode 100644 index 000000000..832e23dec --- /dev/null +++ b/deploy/cephfs/helm/templates/attacher-clusterrolebinding.yaml @@ -0,0 +1,20 @@ +{{- if .Values.rbac.create -}} +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "ceph-csi-cephfs.attacher.fullname" . }} + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.attacher.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +subjects: + - kind: ServiceAccount + name: {{ include "ceph-csi-cephfs.serviceAccountName.attacher" . }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "ceph-csi-cephfs.attacher.fullname" . }} + apiGroup: rbac.authorization.k8s.io +{{- end -}} diff --git a/deploy/cephfs/helm/templates/attacher-service.yaml b/deploy/cephfs/helm/templates/attacher-service.yaml new file mode 100644 index 000000000..379830d53 --- /dev/null +++ b/deploy/cephfs/helm/templates/attacher-service.yaml @@ -0,0 +1,18 @@ +kind: Service +apiVersion: v1 +metadata: + name: {{ include "ceph-csi-cephfs.attacher.fullname" . }} + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.attacher.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + selector: + app: {{ include "ceph-csi-cephfs.name" . }} + component: {{ .Values.attacher.name }} + release: {{ .Release.Name }} + ports: + - name: dummy + port: 12345 diff --git a/deploy/cephfs/helm/templates/attacher-serviceaccount.yaml b/deploy/cephfs/helm/templates/attacher-serviceaccount.yaml new file mode 100644 index 000000000..dbb70ccc2 --- /dev/null +++ b/deploy/cephfs/helm/templates/attacher-serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccounts.attacher.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "ceph-csi-cephfs.serviceAccountName.attacher" . }} + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.attacher.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +{{- end -}} diff --git a/deploy/cephfs/helm/templates/attacher-statefulset.yaml b/deploy/cephfs/helm/templates/attacher-statefulset.yaml new file mode 100644 index 000000000..88514d062 --- /dev/null +++ b/deploy/cephfs/helm/templates/attacher-statefulset.yaml @@ -0,0 +1,60 @@ +kind: StatefulSet +apiVersion: apps/v1beta1 +metadata: + name: {{ include "ceph-csi-cephfs.attacher.fullname" . }} + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.attacher.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + serviceName: {{ include "ceph-csi-cephfs.attacher.fullname" . }} + replicas: {{ .Values.attacher.replicas }} + selector: + matchLabels: + app: {{ include "ceph-csi-cephfs.name" . }} + component: {{ .Values.attacher.name }} + release: {{ .Release.Name }} + template: + metadata: + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.attacher.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + spec: + serviceAccountName: {{ include "ceph-csi-cephfs.serviceAccountName.attacher" . }} + containers: + - name: csi-cephfsplugin-attacher + image: "{{ .Values.attacher.image.repository }}:{{ .Values.attacher.image.tag }}" + args: + - "--v=5" + - "--csi-address=$(ADDRESS)" + env: + - name: ADDRESS + value: "{{ .Values.socketDir }}/{{ .Values.socketFile }}" + imagePullPolicy: {{ .Values.attacher.image.pullPolicy }} + volumeMounts: + - name: socket-dir + mountPath: {{ .Values.socketDir }} + resources: +{{ toYaml .Values.attacher.resources | indent 12 }} + volumes: + - name: socket-dir + hostPath: + path: {{ .Values.socketDir }} + type: DirectoryOrCreate + {{- if .Values.attacher.affinity -}} + affinity: +{{ toYaml .Values.attacher.affinity . | indent 8 }} + {{- end -}} + {{- if .Values.attacher.nodeSelector -}} + nodeSelector: +{{ toYaml .Values.attacher.nodeSelector | indent 8 }} + {{- end -}} + {{- if .Values.attacher.tolerations -}} + tolerations: +{{ toYaml .Values.attacher.tolerations | indent 8 }} + {{- end -}} diff --git a/deploy/cephfs/helm/templates/nodeplugin-clusterrole.yaml b/deploy/cephfs/helm/templates/nodeplugin-clusterrole.yaml new file mode 100644 index 000000000..de4aaeaaa --- /dev/null +++ b/deploy/cephfs/helm/templates/nodeplugin-clusterrole.yaml @@ -0,0 +1,31 @@ +{{- if .Values.rbac.create -}} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }} + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "update"] + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list"] +{{- end -}} diff --git a/deploy/cephfs/helm/templates/nodeplugin-clusterrolebinding.yaml b/deploy/cephfs/helm/templates/nodeplugin-clusterrolebinding.yaml new file mode 100644 index 000000000..24e21351c --- /dev/null +++ b/deploy/cephfs/helm/templates/nodeplugin-clusterrolebinding.yaml @@ -0,0 +1,20 @@ +{{- if .Values.rbac.create -}} +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }} + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +subjects: + - kind: ServiceAccount + name: {{ include "ceph-csi-cephfs.serviceAccountName.nodeplugin" . }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }} + apiGroup: rbac.authorization.k8s.io +{{- end -}} diff --git a/deploy/cephfs/helm/templates/nodeplugin-daemonset.yaml b/deploy/cephfs/helm/templates/nodeplugin-daemonset.yaml new file mode 100644 index 000000000..c56e70bb4 --- /dev/null +++ b/deploy/cephfs/helm/templates/nodeplugin-daemonset.yaml @@ -0,0 +1,150 @@ +kind: DaemonSet +apiVersion: apps/v1beta2 +metadata: + name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }} + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + selector: + matchLabels: + app: {{ include "ceph-csi-cephfs.name" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + template: + metadata: + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + spec: + serviceAccountName: {{ include "ceph-csi-cephfs.serviceAccountName.nodeplugin" . }} + hostNetwork: true + hostPID: true + # to use e.g. Rook orchestrated cluster, and mons' FQDN is + # resolved through k8s service, set dns policy to cluster first + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: driver-registrar + image: "{{ .Values.nodeplugin.registrar.image.repository }}:{{ .Values.nodeplugin.registrar.image.tag }}" + args: + - "--v=5" + - "--csi-address=/csi/{{ .Values.socketFile }}" + - "--kubelet-registration-path={{ .Values.socketDir }}/{{ .Values.socketFile }}" + lifecycle: + preStop: + exec: + command: [ + "/bin/sh", "-c", + 'rm -rf /registration/{{ .Values.driverName }} + /registration/{{ .Values.driverName }}-reg.sock' + ] + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + imagePullPolicy: {{ .Values.nodeplugin.registrar.image.imagePullPolicy }} + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + resources: +{{ toYaml .Values.nodeplugin.registrar.resources | indent 12 }} + - name: csi-cephfsplugin + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + image: "{{ .Values.nodeplugin.plugin.image.repository }}:{{ .Values.nodeplugin.plugin.image.tag }}" + args : + - "--nodeid=$(NODE_ID)" + - "--endpoint=$(CSI_ENDPOINT)" + - "--v=5" + - "--drivername=$(DRIVER_NAME)" + - "--metadatastorage=k8s_configmap" + - "--mountcachedir=/mount-cache-dir" + env: + - name: HOST_ROOTFS + value: "/rootfs" + - name: DRIVER_NAME + value: {{ .Values.driverName }} + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: "unix:/{{ .Values.socketDir }}/{{ .Values.socketFile }}" + imagePullPolicy: {{ .Values.nodeplugin.plugin.image.imagePullPolicy }} + volumeMounts: + - name: mount-cache-dir + mountPath: /mount-cache-dir + - name: plugin-dir + mountPath: {{ .Values.socketDir }} + - name: pods-mount-dir + mountPath: /var/lib/kubelet/pods + mountPropagation: "Bidirectional" + - name: plugin-mount-dir + mountPath: {{ .Values.volumeDevicesDir }} + mountPropagation: "Bidirectional" + - mountPath: /dev + name: host-dev + - mountPath: /rootfs + name: host-rootfs + - mountPath: /sys + name: host-sys + - mountPath: /lib/modules + name: lib-modules + readOnly: true + resources: +{{ toYaml .Values.nodeplugin.plugin.resources | indent 12 }} + volumes: + - name: mount-cache-dir + emptyDir: {} + - name: plugin-dir + hostPath: + path: {{ .Values.socketDir }} + type: DirectoryOrCreate + - name: plugin-mount-dir + hostPath: + path: {{ .Values.volumeDevicesDir }} + type: DirectoryOrCreate + - name: registration-dir + hostPath: + path: {{ .Values.registrationDir }} + type: Directory + - name: pods-mount-dir + hostPath: + path: /var/lib/kubelet/pods + type: Directory + - name: host-dev + hostPath: + path: /dev + - name: host-rootfs + hostPath: + path: / + - name: host-sys + hostPath: + path: /sys + - name: lib-modules + hostPath: + path: /lib/modules + {{- if .Values.nodeplugin.affinity -}} + affinity: +{{ toYaml .Values.nodeplugin.affinity . | indent 8 }} + {{- end -}} + {{- if .Values.nodeplugin.nodeSelector -}} + nodeSelector: +{{ toYaml .Values.nodeplugin.nodeSelector | indent 8 }} + {{- end -}} + {{- if .Values.nodeplugin.tolerations -}} + tolerations: +{{ toYaml .Values.nodeplugin.tolerations | indent 8 }} + {{- end -}} diff --git a/deploy/cephfs/helm/templates/nodeplugin-serviceaccount.yaml b/deploy/cephfs/helm/templates/nodeplugin-serviceaccount.yaml new file mode 100644 index 000000000..88bd8f1bc --- /dev/null +++ b/deploy/cephfs/helm/templates/nodeplugin-serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccounts.nodeplugin.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "ceph-csi-cephfs.serviceAccountName.nodeplugin" . }} + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.nodeplugin.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +{{- end -}} diff --git a/deploy/cephfs/helm/templates/provisioner-clusterrole.yaml b/deploy/cephfs/helm/templates/provisioner-clusterrole.yaml new file mode 100644 index 000000000..07e35a98e --- /dev/null +++ b/deploy/cephfs/helm/templates/provisioner-clusterrole.yaml @@ -0,0 +1,37 @@ +{{- if .Values.rbac.create -}} +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }} + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "create", "delete"] + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csinodeinfos"] + verbs: ["get", "list", "watch"] +{{- end -}} diff --git a/deploy/cephfs/helm/templates/provisioner-clusterrolebinding.yaml b/deploy/cephfs/helm/templates/provisioner-clusterrolebinding.yaml new file mode 100644 index 000000000..82d5d1316 --- /dev/null +++ b/deploy/cephfs/helm/templates/provisioner-clusterrolebinding.yaml @@ -0,0 +1,20 @@ +{{- if .Values.rbac.create -}} +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }} + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +subjects: + - kind: ServiceAccount + name: {{ include "ceph-csi-cephfs.serviceAccountName.provisioner" . }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }} + apiGroup: rbac.authorization.k8s.io +{{- end -}} diff --git a/deploy/cephfs/helm/templates/provisioner-role.yaml b/deploy/cephfs/helm/templates/provisioner-role.yaml new file mode 100644 index 000000000..c6f28c40e --- /dev/null +++ b/deploy/cephfs/helm/templates/provisioner-role.yaml @@ -0,0 +1,19 @@ +{{- if .Values.rbac.create -}} +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }} + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +rules: + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create", "delete"] +{{- end -}} diff --git a/deploy/cephfs/helm/templates/provisioner-rolebinding.yaml b/deploy/cephfs/helm/templates/provisioner-rolebinding.yaml new file mode 100644 index 000000000..63dc9503b --- /dev/null +++ b/deploy/cephfs/helm/templates/provisioner-rolebinding.yaml @@ -0,0 +1,21 @@ +{{- if .Values.rbac.create -}} +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }} + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +subjects: + - kind: ServiceAccount + name: {{ include "ceph-csi-cephfs.serviceAccountName.provisioner" . }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: Role + name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }} + apiGroup: rbac.authorization.k8s.io + namespace: {{ .Release.Namespace }} +{{- end -}} diff --git a/deploy/cephfs/helm/templates/provisioner-service.yaml b/deploy/cephfs/helm/templates/provisioner-service.yaml new file mode 100644 index 000000000..93d62ffb7 --- /dev/null +++ b/deploy/cephfs/helm/templates/provisioner-service.yaml @@ -0,0 +1,18 @@ +kind: Service +apiVersion: v1 +metadata: + name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }} + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + selector: + app: {{ include "ceph-csi-cephfs.name" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + ports: + - name: dummy + port: 12345 diff --git a/deploy/cephfs/helm/templates/provisioner-serviceaccount.yaml b/deploy/cephfs/helm/templates/provisioner-serviceaccount.yaml new file mode 100644 index 000000000..2c1d9f74f --- /dev/null +++ b/deploy/cephfs/helm/templates/provisioner-serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccounts.provisioner.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "ceph-csi-cephfs.serviceAccountName.provisioner" . }} + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +{{- end -}} diff --git a/deploy/cephfs/helm/templates/provisioner-statefulset.yaml b/deploy/cephfs/helm/templates/provisioner-statefulset.yaml new file mode 100644 index 000000000..fe4fc6428 --- /dev/null +++ b/deploy/cephfs/helm/templates/provisioner-statefulset.yaml @@ -0,0 +1,94 @@ +kind: StatefulSet +apiVersion: apps/v1beta1 +metadata: + name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }} + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + serviceName: {{ include "ceph-csi-cephfs.provisioner.fullname" . }} + replicas: {{ .Values.provisioner.replicas }} + selector: + matchLabels: + app: {{ include "ceph-csi-cephfs.name" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + template: + metadata: + labels: + app: {{ include "ceph-csi-cephfs.name" . }} + chart: {{ include "ceph-csi-cephfs.chart" . }} + component: {{ .Values.provisioner.name }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + spec: + serviceAccountName: {{ include "ceph-csi-cephfs.serviceAccountName.provisioner" . }} + containers: + - name: csi-provisioner + image: "{{ .Values.provisioner.image.repository }}:{{ .Values.provisioner.image.tag }}" + args: + - "--csi-address=$(ADDRESS)" + - "--v=5" + env: + - name: ADDRESS + value: "{{ .Values.socketDir }}/{{ .Values.socketFile }}" + imagePullPolicy: {{ .Values.provisioner.image.pullPolicy }} + volumeMounts: + - name: socket-dir + mountPath: {{ .Values.socketDir }} + resources: +{{ toYaml .Values.provisioner.resources | indent 12 }} + - name: csi-cephfsplugin + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + image: "{{ .Values.nodeplugin.plugin.image.repository }}:{{ .Values.nodeplugin.plugin.image.tag }}" + args : + - "--nodeid=$(NODE_ID)" + - "--endpoint=$(CSI_ENDPOINT)" + - "--v=5" + - "--drivername=$(DRIVER_NAME)" + - "--metadatastorage=k8s_configmap" + env: + - name: HOST_ROOTFS + value: "/rootfs" + - name: DRIVER_NAME + value: {{ .Values.driverName }} + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: "unix:/{{ .Values.socketDir }}/{{ .Values.socketFile }}" + imagePullPolicy: {{ .Values.nodeplugin.plugin.image.imagePullPolicy }} + volumeMounts: + - name: socket-dir + mountPath: {{ .Values.socketDir }} + - name: host-rootfs + mountPath: "/rootfs" + resources: +{{ toYaml .Values.nodeplugin.plugin.resources | indent 12 }} + volumes: + - name: socket-dir + emptyDir: {} +#FIXME this seems way too much. Why is it needed at all for this? + - name: host-rootfs + hostPath: + path: / + {{- if .Values.provisioner.affinity -}} + affinity: +{{ toYaml .Values.provisioner.affinity . | indent 8 }} + {{- end -}} + {{- if .Values.provisioner.nodeSelector -}} + nodeSelector: +{{ toYaml .Values.provisioner.nodeSelector | indent 8 }} + {{- end -}} + {{- if .Values.provisioner.tolerations -}} + tolerations: +{{ toYaml .Values.provisioner.tolerations | indent 8 }} + {{- end -}} diff --git a/deploy/cephfs/helm/values.yaml b/deploy/cephfs/helm/values.yaml new file mode 100644 index 000000000..b31c9733e --- /dev/null +++ b/deploy/cephfs/helm/values.yaml @@ -0,0 +1,80 @@ +--- +rbac: + create: true + +serviceAccounts: + attacher: + create: true + name: + nodeplugin: + create: true + name: + provisioner: + create: true + name: + +socketDir: /var/lib/kubelet/plugins/cephfs.csi.ceph.com +socketFile: csi.sock +registrationDir: /var/lib/kubelet/plugins_registry +volumeDevicesDir: /var/lib/kubelet/plugins/kubernetes.io/csi/volumeDevices +driverName: cephfs.csi.ceph.com +attacher: + name: attacher + + replicaCount: 1 + + image: + repository: quay.io/k8scsi/csi-attacher + tag: v1.0.1 + pullPolicy: IfNotPresent + + resources: {} + + nodeSelector: {} + + tolerations: [] + + affinity: {} + +nodeplugin: + name: nodeplugin + + registrar: + image: + repository: quay.io/k8scsi/csi-node-driver-registrar + tag: v1.0.2 + pullPolicy: IfNotPresent + + resources: {} + + plugin: + image: + repository: quay.io/cephcsi/cephfsplugin + tag: v1.0.0 + pullPolicy: IfNotPresent + + resources: {} + + nodeSelector: {} + + tolerations: [] + + affinity: {} + +provisioner: + name: provisioner + + replicaCount: 1 + + image: + repository: quay.io/k8scsi/csi-provisioner + tag: v1.0.1 + pullPolicy: IfNotPresent + + resources: {} + + nodeSelector: {} + + tolerations: [] + + affinity: {} diff --git a/deploy/cephfs/kubernetes/csi-attacher-rbac.yaml b/deploy/cephfs/kubernetes/csi-attacher-rbac.yaml index 94dc8f7a8..3b16a8ea2 100644 --- a/deploy/cephfs/kubernetes/csi-attacher-rbac.yaml +++ b/deploy/cephfs/kubernetes/csi-attacher-rbac.yaml @@ -10,9 +10,6 @@ apiVersion: rbac.authorization.k8s.io/v1 metadata: name: cephfs-external-attacher-runner rules: - - apiGroups: [""] - resources: ["events"] - verbs: ["get", "list", "watch", "update"] - apiGroups: [""] resources: ["persistentvolumes"] verbs: ["get", "list", "watch", "update"] @@ -22,6 +19,9 @@ rules: - apiGroups: ["storage.k8s.io"] resources: ["volumeattachments"] verbs: ["get", "list", "watch", "update"] + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csinodeinfos"] + verbs: ["get", "list", "watch"] --- kind: ClusterRoleBinding diff --git a/deploy/cephfs/kubernetes/csi-cephfsplugin-attacher.yaml b/deploy/cephfs/kubernetes/csi-cephfsplugin-attacher.yaml index 06f27ca80..1cd97126b 100644 --- a/deploy/cephfs/kubernetes/csi-cephfsplugin-attacher.yaml +++ b/deploy/cephfs/kubernetes/csi-cephfsplugin-attacher.yaml @@ -34,13 +34,13 @@ spec: - "--csi-address=$(ADDRESS)" env: - name: ADDRESS - value: /var/lib/kubelet/plugins/csi-cephfsplugin/csi.sock + value: /var/lib/kubelet/plugins/cephfs.csi.ceph.com/csi.sock imagePullPolicy: "IfNotPresent" volumeMounts: - name: socket-dir - mountPath: /var/lib/kubelet/plugins/csi-cephfsplugin + mountPath: /var/lib/kubelet/plugins/cephfs.csi.ceph.com volumes: - name: socket-dir hostPath: - path: /var/lib/kubelet/plugins/csi-cephfsplugin + path: /var/lib/kubelet/plugins/cephfs.csi.ceph.com type: DirectoryOrCreate diff --git a/deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml b/deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml index b3c2ffc0d..af5962933 100644 --- a/deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml +++ b/deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml @@ -34,11 +34,11 @@ spec: - "--v=5" env: - name: ADDRESS - value: /var/lib/kubelet/plugins/csi-cephfsplugin/csi-provisioner.sock + value: unix:///csi/csi-provisioner.sock imagePullPolicy: "IfNotPresent" volumeMounts: - name: socket-dir - mountPath: /var/lib/kubelet/plugins/csi-cephfsplugin + mountPath: /csi - name: csi-cephfsplugin securityContext: privileged: true @@ -49,7 +49,7 @@ spec: - "--nodeid=$(NODE_ID)" - "--endpoint=$(CSI_ENDPOINT)" - "--v=5" - - "--drivername=csi-cephfsplugin" + - "--drivername=cephfs.csi.ceph.com" - "--metadatastorage=k8s_configmap" env: - name: NODE_ID @@ -61,11 +61,11 @@ spec: fieldRef: fieldPath: metadata.namespace - name: CSI_ENDPOINT - value: unix://var/lib/kubelet/plugins/csi-cephfsplugin/csi-provisioner.sock + value: unix:///csi/csi-provisioner.sock imagePullPolicy: "IfNotPresent" volumeMounts: - name: socket-dir - mountPath: /var/lib/kubelet/plugins/csi-cephfsplugin + mountPath: /csi - name: host-sys mountPath: /sys - name: lib-modules @@ -76,7 +76,7 @@ spec: volumes: - name: socket-dir hostPath: - path: /var/lib/kubelet/plugins/csi-cephfsplugin + path: /var/lib/kubelet/plugins/cephfs.csi.ceph.com type: DirectoryOrCreate - name: host-sys hostPath: diff --git a/deploy/cephfs/kubernetes/csi-cephfsplugin.yaml b/deploy/cephfs/kubernetes/csi-cephfsplugin.yaml index 9e482c75a..849cf57b3 100644 --- a/deploy/cephfs/kubernetes/csi-cephfsplugin.yaml +++ b/deploy/cephfs/kubernetes/csi-cephfsplugin.yaml @@ -23,7 +23,7 @@ spec: args: - "--v=5" - "--csi-address=/csi/csi.sock" - - "--kubelet-registration-path=/var/lib/kubelet/plugins/csi-cephfsplugin/csi.sock" + - "--kubelet-registration-path=/var/lib/kubelet/plugins/cephfs.csi.ceph.com/csi.sock" lifecycle: preStop: exec: @@ -53,8 +53,9 @@ spec: - "--nodeid=$(NODE_ID)" - "--endpoint=$(CSI_ENDPOINT)" - "--v=5" - - "--drivername=csi-cephfsplugin" + - "--drivername=cephfs.csi.ceph.com" - "--metadatastorage=k8s_configmap" + - "--mountcachedir=/mount-cache-dir" env: - name: NODE_ID valueFrom: @@ -65,11 +66,13 @@ spec: fieldRef: fieldPath: metadata.namespace - name: CSI_ENDPOINT - value: unix://var/lib/kubelet/plugins/csi-cephfsplugin/csi.sock + value: unix:///csi/csi.sock imagePullPolicy: "IfNotPresent" volumeMounts: + - name: mount-cache-dir + mountPath: /mount-cache-dir - name: plugin-dir - mountPath: /var/lib/kubelet/plugins/csi-cephfsplugin + mountPath: /csi - name: csi-plugins-dir mountPath: /var/lib/kubelet/plugins/kubernetes.io/csi mountPropagation: "Bidirectional" @@ -84,9 +87,11 @@ spec: - name: host-dev mountPath: /dev volumes: + - name: mount-cache-dir + emptyDir: {} - name: plugin-dir hostPath: - path: /var/lib/kubelet/plugins/csi-cephfsplugin/ + path: /var/lib/kubelet/plugins/cephfs.csi.ceph.com/ type: DirectoryOrCreate - name: csi-plugins-dir hostPath: diff --git a/deploy/cephfs/kubernetes/csi-nodeplugin-rbac.yaml b/deploy/cephfs/kubernetes/csi-nodeplugin-rbac.yaml index 7e6b075b6..918bdc983 100644 --- a/deploy/cephfs/kubernetes/csi-nodeplugin-rbac.yaml +++ b/deploy/cephfs/kubernetes/csi-nodeplugin-rbac.yaml @@ -10,6 +10,9 @@ apiVersion: rbac.authorization.k8s.io/v1 metadata: name: cephfs-csi-nodeplugin rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list"] - apiGroups: [""] resources: ["nodes"] verbs: ["get", "list", "update"] @@ -22,9 +25,6 @@ rules: - apiGroups: ["storage.k8s.io"] resources: ["volumeattachments"] verbs: ["get", "list", "watch", "update"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list"] --- kind: ClusterRoleBinding diff --git a/deploy/cephfs/kubernetes/csi-provisioner-rbac.yaml b/deploy/cephfs/kubernetes/csi-provisioner-rbac.yaml index 796dc86b0..80ef301a9 100644 --- a/deploy/cephfs/kubernetes/csi-provisioner-rbac.yaml +++ b/deploy/cephfs/kubernetes/csi-provisioner-rbac.yaml @@ -10,9 +10,15 @@ apiVersion: rbac.authorization.k8s.io/v1 metadata: name: cephfs-external-provisioner-runner rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["secrets"] verbs: ["get", "list"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] - apiGroups: [""] resources: ["persistentvolumes"] verbs: ["get", "list", "watch", "create", "delete"] @@ -22,12 +28,9 @@ rules: - apiGroups: ["storage.k8s.io"] resources: ["storageclasses"] verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list", "create", "delete"] + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csinodeinfos"] + verbs: ["get", "list", "watch"] --- kind: ClusterRoleBinding @@ -42,3 +45,35 @@ roleRef: kind: ClusterRole name: cephfs-external-provisioner-runner apiGroup: rbac.authorization.k8s.io + +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + # replace with non-default namespace name + namespace: default + name: cephfs-external-provisioner-cfg +rules: + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "create", "delete"] + +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cephfs-csi-provisioner-role-cfg + # replace with non-default namespace name + namespace: default +subjects: + - kind: ServiceAccount + name: cephfs-csi-provisioner + # replace with non-default namespace name + namespace: default +roleRef: + kind: Role + name: cephfs-external-provisioner-cfg + apiGroup: rbac.authorization.k8s.io diff --git a/deploy/rbd/helm/Chart.yaml b/deploy/rbd/helm/Chart.yaml index d91c4e041..30585b9d8 100644 --- a/deploy/rbd/helm/Chart.yaml +++ b/deploy/rbd/helm/Chart.yaml @@ -4,7 +4,7 @@ appVersion: "1.0.0" description: "Container Storage Interface (CSI) driver, provisioner, snapshotter, and attacher for Ceph RBD" name: ceph-csi-rbd -version: 0.4.0 +version: 0.5.1 keywords: - ceph - rbd diff --git a/deploy/rbd/helm/README.md b/deploy/rbd/helm/README.md index 42a55e55b..250c10add 100644 --- a/deploy/rbd/helm/README.md +++ b/deploy/rbd/helm/README.md @@ -4,7 +4,7 @@ The ceph-csi-rbd chart adds rbd volume support to your cluster. ## Install Chart -To install the Chart into your Kubernetes cluster : +To install the Chart into your Kubernetes cluster ```bash helm install --namespace "ceph-csi-rbd" --name "ceph-csi-rbd" ceph-csi/ceph-csi-rbd @@ -16,9 +16,14 @@ After installation succeeds, you can get a status of Chart helm status "ceph-csi-rbd" ``` -If you want to delete your Chart, use this command: +If you want to delete your Chart, use this command ```bash helm delete --purge "ceph-csi-rbd" ``` +If you want to delete the namespace, use this command + +```bash +kubectl delete namespace ceph-csi-rbd +``` diff --git a/deploy/rbd/helm/templates/attacher-clusterrole.yaml b/deploy/rbd/helm/templates/attacher-clusterrole.yaml index 3ebc0438d..59507abc3 100644 --- a/deploy/rbd/helm/templates/attacher-clusterrole.yaml +++ b/deploy/rbd/helm/templates/attacher-clusterrole.yaml @@ -22,4 +22,7 @@ rules: - apiGroups: ["storage.k8s.io"] resources: ["volumeattachments"] verbs: ["get", "list", "watch", "update"] + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csinodeinfos"] + verbs: ["get", "list", "watch"] {{- end -}} diff --git a/deploy/rbd/helm/templates/nodeplugin-daemonset.yaml b/deploy/rbd/helm/templates/nodeplugin-daemonset.yaml index 31ffa1db5..355983b49 100644 --- a/deploy/rbd/helm/templates/nodeplugin-daemonset.yaml +++ b/deploy/rbd/helm/templates/nodeplugin-daemonset.yaml @@ -39,7 +39,11 @@ spec: lifecycle: preStop: exec: - command: ["/bin/sh", "-c", "rm -rf /registration/csi-rbdplugin /registration/csi-rbdplugin-reg.sock"] + command: [ + "/bin/sh", "-c", + 'rm -rf /registration/{{ .Values.driverName }} + /registration/{{ .Values.driverName }}-reg.sock' + ] env: - name: KUBE_NODE_NAME valueFrom: @@ -64,12 +68,14 @@ spec: - "--nodeid=$(NODE_ID)" - "--endpoint=$(CSI_ENDPOINT)" - "--v=5" - - "--drivername=csi-rbdplugin" + - "--drivername=$(DRIVER_NAME)" - "--containerized=true" - "--metadatastorage=k8s_configmap" env: - name: HOST_ROOTFS value: "/rootfs" + - name: DRIVER_NAME + value: {{ .Values.driverName }} - name: NODE_ID valueFrom: fieldRef: diff --git a/deploy/rbd/helm/templates/provisioner-clusterrole.yaml b/deploy/rbd/helm/templates/provisioner-clusterrole.yaml index 1891dc6b1..d324e455b 100644 --- a/deploy/rbd/helm/templates/provisioner-clusterrole.yaml +++ b/deploy/rbd/helm/templates/provisioner-clusterrole.yaml @@ -10,6 +10,9 @@ metadata: release: {{ .Release.Name }} heritage: {{ .Release.Service }} rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["secrets"] verbs: ["get", "list"] @@ -43,4 +46,7 @@ rules: - apiGroups: ["apiextensions.k8s.io"] resources: ["customresourcedefinitions"] verbs: ["create"] + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csinodeinfos"] + verbs: ["get", "list", "watch"] {{- end -}} diff --git a/deploy/rbd/helm/templates/provisioner-statefulset.yaml b/deploy/rbd/helm/templates/provisioner-statefulset.yaml index 2a455ee50..269cb0a44 100644 --- a/deploy/rbd/helm/templates/provisioner-statefulset.yaml +++ b/deploy/rbd/helm/templates/provisioner-statefulset.yaml @@ -69,12 +69,14 @@ spec: - "--nodeid=$(NODE_ID)" - "--endpoint=$(CSI_ENDPOINT)" - "--v=5" - - "--drivername=csi-rbdplugin" + - "--drivername=$(DRIVER_NAME)" - "--containerized=true" - "--metadatastorage=k8s_configmap" env: - name: HOST_ROOTFS value: "/rootfs" + - name: DRIVER_NAME + value: {{ .Values.driverName }} - name: NODE_ID valueFrom: fieldRef: diff --git a/deploy/rbd/helm/values.yaml b/deploy/rbd/helm/values.yaml index bf4c2fb5b..fdeb5d6d5 100644 --- a/deploy/rbd/helm/values.yaml +++ b/deploy/rbd/helm/values.yaml @@ -13,10 +13,11 @@ serviceAccounts: create: true name: -socketDir: /var/lib/kubelet/plugins/csi-rbdplugin +socketDir: /var/lib/kubelet/plugins/rbd.csi.ceph.com socketFile: csi.sock registrationDir: /var/lib/kubelet/plugins_registry volumeDevicesDir: /var/lib/kubelet/plugins/kubernetes.io/csi/volumeDevices +driverName: rbd.csi.ceph.com attacher: name: attacher diff --git a/deploy/rbd/kubernetes/csi-attacher-rbac.yaml b/deploy/rbd/kubernetes/csi-attacher-rbac.yaml index 7160e293e..e502da5c9 100644 --- a/deploy/rbd/kubernetes/csi-attacher-rbac.yaml +++ b/deploy/rbd/kubernetes/csi-attacher-rbac.yaml @@ -10,9 +10,6 @@ apiVersion: rbac.authorization.k8s.io/v1 metadata: name: rbd-external-attacher-runner rules: - - apiGroups: [""] - resources: ["events"] - verbs: ["get", "list", "watch", "update"] - apiGroups: [""] resources: ["persistentvolumes"] verbs: ["get", "list", "watch", "update"] @@ -22,6 +19,9 @@ rules: - apiGroups: ["storage.k8s.io"] resources: ["volumeattachments"] verbs: ["get", "list", "watch", "update"] + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csinodeinfos"] + verbs: ["get", "list", "watch"] --- kind: ClusterRoleBinding diff --git a/deploy/rbd/kubernetes/csi-nodeplugin-rbac.yaml b/deploy/rbd/kubernetes/csi-nodeplugin-rbac.yaml index c960408e6..9a5ffed1b 100644 --- a/deploy/rbd/kubernetes/csi-nodeplugin-rbac.yaml +++ b/deploy/rbd/kubernetes/csi-nodeplugin-rbac.yaml @@ -10,6 +10,9 @@ apiVersion: rbac.authorization.k8s.io/v1 metadata: name: rbd-csi-nodeplugin rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] - apiGroups: [""] resources: ["nodes"] verbs: ["get", "list", "update"] diff --git a/deploy/rbd/kubernetes/csi-provisioner-rbac.yaml b/deploy/rbd/kubernetes/csi-provisioner-rbac.yaml index 71ef4f160..75615b054 100644 --- a/deploy/rbd/kubernetes/csi-provisioner-rbac.yaml +++ b/deploy/rbd/kubernetes/csi-provisioner-rbac.yaml @@ -10,9 +10,15 @@ apiVersion: rbac.authorization.k8s.io/v1 metadata: name: rbd-external-provisioner-runner rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["secrets"] verbs: ["get", "list"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] - apiGroups: [""] resources: ["persistentvolumes"] verbs: ["get", "list", "watch", "create", "delete"] @@ -22,18 +28,9 @@ rules: - apiGroups: ["storage.k8s.io"] resources: ["storageclasses"] verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["events"] - verbs: ["list", "watch", "create", "update", "patch"] - - apiGroups: [""] - resources: ["endpoints"] - verbs: ["get", "create", "update"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshots"] verbs: ["get", "list", "watch", "update"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["get", "list", "create", "delete"] - apiGroups: ["snapshot.storage.k8s.io"] resources: ["volumesnapshotcontents"] verbs: ["create", "get", "list", "watch", "update", "delete"] @@ -43,6 +40,9 @@ rules: - apiGroups: ["apiextensions.k8s.io"] resources: ["customresourcedefinitions"] verbs: ["create"] + - apiGroups: ["csi.storage.k8s.io"] + resources: ["csinodeinfos"] + verbs: ["get", "list", "watch"] --- kind: ClusterRoleBinding @@ -57,3 +57,35 @@ roleRef: kind: ClusterRole name: rbd-external-provisioner-runner apiGroup: rbac.authorization.k8s.io + +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + # replace with non-default namespace name + namespace: default + name: rbd-external-provisioner-cfg +rules: + - apiGroups: [""] + resources: ["endpoints"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create", "delete"] + +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-csi-provisioner-role-cfg + # replace with non-default namespace name + namespace: default +subjects: + - kind: ServiceAccount + name: rbd-csi-provisioner + # replace with non-default namespace name + namespace: default +roleRef: + kind: Role + name: rbd-external-provisioner-cfg + apiGroup: rbac.authorization.k8s.io diff --git a/deploy/rbd/kubernetes/csi-rbdplugin-attacher.yaml b/deploy/rbd/kubernetes/csi-rbdplugin-attacher.yaml index 4b7c15e83..81029b733 100644 --- a/deploy/rbd/kubernetes/csi-rbdplugin-attacher.yaml +++ b/deploy/rbd/kubernetes/csi-rbdplugin-attacher.yaml @@ -34,13 +34,13 @@ spec: - "--csi-address=$(ADDRESS)" env: - name: ADDRESS - value: /var/lib/kubelet/plugins/csi-rbdplugin/csi.sock + value: unix:///csi/csi-attacher.sock imagePullPolicy: "IfNotPresent" volumeMounts: - name: socket-dir - mountPath: /var/lib/kubelet/plugins/csi-rbdplugin + mountPath: /csi volumes: - name: socket-dir hostPath: - path: /var/lib/kubelet/plugins/csi-rbdplugin + path: /var/lib/kubelet/plugins/rbd.csi.ceph.com type: DirectoryOrCreate diff --git a/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml b/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml index cc647b7b9..bd14fa363 100644 --- a/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml +++ b/deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml @@ -34,11 +34,11 @@ spec: - "--v=5" env: - name: ADDRESS - value: /var/lib/kubelet/plugins/csi-rbdplugin/csi-provisioner.sock + value: unix:///csi/csi-provisioner.sock imagePullPolicy: "IfNotPresent" volumeMounts: - name: socket-dir - mountPath: /var/lib/kubelet/plugins/csi-rbdplugin + mountPath: /csi - name: csi-snapshotter image: quay.io/k8scsi/csi-snapshotter:v1.0.1 args: @@ -47,13 +47,13 @@ spec: - "--v=5" env: - name: ADDRESS - value: /var/lib/kubelet/plugins/csi-rbdplugin/csi-provisioner.sock + value: unix:///csi/csi-provisioner.sock imagePullPolicy: Always securityContext: privileged: true volumeMounts: - name: socket-dir - mountPath: /var/lib/kubelet/plugins/csi-rbdplugin + mountPath: /csi - name: csi-rbdplugin securityContext: privileged: true @@ -64,9 +64,10 @@ spec: - "--nodeid=$(NODE_ID)" - "--endpoint=$(CSI_ENDPOINT)" - "--v=5" - - "--drivername=csi-rbdplugin" + - "--drivername=rbd.csi.ceph.com" - "--containerized=true" - "--metadatastorage=k8s_configmap" + - "--configroot=k8s_objects" env: - name: HOST_ROOTFS value: "/rootfs" @@ -79,11 +80,11 @@ spec: fieldRef: fieldPath: metadata.namespace - name: CSI_ENDPOINT - value: unix://var/lib/kubelet/plugins/csi-rbdplugin/csi-provisioner.sock + value: unix:///csi/csi-provisioner.sock imagePullPolicy: "IfNotPresent" volumeMounts: - name: socket-dir - mountPath: /var/lib/kubelet/plugins/csi-rbdplugin + mountPath: /csi - mountPath: /dev name: host-dev - mountPath: /rootfs @@ -108,5 +109,5 @@ spec: path: /lib/modules - name: socket-dir hostPath: - path: /var/lib/kubelet/plugins/csi-rbdplugin + path: /var/lib/kubelet/plugins/rbd.csi.ceph.com type: DirectoryOrCreate diff --git a/deploy/rbd/kubernetes/csi-rbdplugin.yaml b/deploy/rbd/kubernetes/csi-rbdplugin.yaml index 86291dc7e..95bd27b7d 100644 --- a/deploy/rbd/kubernetes/csi-rbdplugin.yaml +++ b/deploy/rbd/kubernetes/csi-rbdplugin.yaml @@ -24,14 +24,14 @@ spec: args: - "--v=5" - "--csi-address=/csi/csi.sock" - - "--kubelet-registration-path=/var/lib/kubelet/plugins/csi-rbdplugin/csi.sock" + - "--kubelet-registration-path=/var/lib/kubelet/plugins/rbd.csi.ceph.com/csi.sock" lifecycle: preStop: exec: command: [ "/bin/sh", "-c", - "rm -rf /registration/csi-rbdplugin \ - /registration/csi-rbdplugin-reg.sock" + "rm -rf /registration/rbd.csi.ceph.com \ + /registration/rbd.csi.ceph.com-reg.sock" ] env: - name: KUBE_NODE_NAME @@ -54,9 +54,10 @@ spec: - "--nodeid=$(NODE_ID)" - "--endpoint=$(CSI_ENDPOINT)" - "--v=5" - - "--drivername=csi-rbdplugin" + - "--drivername=rbd.csi.ceph.com" - "--containerized=true" - "--metadatastorage=k8s_configmap" + - "--configroot=k8s_objects" env: - name: HOST_ROOTFS value: "/rootfs" @@ -69,11 +70,11 @@ spec: fieldRef: fieldPath: metadata.namespace - name: CSI_ENDPOINT - value: unix://var/lib/kubelet/plugins_registry/csi-rbdplugin/csi.sock + value: unix:///csi/csi.sock imagePullPolicy: "IfNotPresent" volumeMounts: - name: plugin-dir - mountPath: /var/lib/kubelet/plugins_registry/csi-rbdplugin + mountPath: /csi - name: pods-mount-dir mountPath: /var/lib/kubelet/pods mountPropagation: "Bidirectional" @@ -92,7 +93,7 @@ spec: volumes: - name: plugin-dir hostPath: - path: /var/lib/kubelet/plugins/csi-rbdplugin + path: /var/lib/kubelet/plugins/rbd.csi.ceph.com type: DirectoryOrCreate - name: plugin-mount-dir hostPath: diff --git a/docs/deploy-cephfs.md b/docs/deploy-cephfs.md index 7db96cb3c..6266107e7 100644 --- a/docs/deploy-cephfs.md +++ b/docs/deploy-cephfs.md @@ -5,8 +5,8 @@ and attach and mount existing ones to workloads. ## Building -CSI CephFS plugin can be compiled in a form of a binary file or in a form of a -Docker image. +CSI CephFS plugin can be compiled in the form of a binary file or in the form +of a Docker image. When compiled as a binary file, the result is stored in `_output/` directory with the name `cephfsplugin`. When compiled as an image, it's stored in the local Docker image store. @@ -30,15 +30,17 @@ make image-cephfsplugin Option | Default value | Description --------------------|-----------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- `--endpoint` | `unix://tmp/csi.sock` | CSI endpoint, must be a UNIX socket -`--drivername` | `csi-cephfsplugin` | name of the driver (Kubernetes: `provisioner` field in StorageClass must correspond to this value) +`--drivername` | `cephfs.csi.ceph.com` | name of the driver (Kubernetes: `provisioner` field in StorageClass must correspond to this value) `--nodeid` | _empty_ | This node's ID `--volumemounter` | _empty_ | default volume mounter. Available options are `kernel` and `fuse`. This is the mount method used if volume parameters don't specify otherwise. If left unspecified, the driver will first probe for `ceph-fuse` in system's path and will choose Ceph kernel client if probing failed. -`--metadatastorage` | _empty_ | Whether should metadata be kept on node as file or in a k8s configmap (`node` or `k8s_configmap`) +`--metadatastorage` | _empty_ | Whether metadata should be kept on node as file or in a k8s configmap (`node` or `k8s_configmap`) +`--mountcachedir` | _empty_ | volume mount cache info save dir. If left unspecified, the dirver will not record mount info, or it will save mount info and when driver restart it will remount volume it cached. -**Available environmental variables:** `KUBERNETES_CONFIG_PATH`: if you use -`k8s_configmap` as metadata store, specify the path of your k8s config file (if -not specified, the plugin will assume you're running it inside a k8s cluster and -find the config itself). +**Available environmental variables:** + +`KUBERNETES_CONFIG_PATH`: if you use `k8s_configmap` as metadata store, specify +the path of your k8s config file (if not specified, the plugin will assume +you're running it inside a k8s cluster and find the config itself). `POD_NAMESPACE`: if you use `k8s_configmap` as metadata store, `POD_NAMESPACE` is used to define in which namespace you want the configmaps to be stored diff --git a/docs/deploy-rbd.md b/docs/deploy-rbd.md index 000228044..50bdc5325 100644 --- a/docs/deploy-rbd.md +++ b/docs/deploy-rbd.md @@ -1,7 +1,7 @@ # CSI RBD Plugin The RBD CSI plugin is able to provision new RBD images and -attach and mount those to worlkoads. +attach and mount those to workloads. ## Building @@ -29,12 +29,14 @@ make image-rbdplugin Option | Default value | Description ------ | ------------- | ----------- `--endpoint` | `unix://tmp/csi.sock` | CSI endpoint, must be a UNIX socket -`--drivername` | `csi-cephfsplugin` | name of the driver (Kubernetes: `provisioner` field in StorageClass must correspond to this value) +`--drivername` | `rbd.csi.ceph.com` | name of the driver (Kubernetes: `provisioner` field in StorageClass must correspond to this value) `--nodeid` | _empty_ | This node's ID `--containerized` | true | Whether running in containerized mode `--metadatastorage` | _empty_ | Whether should metadata be kept on node as file or in a k8s configmap (`node` or `k8s_configmap`) +`--configroot` | `/etc/csi-config` | Directory in which CSI specific Ceph cluster configurations are present, OR the value `k8s_objects` if present as kubernetes secrets" **Available environmental variables:** + `HOST_ROOTFS`: rbdplugin searches `/proc` directory under the directory set by `HOST_ROOTFS`. `KUBERNETES_CONFIG_PATH`: if you use `k8s_configmap` as metadata store, specify @@ -49,8 +51,9 @@ the configmaps to be stored Parameter | Required | Description --------- | -------- | ----------- -`monitors` | one of `monitors` and `monValueFromSecret` must be set | Comma separated list of Ceph monitors (e.g. `192.168.100.1:6789,192.168.100.2:6789,192.168.100.3:6789`) -`monValueFromSecret` | one of `monitors` and `monValueFromSecret` must be set | a string pointing the key in the credential secret, whose value is the mon. This is used for the case when the monitors' IP or hostnames are changed, the secret can be updated to pick up the new monitors. +`monitors` | one of `monitors`, `clusterID` or `monValueFromSecret` must be set | Comma separated list of Ceph monitors (e.g. `192.168.100.1:6789,192.168.100.2:6789,192.168.100.3:6789`) +`monValueFromSecret` | one of `monitors`, `clusterID` or and `monValueFromSecret` must be set | a string pointing the key in the credential secret, whose value is the mon. This is used for the case when the monitors' IP or hostnames are changed, the secret can be updated to pick up the new monitors. +`clusterID` | one of `monitors`, `clusterID` or `monValueFromSecret` must be set | String representing a Ceph cluster, must be unique across all Ceph clusters in use for provisioning, cannot be greater than 36 bytes in length, and should remain immutable for the lifetime of the Ceph cluster in use `pool` | yes | Ceph pool into which the RBD image shall be created `imageFormat` | no | RBD image format. Defaults to `2`. See [man pages](http://docs.ceph.com/docs/mimic/man/8/rbd/#cmdoption-rbd-image-format) `imageFeatures` | no | RBD image features. Available for `imageFormat=2`. CSI RBD currently supports only `layering` feature. See [man pages](http://docs.ceph.com/docs/mimic/man/8/rbd/#cmdoption-rbd-image-feature) @@ -58,13 +61,22 @@ Parameter | Required | Description `csi.storage.k8s.io/provisioner-secret-namespace`, `csi.storage.k8s.io/node-publish-secret-namespace` | for Kubernetes | namespaces of the above Secret objects `mounter`| no | if set to `rbd-nbd`, use `rbd-nbd` on nodes that have `rbd-nbd` and `nbd` kernel modules to map rbd images +NOTE: If `clusterID` parameter is used, then an accompanying Ceph cluster +configuration secret or config files needs to be provided to the running pods. +Refer to [Cluster ID based configuration](../examples/README.md#cluster-id-based-configuration) +for more information. A suggested way to populate the clusterID is to use the +output of `ceph fsid` of the Ceph cluster to be used for provisioning. + **Required secrets:** Admin credentials are required for provisioning new RBD images `ADMIN_NAME`: `ADMIN_PASSWORD` - note that the key of the key-value pair is the name of the client with admin privileges, and the value is its password -Also note that CSI RBD expects admin keyring and Ceph config file in `/etc/ceph`. +If clusterID is specified, then a secret with various keys and values as +specified in `examples/rbd/template-ceph-cluster-ID-secret.yaml` needs to be +created, with the secret name matching the string value provided as the +`clusterID`. ## Deployment with Kubernetes @@ -110,7 +122,7 @@ Deploys a daemon set with two containers: CSI driver-registrar and the CSI RBD d ## Verifying the deployment in Kubernetes -After successfuly completing the steps above, you should see output similar to this: +After successfully completing the steps above, you should see output similar to this: ```bash $ kubectl get all diff --git a/examples/README.md b/examples/README.md index beada6280..40b6fef26 100644 --- a/examples/README.md +++ b/examples/README.md @@ -7,11 +7,16 @@ By default, they look for the YAML manifests in `../../deploy/{rbd,cephfs}/kubernetes`. You can override this path by running `$ ./plugin-deploy.sh /path/to/my/manifests`. -Once the plugin is successfuly deployed, you'll need to customize +Once the plugin is successfully deployed, you'll need to customize `storageclass.yaml` and `secret.yaml` manifests to reflect your Ceph cluster setup. Please consult the documentation for info about available parameters. +**NOTE:** See section +[Cluster ID based configuration](#cluster-id-based-configuration) if using +the `clusterID` instead of `monitors` or `monValueFromSecret` option in the +storage class for RBD based provisioning before proceeding. + After configuring the secrets, monitors, etc. you can deploy a testing Pod mounting a RBD image / CephFS volume: @@ -108,9 +113,142 @@ one of your Ceph pod. To restore the snapshot to a new PVC, deploy [pvc-restore.yaml](./rbd/pvc-restore.yaml) and a testing pod -[pod-restore.yaml](./rbd/pvc-restore.yaml): +[pod-restore.yaml](./rbd/pod-restore.yaml): ```bash kubectl create -f pvc-restore.yaml kubectl create -f pod-restore.yaml ``` + +## How to test RBD MULTI_NODE_MULTI_WRITER BLOCK feature + +Requires feature-gates: `BlockVolume=true` `CSIBlockVolume=true` + +*NOTE* The MULTI_NODE_MULTI_WRITER capability is only available for +Volumes that are of access_type `block` + +*WARNING* This feature is strictly for workloads that know how to deal +with concurrent access to the Volume (eg Active/Passive applications). +Using RWX modes on non clustered file systems with applications trying +to simultaneously access the Volume will likely result in data corruption! + +Following are examples for issuing a request for a `Block` +`ReadWriteMany` Claim, and using the resultant Claim for a POD + +```yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: block-pvc +spec: + accessModes: + - ReadWriteMany + volumeMode: Block + resources: + requests: + storage: 1Gi + storageClassName: csi-rbd +``` + +Create a POD that uses this PVC: + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: my-pod +spec: + containers: + - name: my-container + image: debian + command: ["/bin/bash", "-c"] + args: [ "tail -f /dev/null" ] + volumeDevices: + - devicePath: /dev/rbdblock + name: my-volume + imagePullPolicy: IfNotPresent + volumes: + - name: my-volume + persistentVolumeClaim: + claimName: block-pvc + +``` + +Now, we can create a second POD (ensure the POD is scheduled on a different +node; multiwriter single node works without this feature) that also uses this +PVC at the same time, again wait for the pod to enter running state, and verify +the block device is available. + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: another-pod +spec: + containers: + - name: my-container + image: debian + command: ["/bin/bash", "-c"] + args: [ "tail -f /dev/null" ] + volumeDevices: + - devicePath: /dev/rbdblock + name: my-volume + imagePullPolicy: IfNotPresent + volumes: + - name: my-volume + persistentVolumeClaim: + claimName: block-pvc +``` + +Wait for the PODs to enter Running state, check that our block device +is available in the container at `/dev/rdbblock` in both containers: + +```bash +$ kubectl exec -it my-pod -- fdisk -l /dev/rbdblock +Disk /dev/rbdblock: 1 GiB, 1073741824 bytes, 2097152 sectors +Units: sectors of 1 * 512 = 512 bytes +Sector size (logical/physical): 512 bytes / 512 bytes +I/O size (minimum/optimal): 4194304 bytes / 4194304 bytes +``` + +```bash +$ kubectl exec -it another-pod -- fdisk -l /dev/rbdblock +Disk /dev/rbdblock: 1 GiB, 1073741824 bytes, 2097152 sectors +Units: sectors of 1 * 512 = 512 bytes +Sector size (logical/physical): 512 bytes / 512 bytes +I/O size (minimum/optimal): 4194304 bytes / 4194304 bytes +``` + +## Cluster ID based configuration + +Before creating a storage class that uses the option `clusterID` to refer to a +Ceph cluster, the following actions need to be completed. + +Get the following information from the Ceph cluster, + +* Admin ID and key, that has privileges to perform CRUD operations on the Ceph + cluster and pools of choice + * Key is typically the output of, `ceph auth get-key client.admin` where + `admin` is the Admin ID + * Used to substitute admin/user id and key values in the files below +* Ceph monitor list + * Typically in the output of `ceph mon dump` + * Used to prepare comma separated MON list where required in the files below +* Ceph Cluster fsid + * If choosing to use the Ceph cluster fsid as the unique value of clusterID, + * Output of `ceph fsid` + * Used to substitute `` references in the files below + +Update the template +[template-ceph-cluster-ID-secret.yaml](./rbd/template-ceph-cluster-ID-secret.yaml) +with values from +a Ceph cluster and replace `` with the chosen clusterID to create +the following secret, + +* `kubectl create -f rbd/template-ceph-cluster-ID-secret.yaml` + +Storage class and snapshot class, using `` as the value for the +option `clusterID`, can now be created on the cluster. + +Remaining steps to test functionality remains the same as mentioned in the +sections above. diff --git a/examples/cephfs/storageclass.yaml b/examples/cephfs/storageclass.yaml index fb8dd0aa9..771c33998 100644 --- a/examples/cephfs/storageclass.yaml +++ b/examples/cephfs/storageclass.yaml @@ -3,7 +3,7 @@ apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: csi-cephfs -provisioner: csi-cephfsplugin +provisioner: cephfs.csi.ceph.com parameters: # Comma separated list of Ceph monitors # if using FQDN, make sure csi plugin's dns policy is appropriate. diff --git a/examples/rbd/raw-block-pod.yaml b/examples/rbd/raw-block-pod.yaml new file mode 100644 index 000000000..c433e98cc --- /dev/null +++ b/examples/rbd/raw-block-pod.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: v1 +kind: Pod +metadata: + name: pod-with-raw-block-volume +spec: + containers: + - name: fc-container + image: fedora:26 + command: ["/bin/sh", "-c"] + args: ["tail -f /dev/null"] + volumeDevices: + - name: data + devicePath: /dev/xvda + volumes: + - name: data + persistentVolumeClaim: + claimName: raw-block-pvc diff --git a/examples/rbd/raw-block-pvc.yaml b/examples/rbd/raw-block-pvc.yaml new file mode 100644 index 000000000..ee37ff33c --- /dev/null +++ b/examples/rbd/raw-block-pvc.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: raw-block-pvc +spec: + accessModes: + - ReadWriteOnce + volumeMode: Block + resources: + requests: + storage: 1Gi + storageClassName: csi-rbd diff --git a/examples/rbd/snapshotclass.yaml b/examples/rbd/snapshotclass.yaml index 778f2b084..2b1c01dca 100644 --- a/examples/rbd/snapshotclass.yaml +++ b/examples/rbd/snapshotclass.yaml @@ -3,9 +3,21 @@ apiVersion: snapshot.storage.k8s.io/v1alpha1 kind: VolumeSnapshotClass metadata: name: csi-rbdplugin-snapclass -snapshotter: csi-rbdplugin +snapshotter: rbd.csi.ceph.com parameters: pool: rbd + # Comma separated list of Ceph monitors + # if using FQDN, make sure csi plugin's dns policy is appropriate. monitors: mon1:port,mon2:port,... + # OR, + # String representing a Ceph cluster to provision storage from. + # Should be unique across all Ceph clusters in use for provisioning, + # cannot be greater than 36 bytes in length, and should remain immutable for + # the lifetime of the StorageClass in use. + # If using clusterID, ensure to create a secret, as in + # template-ceph-cluster-ID-secret.yaml, to accompany the string chosen to + # represent the Ceph cluster in clusterID + # clusterID: + csi.storage.k8s.io/snapshotter-secret-name: csi-rbd-secret csi.storage.k8s.io/snapshotter-secret-namespace: default diff --git a/examples/rbd/storageclass.yaml b/examples/rbd/storageclass.yaml index 320a489a8..0ec689477 100644 --- a/examples/rbd/storageclass.yaml +++ b/examples/rbd/storageclass.yaml @@ -3,12 +3,21 @@ apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: csi-rbd -provisioner: csi-rbdplugin +provisioner: rbd.csi.ceph.com parameters: # Comma separated list of Ceph monitors # if using FQDN, make sure csi plugin's dns policy is appropriate. monitors: mon1:port,mon2:port,... - + # OR, + # String representing a Ceph cluster to provision storage from. + # Should be unique across all Ceph clusters in use for provisioning, + # cannot be greater than 36 bytes in length, and should remain immutable for + # the lifetime of the StorageClass in use. + # If using clusterID, ensure to create a secret, as in + # template-ceph-cluster-ID-secret.yaml, to accompany the string chosen to + # represent the Ceph cluster in clusterID + # clusterID: + # OR, # if "monitors" parameter is not set, driver to get monitors from same # secret as admin/user credentials. "monValueFromSecret" provides the # key in the secret whose value is the mons @@ -25,12 +34,18 @@ parameters: imageFeatures: layering # The secrets have to contain Ceph admin credentials. + # NOTE: If using "clusterID" instead of "monitors" above, the following + # secrets MAY be added to the ceph-cluster- secret and skipped + # here csi.storage.k8s.io/provisioner-secret-name: csi-rbd-secret csi.storage.k8s.io/provisioner-secret-namespace: default csi.storage.k8s.io/node-publish-secret-name: csi-rbd-secret csi.storage.k8s.io/node-publish-secret-namespace: default # Ceph users for operating RBD + # NOTE: If using "clusterID" instead of "monitors" above, the following + # IDs MAY be added to the ceph-cluster- secret and skipped + # here adminid: admin userid: kubernetes # uncomment the following to use rbd-nbd as mounter on supported nodes diff --git a/examples/rbd/template-ceph-cluster-ID-secret.yaml b/examples/rbd/template-ceph-cluster-ID-secret.yaml new file mode 100644 index 000000000..ef778544f --- /dev/null +++ b/examples/rbd/template-ceph-cluster-ID-secret.yaml @@ -0,0 +1,36 @@ +--- +# This is a template secret that helps define a Ceph cluster configuration +# as required by the CSI driver. This is used when a StorageClass has the +# "clusterID" defined as one of the parameters, to provide the CSI instance +# Ceph cluster configuration information. +apiVersion: v1 +kind: Secret +metadata: + # The is used by the CSI plugin to uniquely identify and use a + # Ceph cluster, the value MUST match the value provided as `clusterID` in the + # StorageClass + name: ceph-cluster- + namespace: default +data: + # Base64 encoded and comma separated Ceph cluster monitor list + # - Typically output of: `echo -n "mon1:port,mon2:port,..." | base64` + monitors: + # Base64 encoded and comma separated list of pool names from which volumes + # can be provisioned + pools: + # Base64 encoded admin ID to use for provisioning + # - Typically output of: `echo -n "" | base64` + # Substitute the entire string including angle braces, with the base64 value + adminid: + # Base64 encoded key of the provisioner admin ID + # - Output of: `ceph auth get-key client. | base64` + # Substitute the entire string including angle braces, with the base64 value + adminkey: + # Base64 encoded user ID to use for publishing + # - Typically output of: `echo -n "" | base64` + # Substitute the entire string including angle braces, with the base64 value + userid: + # Base64 encoded key of the publisher user ID + # - Output of: `ceph auth get-key client. | base64` + # Substitute the entire string including angle braces, with the base64 value + userkey: diff --git a/examples/rbd/template-csi-rbdplugin-patch.yaml b/examples/rbd/template-csi-rbdplugin-patch.yaml new file mode 100644 index 000000000..77570ae96 --- /dev/null +++ b/examples/rbd/template-csi-rbdplugin-patch.yaml @@ -0,0 +1,33 @@ +--- +# This is a patch to the existing daemonset deployment of CSI rbdplugin. +# +# This is to be used when using `clusterID` instead of monitors or +# monValueFromSecret in the StorageClass to specify the Ceph cluster to +# provision storage from, AND when the value of `--configroot` option to the +# CSI pods is NOT "k8s_objects". +# +# This patch file, patches in the specified secret for the 'clusterID' as a +# volume, instead of the Ceph CSI plugin actively fetching and using kubernetes +# secrets. +# +# Post substituting the in all places execute, +# `kubectl patch daemonset csi-rbdplugin --patch\ +# "$(cat template-csi-rbdplugin-patch.yaml)"` +# to patch the daemonset deployment. +# +# `kubectl patch statefulset csi-rbdplugin-provisioner --patch\ +# "$(cat template-csi-rbdplugin-patch.yaml)"` +# to patch the statefulset deployment. +spec: + template: + spec: + containers: + - name: csi-rbdplugin + volumeMounts: + - name: ceph-cluster- + mountPath: "/etc/csi-config/ceph-cluster-" + readOnly: true + volumes: + - name: ceph-cluster- + secret: + secretName: ceph-cluster- diff --git a/pkg/cephfs/cephconf.go b/pkg/cephfs/cephconf.go index 65e79a1d1..dac9dc901 100644 --- a/pkg/cephfs/cephconf.go +++ b/pkg/cephfs/cephconf.go @@ -17,13 +17,8 @@ limitations under the License. package cephfs import ( - "fmt" "io/ioutil" "os" - "path" - "text/template" - - "k8s.io/klog" ) var cephConfig = []byte(`[global] @@ -35,39 +30,11 @@ auth_client_required = cephx fuse_set_user_groups = false `) -const cephKeyring = `[client.{{.UserID}}] -key = {{.Key}} -` - -const cephSecret = `{{.Key}}` // #nosec - const ( - cephConfigRoot = "/etc/ceph" - cephConfigPath = "/etc/ceph/ceph.conf" - cephKeyringFileNameFmt = "ceph.share.%s.client.%s.keyring" - cephSecretFileNameFmt = "ceph.share.%s.client.%s.secret" // #nosec + cephConfigRoot = "/etc/ceph" + cephConfigPath = "/etc/ceph/ceph.conf" ) -var ( - cephKeyringTempl *template.Template - cephSecretTempl *template.Template -) - -func init() { - fm := map[string]interface{}{ - "perms": func(readOnly bool) string { - if readOnly { - return "r" - } - - return "rw" - }, - } - - cephKeyringTempl = template.Must(template.New("keyring").Funcs(fm).Parse(cephKeyring)) - cephSecretTempl = template.Must(template.New("secret").Parse(cephSecret)) -} - func createCephConfigRoot() error { return os.MkdirAll(cephConfigRoot, 0755) // #nosec } @@ -79,51 +46,3 @@ func writeCephConfig() error { return ioutil.WriteFile(cephConfigPath, cephConfig, 0640) } - -func writeCephTemplate(fileName string, m os.FileMode, t *template.Template, data interface{}) error { - if err := createCephConfigRoot(); err != nil { - return err - } - - f, err := os.OpenFile(path.Join(cephConfigRoot, fileName), os.O_CREATE|os.O_EXCL|os.O_WRONLY, m) - if err != nil { - if os.IsExist(err) { - return nil - } - return err - } - - defer func() { - if err := f.Close(); err != nil { - klog.Errorf("failed to close file %s with error %s", f.Name(), err) - } - }() - - return t.Execute(f, data) -} - -type cephKeyringData struct { - UserID, Key string - VolumeID volumeID -} - -func (d *cephKeyringData) writeToFile() error { - return writeCephTemplate(fmt.Sprintf(cephKeyringFileNameFmt, d.VolumeID, d.UserID), 0600, cephKeyringTempl, d) -} - -type cephSecretData struct { - UserID, Key string - VolumeID volumeID -} - -func (d *cephSecretData) writeToFile() error { - return writeCephTemplate(fmt.Sprintf(cephSecretFileNameFmt, d.VolumeID, d.UserID), 0600, cephSecretTempl, d) -} - -func getCephSecretPath(volID volumeID, userID string) string { - return path.Join(cephConfigRoot, fmt.Sprintf(cephSecretFileNameFmt, volID, userID)) -} - -func getCephKeyringPath(volID volumeID, userID string) string { - return path.Join(cephConfigRoot, fmt.Sprintf(cephKeyringFileNameFmt, volID, userID)) -} diff --git a/pkg/cephfs/cephuser.go b/pkg/cephfs/cephuser.go index f642fd654..113933019 100644 --- a/pkg/cephfs/cephuser.go +++ b/pkg/cephfs/cephuser.go @@ -17,12 +17,7 @@ limitations under the License. package cephfs import ( - "bytes" - "encoding/json" "fmt" - "os" - - "k8s.io/klog" ) const ( @@ -53,83 +48,61 @@ func getCephUserName(volID volumeID) string { return cephUserPrefix + string(volID) } -func getCephUser(volOptions *volumeOptions, adminCr *credentials, volID volumeID) (*cephEntity, error) { - entityName := cephEntityClientPrefix + getCephUserName(volID) - +func getSingleCephEntity(args ...string) (*cephEntity, error) { var ents []cephEntity - args := [...]string{ - "-m", volOptions.Monitors, - "auth", "-f", "json", "-c", cephConfigPath, "-n", cephEntityClientPrefix + adminCr.id, "--keyring", getCephKeyringPath(volID, adminCr.id), - "get", entityName, - } - - out, err := execCommand("ceph", args[:]...) - if err != nil { - return nil, fmt.Errorf("cephfs: ceph failed with following error: %s\ncephfs: ceph output: %s", err, out) - } - - // Workaround for output from `ceph auth get` - // Contains non-json data: "exported keyring for ENTITY\n\n" - offset := bytes.Index(out, []byte("[{")) - - if err = json.NewDecoder(bytes.NewReader(out[offset:])).Decode(&ents); err != nil { - return nil, fmt.Errorf("failed to decode json: %v", err) + if err := execCommandJSON(&ents, "ceph", args...); err != nil { + return nil, err } if len(ents) != 1 { - return nil, fmt.Errorf("got unexpected number of entities for %s: expected 1, got %d", entityName, len(ents)) + return nil, fmt.Errorf("got unexpected number of entities: expected 1, got %d", len(ents)) } return &ents[0], nil } +func genUserIDs(adminCr *credentials, volID volumeID) (adminID, userID string) { + return cephEntityClientPrefix + adminCr.id, cephEntityClientPrefix + getCephUserName(volID) +} + +func getCephUser(volOptions *volumeOptions, adminCr *credentials, volID volumeID) (*cephEntity, error) { + adminID, userID := genUserIDs(adminCr, volID) + + return getSingleCephEntity( + "-m", volOptions.Monitors, + "-n", adminID, + "--key="+adminCr.key, + "-c", cephConfigPath, + "-f", "json", + "auth", "get", userID, + ) +} + func createCephUser(volOptions *volumeOptions, adminCr *credentials, volID volumeID) (*cephEntity, error) { - caps := cephEntityCaps{ - Mds: fmt.Sprintf("allow rw path=%s", getVolumeRootPathCeph(volID)), - Mon: "allow r", - Osd: fmt.Sprintf("allow rw pool=%s namespace=%s", volOptions.Pool, getVolumeNamespace(volID)), - } + adminID, userID := genUserIDs(adminCr, volID) - var ents []cephEntity - args := [...]string{ + return getSingleCephEntity( "-m", volOptions.Monitors, - "auth", "-f", "json", "-c", cephConfigPath, "-n", cephEntityClientPrefix + adminCr.id, "--keyring", getCephKeyringPath(volID, adminCr.id), - "get-or-create", cephEntityClientPrefix + getCephUserName(volID), - "mds", caps.Mds, - "mon", caps.Mon, - "osd", caps.Osd, - } - - if err := execCommandJSON(&ents, args[:]...); err != nil { - return nil, fmt.Errorf("error creating ceph user: %v", err) - } - - return &ents[0], nil + "-n", adminID, + "--key="+adminCr.key, + "-c", cephConfigPath, + "-f", "json", + "auth", "get-or-create", userID, + // User capabilities + "mds", fmt.Sprintf("allow rw path=%s", getVolumeRootPathCeph(volID)), + "mon", "allow r", + "osd", fmt.Sprintf("allow rw pool=%s namespace=%s", volOptions.Pool, getVolumeNamespace(volID)), + ) } func deleteCephUser(volOptions *volumeOptions, adminCr *credentials, volID volumeID) error { - userID := getCephUserName(volID) + adminID, userID := genUserIDs(adminCr, volID) - args := [...]string{ + return execCommandErr("ceph", "-m", volOptions.Monitors, - "-c", cephConfigPath, "-n", cephEntityClientPrefix + adminCr.id, "--keyring", getCephKeyringPath(volID, adminCr.id), - "auth", "rm", cephEntityClientPrefix + userID, - } - - var err error - if err = execCommandAndValidate("ceph", args[:]...); err != nil { - return err - } - - keyringPath := getCephKeyringPath(volID, adminCr.id) - if err = os.Remove(keyringPath); err != nil { - klog.Errorf("failed to remove keyring file %s with error %s", keyringPath, err) - } - - secretPath := getCephSecretPath(volID, adminCr.id) - if err = os.Remove(secretPath); err != nil { - klog.Errorf("failed to remove secret file %s with error %s", secretPath, err) - } - - return nil + "-n", adminID, + "--key="+adminCr.key, + "-c", cephConfigPath, + "auth", "rm", userID, + ) } diff --git a/pkg/cephfs/controllerserver.go b/pkg/cephfs/controllerserver.go index 76bb99059..c2e596cbf 100644 --- a/pkg/cephfs/controllerserver.go +++ b/pkg/cephfs/controllerserver.go @@ -17,15 +17,15 @@ limitations under the License. package cephfs import ( + csicommon "github.com/ceph/ceph-csi/pkg/csi-common" + "github.com/ceph/ceph-csi/pkg/util" + + "github.com/container-storage-interface/spec/lib/go/csi" "golang.org/x/net/context" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "k8s.io/klog" - - "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/kubernetes-csi/drivers/pkg/csi-common" - - "github.com/ceph/ceph-csi/pkg/util" + "k8s.io/kubernetes/pkg/util/keymutex" ) // ControllerServer struct of CEPH CSI driver with supported methods of CSI @@ -40,6 +40,10 @@ type controllerCacheEntry struct { VolumeID volumeID } +var ( + mtxControllerVolumeID = keymutex.NewHashed(0) +) + // CreateVolume creates the volume in backend and store the volume metadata func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) { if err := cs.validateCreateVolumeRequest(req); err != nil { @@ -58,6 +62,9 @@ func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol volID := makeVolumeID(req.GetName()) + mtxControllerVolumeID.LockKey(string(volID)) + defer mustUnlock(mtxControllerVolumeID, string(volID)) + // Create a volume in case the user didn't provide one if volOptions.ProvisionVolume { @@ -67,11 +74,6 @@ func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol return nil, status.Error(codes.InvalidArgument, err.Error()) } - if err = storeCephCredentials(volID, cr); err != nil { - klog.Errorf("failed to store admin credentials for '%s': %v", cr.id, err) - return nil, status.Error(codes.Internal, err.Error()) - } - if err = createVolume(volOptions, cr, volID, req.GetCapacityRange().GetRequiredBytes()); err != nil { klog.Errorf("failed to create volume %s: %v", req.GetName(), err) return nil, status.Error(codes.Internal, err.Error()) @@ -102,8 +104,9 @@ func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol }, nil } -// DeleteVolume deletes the volume in backend and removes the volume metadata -// from store +// DeleteVolume deletes the volume in backend +// and removes the volume metadata from store +// nolint: gocyclo func (cs *ControllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) { if err := cs.validateDeleteVolumeRequest(); err != nil { klog.Errorf("DeleteVolumeRequest validation failed: %v", err) @@ -113,11 +116,15 @@ func (cs *ControllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVol var ( volID = volumeID(req.GetVolumeId()) secrets = req.GetSecrets() - err error ) ce := &controllerCacheEntry{} - if err = cs.MetadataStore.Get(string(volID), ce); err != nil { + if err := cs.MetadataStore.Get(string(volID), ce); err != nil { + if err, ok := err.(*util.CacheEntryNotFound); ok { + klog.Infof("cephfs: metadata for volume %s not found, assuming the volume to be already deleted (%v)", volID, err) + return &csi.DeleteVolumeResponse{}, nil + } + return nil, status.Error(codes.Internal, err.Error()) } @@ -143,6 +150,9 @@ func (cs *ControllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVol return nil, status.Error(codes.InvalidArgument, err.Error()) } + mtxControllerVolumeID.LockKey(string(volID)) + defer mustUnlock(mtxControllerVolumeID, string(volID)) + if err = purgeVolume(volID, cr, &ce.VolOptions); err != nil { klog.Errorf("failed to delete volume %s: %v", volID, err) return nil, status.Error(codes.Internal, err.Error()) diff --git a/pkg/cephfs/driver.go b/pkg/cephfs/driver.go index d69d11aa7..7c5dfc61e 100644 --- a/pkg/cephfs/driver.go +++ b/pkg/cephfs/driver.go @@ -19,19 +19,21 @@ package cephfs import ( "k8s.io/klog" - "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/kubernetes-csi/drivers/pkg/csi-common" - + csicommon "github.com/ceph/ceph-csi/pkg/csi-common" "github.com/ceph/ceph-csi/pkg/util" + + "github.com/container-storage-interface/spec/lib/go/csi" ) const ( - // PluginFolder defines the location of ceph plugin - PluginFolder = "/var/lib/kubelet/plugins/csi-cephfsplugin" + // version of ceph driver version = "1.0.0" ) +// PluginFolder defines the location of ceph plugin +var PluginFolder = "/var/lib/kubelet/plugins/" + // Driver contains the default identity,node and controller struct type Driver struct { cd *csicommon.CSIDriver @@ -75,7 +77,7 @@ func NewNodeServer(d *csicommon.CSIDriver) *NodeServer { // Run start a non-blocking grpc controller,node and identityserver for // ceph CSI driver which can serve multiple parallel requests -func (fs *Driver) Run(driverName, nodeID, endpoint, volumeMounter string, cachePersister util.CachePersister) { +func (fs *Driver) Run(driverName, nodeID, endpoint, volumeMounter, mountCacheDir string, cachePersister util.CachePersister) { klog.Infof("Driver: %v version: %v", driverName, version) // Configuration @@ -103,6 +105,13 @@ func (fs *Driver) Run(driverName, nodeID, endpoint, volumeMounter string, cacheP klog.Fatalf("failed to write ceph configuration file: %v", err) } + initVolumeMountCache(driverName, mountCacheDir, cachePersister) + if mountCacheDir != "" { + if err := remountCachedVolumes(); err != nil { + klog.Warningf("failed to remount cached volumes: %v", err) + //ignore remount fail + } + } // Initialize default library driver fs.cd = csicommon.NewCSIDriver(driverName, version, nodeID) diff --git a/pkg/cephfs/identityserver.go b/pkg/cephfs/identityserver.go index 9f3a6b4fd..c8d5edc52 100644 --- a/pkg/cephfs/identityserver.go +++ b/pkg/cephfs/identityserver.go @@ -19,8 +19,9 @@ package cephfs import ( "context" + csicommon "github.com/ceph/ceph-csi/pkg/csi-common" + "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/kubernetes-csi/drivers/pkg/csi-common" ) // IdentityServer struct of ceph CSI driver with supported methods of CSI diff --git a/pkg/cephfs/mountcache.go b/pkg/cephfs/mountcache.go new file mode 100644 index 000000000..c8bc2a03b --- /dev/null +++ b/pkg/cephfs/mountcache.go @@ -0,0 +1,311 @@ +package cephfs + +import ( + "encoding/base64" + "os" + "sync" + "syscall" + "time" + + "github.com/ceph/ceph-csi/pkg/util" + "github.com/pkg/errors" + "k8s.io/klog" +) + +type volumeMountCacheEntry struct { + DriverVersion string `json:"driverVersion"` + + VolumeID string `json:"volumeID"` + Secrets map[string]string `json:"secrets"` + StagingPath string `json:"stagingPath"` + TargetPaths map[string]bool `json:"targetPaths"` + CreateTime time.Time `json:"createTime"` +} + +type volumeMountCacheMap struct { + volumes map[string]volumeMountCacheEntry + nodeCacheStore util.NodeCache + metadataStore util.CachePersister +} + +var ( + volumeMountCachePrefix = "cephfs-mount-cache-" + volumeMountCache volumeMountCacheMap + volumeMountCacheMtx sync.Mutex +) + +func initVolumeMountCache(driverName string, mountCacheDir string, cachePersister util.CachePersister) { + volumeMountCache.volumes = make(map[string]volumeMountCacheEntry) + + volumeMountCache.metadataStore = cachePersister + volumeMountCache.nodeCacheStore.BasePath = mountCacheDir + volumeMountCache.nodeCacheStore.CacheDir = driverName + klog.Infof("mount-cache: name: %s, version: %s, mountCacheDir: %s", driverName, version, mountCacheDir) +} + +func remountCachedVolumes() error { + if err := os.MkdirAll(volumeMountCache.nodeCacheStore.BasePath, 0755); err != nil { + klog.Errorf("mount-cache: failed to create %s: %v", volumeMountCache.nodeCacheStore.BasePath, err) + return err + } + var remountFailCount, remountSuccCount int64 + me := &volumeMountCacheEntry{} + ce := &controllerCacheEntry{} + err := volumeMountCache.nodeCacheStore.ForAll(volumeMountCachePrefix, me, func(identifier string) error { + volID := me.VolumeID + if err := volumeMountCache.metadataStore.Get(volID, ce); err != nil { + if err, ok := err.(*util.CacheEntryNotFound); ok { + klog.Infof("mount-cache: metadata not found, assuming the volume %s to be already deleted (%v)", volID, err) + if err := volumeMountCache.nodeCacheStore.Delete(genVolumeMountCacheFileName(volID)); err == nil { + klog.Infof("mount-cache: metadata not found, delete volume cache entry for volume %s", volID) + } + } + } else { + if err := mountOneCacheEntry(ce, me); err == nil { + remountSuccCount++ + volumeMountCache.volumes[me.VolumeID] = *me + klog.Infof("mount-cache: successfully remounted volume %s", volID) + } else { + remountFailCount++ + klog.Errorf("mount-cache: failed to remount volume %s", volID) + } + } + return nil + }) + if err != nil { + klog.Infof("mount-cache: metastore list cache fail %v", err) + return err + } + if remountFailCount > 0 { + klog.Infof("mount-cache: successfully remounted %d volumes, failed to remount %d volumes", remountSuccCount, remountFailCount) + } else { + klog.Infof("mount-cache: successfully remounted %d volumes", remountSuccCount) + } + return nil +} + +func mountOneCacheEntry(ce *controllerCacheEntry, me *volumeMountCacheEntry) error { + volumeMountCacheMtx.Lock() + defer volumeMountCacheMtx.Unlock() + + var ( + err error + cr *credentials + ) + volID := ce.VolumeID + volOptions := ce.VolOptions + + if volOptions.ProvisionVolume { + volOptions.RootPath = getVolumeRootPathCeph(volID) + cr, err = getAdminCredentials(decodeCredentials(me.Secrets)) + if err != nil { + return err + } + var entity *cephEntity + entity, err = getCephUser(&volOptions, cr, volID) + if err != nil { + return err + } + cr = entity.toCredentials() + } else { + cr, err = getUserCredentials(decodeCredentials(me.Secrets)) + if err != nil { + return err + } + } + + err = cleanupMountPoint(me.StagingPath) + if err != nil { + klog.Infof("mount-cache: failed to cleanup volume mount point %s, remove it: %s %v", volID, me.StagingPath, err) + return err + } + + isMnt, err := isMountPoint(me.StagingPath) + if err != nil { + isMnt = false + klog.Infof("mount-cache: failed to check volume mounted %s: %s %v", volID, me.StagingPath, err) + } + + if !isMnt { + m, err := newMounter(&volOptions) + if err != nil { + klog.Errorf("mount-cache: failed to create mounter for volume %s: %v", volID, err) + return err + } + if err := m.mount(me.StagingPath, cr, &volOptions); err != nil { + klog.Errorf("mount-cache: failed to mount volume %s: %v", volID, err) + return err + } + } + for targetPath, readOnly := range me.TargetPaths { + if err := cleanupMountPoint(targetPath); err == nil { + if err := bindMount(me.StagingPath, targetPath, readOnly); err != nil { + klog.Errorf("mount-cache: failed to bind-mount volume %s: %s %s %v %v", + volID, me.StagingPath, targetPath, readOnly, err) + } else { + klog.Infof("mount-cache: successfully bind-mounted volume %s: %s %s %v", + volID, me.StagingPath, targetPath, readOnly) + } + } + } + return nil +} + +func cleanupMountPoint(mountPoint string) error { + if _, err := os.Stat(mountPoint); err != nil { + if isCorruptedMnt(err) { + klog.Infof("mount-cache: corrupted mount point %s, need unmount", mountPoint) + err := execCommandErr("umount", mountPoint) + if err != nil { + klog.Infof("mount-cache: failed to umount %s %v", mountPoint, err) + //ignore error return err + } + } + } + if _, err := os.Stat(mountPoint); err != nil { + klog.Errorf("mount-cache: failed to stat mount point %s %v", mountPoint, err) + return err + } + return nil +} + +func isCorruptedMnt(err error) bool { + var underlyingError error + switch pe := err.(type) { + case nil: + return false + case *os.PathError: + underlyingError = pe.Err + case *os.LinkError: + underlyingError = pe.Err + case *os.SyscallError: + underlyingError = pe.Err + default: + return false + } + + CorruptedErrors := []error{ + syscall.ENOTCONN, syscall.ESTALE, syscall.EIO, syscall.EACCES} + + for _, v := range CorruptedErrors { + if underlyingError == v { + return true + } + } + return false +} + +func genVolumeMountCacheFileName(volID string) string { + cachePath := volumeMountCachePrefix + volID + return cachePath +} +func (mc *volumeMountCacheMap) isEnable() bool { + //if mount cache dir unset, disable state + return mc.nodeCacheStore.BasePath != "" +} + +func (mc *volumeMountCacheMap) nodeStageVolume(volID string, stagingTargetPath string, secrets map[string]string) error { + if !mc.isEnable() { + return nil + } + volumeMountCacheMtx.Lock() + defer volumeMountCacheMtx.Unlock() + + lastTargetPaths := make(map[string]bool) + me, ok := volumeMountCache.volumes[volID] + if ok { + if me.StagingPath == stagingTargetPath { + klog.Warningf("mount-cache: node unexpected restage volume for volume %s", volID) + return nil + } + lastTargetPaths = me.TargetPaths + klog.Warningf("mount-cache: node stage volume ignore last cache entry for volume %s", volID) + } + + me = volumeMountCacheEntry{DriverVersion: version} + + me.VolumeID = volID + me.Secrets = encodeCredentials(secrets) + me.StagingPath = stagingTargetPath + me.TargetPaths = lastTargetPaths + + me.CreateTime = time.Now() + volumeMountCache.volumes[volID] = me + return mc.nodeCacheStore.Create(genVolumeMountCacheFileName(volID), me) +} + +func (mc *volumeMountCacheMap) nodeUnStageVolume(volID string) error { + if !mc.isEnable() { + return nil + } + volumeMountCacheMtx.Lock() + defer volumeMountCacheMtx.Unlock() + delete(volumeMountCache.volumes, volID) + return mc.nodeCacheStore.Delete(genVolumeMountCacheFileName(volID)) +} + +func (mc *volumeMountCacheMap) nodePublishVolume(volID string, targetPath string, readOnly bool) error { + if !mc.isEnable() { + return nil + } + volumeMountCacheMtx.Lock() + defer volumeMountCacheMtx.Unlock() + + _, ok := volumeMountCache.volumes[volID] + if !ok { + return errors.New("mount-cache: node publish volume failed to find cache entry for volume") + } + volumeMountCache.volumes[volID].TargetPaths[targetPath] = readOnly + return mc.updateNodeCache(volID) +} + +func (mc *volumeMountCacheMap) nodeUnPublishVolume(volID string, targetPath string) error { + if !mc.isEnable() { + return nil + } + volumeMountCacheMtx.Lock() + defer volumeMountCacheMtx.Unlock() + + _, ok := volumeMountCache.volumes[volID] + if !ok { + return errors.New("mount-cache: node unpublish volume failed to find cache entry for volume") + } + delete(volumeMountCache.volumes[volID].TargetPaths, targetPath) + return mc.updateNodeCache(volID) +} + +func (mc *volumeMountCacheMap) updateNodeCache(volID string) error { + me := volumeMountCache.volumes[volID] + if err := volumeMountCache.nodeCacheStore.Delete(genVolumeMountCacheFileName(volID)); err == nil { + klog.Infof("mount-cache: metadata not found, delete mount cache failed for volume %s", volID) + } + return mc.nodeCacheStore.Create(genVolumeMountCacheFileName(volID), me) +} + +func encodeCredentials(input map[string]string) (output map[string]string) { + output = make(map[string]string) + for key, value := range input { + nKey := base64.StdEncoding.EncodeToString([]byte(key)) + nValue := base64.StdEncoding.EncodeToString([]byte(value)) + output[nKey] = nValue + } + return output +} + +func decodeCredentials(input map[string]string) (output map[string]string) { + output = make(map[string]string) + for key, value := range input { + nKey, err := base64.StdEncoding.DecodeString(key) + if err != nil { + klog.Errorf("mount-cache: decode secret fail") + continue + } + nValue, err := base64.StdEncoding.DecodeString(value) + if err != nil { + klog.Errorf("mount-cache: decode secret fail") + continue + } + output[string(nKey)] = string(nValue) + } + return output +} diff --git a/pkg/cephfs/mountcache_test.go b/pkg/cephfs/mountcache_test.go new file mode 100644 index 000000000..e27053cd4 --- /dev/null +++ b/pkg/cephfs/mountcache_test.go @@ -0,0 +1,38 @@ +package cephfs + +import ( + "testing" +) + +func init() { +} + +func TestMountOneCacheEntry(t *testing.T) { +} + +func TestRemountHisMountedPath(t *testing.T) { +} + +func TestNodeStageVolume(t *testing.T) { +} + +func TestNodeUnStageVolume(t *testing.T) { +} + +func TestNodePublishVolume(t *testing.T) { +} + +func TestNodeUnpublishVolume(t *testing.T) { +} + +func TestEncodeDecodeCredentials(t *testing.T) { + secrets := make(map[string]string) + secrets["user_1"] = "value_1" + enSecrets := encodeCredentials(secrets) + deSecrets := decodeCredentials(enSecrets) + for key, value := range secrets { + if deSecrets[key] != value { + t.Errorf("key %s of credentials's value %s change after decode %s ", key, value, deSecrets[key]) + } + } +} diff --git a/pkg/cephfs/nodeserver.go b/pkg/cephfs/nodeserver.go index 4fff317b4..273471e05 100644 --- a/pkg/cephfs/nodeserver.go +++ b/pkg/cephfs/nodeserver.go @@ -21,12 +21,13 @@ import ( "fmt" "os" + csicommon "github.com/ceph/ceph-csi/pkg/csi-common" + + "github.com/container-storage-interface/spec/lib/go/csi" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "k8s.io/klog" - - "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/kubernetes-csi/drivers/pkg/csi-common" + "k8s.io/kubernetes/pkg/util/keymutex" ) // NodeServer struct of ceph CSI driver with supported methods of CSI @@ -35,6 +36,10 @@ type NodeServer struct { *csicommon.DefaultNodeServer } +var ( + mtxNodeVolumeID = keymutex.NewHashed(0) +) + func getCredentialsForVolume(volOptions *volumeOptions, volID volumeID, req *csi.NodeStageVolumeRequest) (*credentials, error) { var ( cr *credentials @@ -44,17 +49,13 @@ func getCredentialsForVolume(volOptions *volumeOptions, volID volumeID, req *csi if volOptions.ProvisionVolume { // The volume is provisioned dynamically, get the credentials directly from Ceph - // First, store admin credentials - those are needed for retrieving the user credentials + // First, get admin credentials - those are needed for retrieving the user credentials adminCr, err := getAdminCredentials(secrets) if err != nil { return nil, fmt.Errorf("failed to get admin credentials from node stage secrets: %v", err) } - if err = storeCephCredentials(volID, adminCr); err != nil { - return nil, fmt.Errorf("failed to store ceph admin credentials: %v", err) - } - // Then get the ceph user entity, err := getCephUser(volOptions, adminCr, volID) @@ -74,10 +75,6 @@ func getCredentialsForVolume(volOptions *volumeOptions, volID volumeID, req *csi cr = userCr } - if err := storeCephCredentials(volID, cr); err != nil { - return nil, fmt.Errorf("failed to store ceph user credentials: %v", err) - } - return cr, nil } @@ -108,6 +105,9 @@ func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol return nil, status.Error(codes.Internal, err.Error()) } + mtxNodeVolumeID.LockKey(string(volID)) + defer mustUnlock(mtxNodeVolumeID, string(volID)) + // Check if the volume is already mounted isMnt, err := isMountPoint(stagingTargetPath) @@ -150,10 +150,13 @@ func (*NodeServer) mount(volOptions *volumeOptions, req *csi.NodeStageVolumeRequ klog.V(4).Infof("cephfs: mounting volume %s with %s", volID, m.name()) - if err = m.mount(stagingTargetPath, cr, volOptions, volID); err != nil { + if err = m.mount(stagingTargetPath, cr, volOptions); err != nil { klog.Errorf("failed to mount volume %s: %v", volID, err) return status.Error(codes.Internal, err.Error()) } + if err := volumeMountCache.nodeStageVolume(req.GetVolumeId(), stagingTargetPath, req.GetSecrets()); err != nil { + klog.Warningf("mount-cache: failed to stage volume %s %s: %v", volID, stagingTargetPath, err) + } return nil } @@ -195,6 +198,10 @@ func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis return nil, status.Error(codes.Internal, err.Error()) } + if err := volumeMountCache.nodePublishVolume(volID, targetPath, req.GetReadonly()); err != nil { + klog.Warningf("mount-cache: failed to publish volume %s %s: %v", volID, targetPath, err) + } + klog.Infof("cephfs: successfully bind-mounted volume %s to %s", volID, targetPath) return &csi.NodePublishVolumeResponse{}, nil @@ -209,6 +216,11 @@ func (ns *NodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpu targetPath := req.GetTargetPath() + volID := req.GetVolumeId() + if err = volumeMountCache.nodeUnPublishVolume(volID, targetPath); err != nil { + klog.Warningf("mount-cache: failed to unpublish volume %s %s: %v", volID, targetPath, err) + } + // Unmount the bind-mount if err = unmountVolume(targetPath); err != nil { return nil, status.Error(codes.Internal, err.Error()) @@ -232,6 +244,11 @@ func (ns *NodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstag stagingTargetPath := req.GetStagingTargetPath() + volID := req.GetVolumeId() + if err = volumeMountCache.nodeUnStageVolume(volID); err != nil { + klog.Warningf("mount-cache: failed to unstage volume %s %s: %v", volID, stagingTargetPath, err) + } + // Unmount the volume if err = unmountVolume(stagingTargetPath); err != nil { return nil, status.Error(codes.Internal, err.Error()) @@ -241,7 +258,7 @@ func (ns *NodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstag return nil, status.Error(codes.Internal, err.Error()) } - klog.Infof("cephfs: successfully umounted volume %s from %s", req.GetVolumeId(), stagingTargetPath) + klog.Infof("cephfs: successfully unmounted volume %s from %s", req.GetVolumeId(), stagingTargetPath) return &csi.NodeUnstageVolumeResponse{}, nil } diff --git a/pkg/cephfs/util.go b/pkg/cephfs/util.go index e9314f532..19928c0fa 100644 --- a/pkg/cephfs/util.go +++ b/pkg/cephfs/util.go @@ -21,49 +21,70 @@ import ( "encoding/json" "errors" "fmt" + "os" "os/exec" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "k8s.io/klog" + "github.com/ceph/ceph-csi/pkg/util" "github.com/container-storage-interface/spec/lib/go/csi" + "k8s.io/kubernetes/pkg/util/keymutex" "k8s.io/kubernetes/pkg/util/mount" ) type volumeID string +func mustUnlock(m keymutex.KeyMutex, key string) { + if err := m.UnlockKey(key); err != nil { + klog.Fatalf("failed to unlock mutex for %s: %v", key, err) + } +} + func makeVolumeID(volName string) volumeID { return volumeID("csi-cephfs-" + volName) } -func execCommand(command string, args ...string) ([]byte, error) { - klog.V(4).Infof("cephfs: EXEC %s %s", command, args) +func execCommand(program string, args ...string) (stdout, stderr []byte, err error) { + var ( + cmd = exec.Command(program, args...) // nolint: gosec + sanitizedArgs = util.StripSecretInArgs(args) + stdoutBuf bytes.Buffer + stderrBuf bytes.Buffer + ) - cmd := exec.Command(command, args...) // #nosec - return cmd.CombinedOutput() + cmd.Stdout = &stdoutBuf + cmd.Stderr = &stderrBuf + + klog.V(4).Infof("cephfs: EXEC %s %s", program, sanitizedArgs) + + if err := cmd.Run(); err != nil { + return nil, nil, fmt.Errorf("an error occurred while running (%d) %s %v: %v: %s", + cmd.Process.Pid, program, sanitizedArgs, err, stderrBuf.Bytes()) + } + + return stdoutBuf.Bytes(), stderrBuf.Bytes(), nil } -func execCommandAndValidate(program string, args ...string) error { - out, err := execCommand(program, args...) +func execCommandErr(program string, args ...string) error { + _, _, err := execCommand(program, args...) + return err +} + +func execCommandJSON(v interface{}, program string, args ...string) error { + stdout, _, err := execCommand(program, args...) if err != nil { - return fmt.Errorf("cephfs: %s failed with following error: %s\ncephfs: %s output: %s", program, err, program, out) + return err + } + + if err = json.Unmarshal(stdout, v); err != nil { + return fmt.Errorf("failed to unmarshal JSON for %s %v: %s: %v", program, util.StripSecretInArgs(args), stdout, err) } return nil } -func execCommandJSON(v interface{}, args ...string) error { - program := "ceph" - out, err := execCommand(program, args...) - - if err != nil { - return fmt.Errorf("cephfs: %s failed with following error: %s\ncephfs: %s output: %s", program, err, program, out) - } - - return json.NewDecoder(bytes.NewReader(out)).Decode(v) -} - // Used in isMountPoint() var dummyMount = mount.New("") @@ -76,31 +97,12 @@ func isMountPoint(p string) (bool, error) { return !notMnt, nil } -func storeCephCredentials(volID volumeID, cr *credentials) error { - keyringData := cephKeyringData{ - UserID: cr.id, - Key: cr.key, - VolumeID: volID, - } - - if err := keyringData.writeToFile(); err != nil { - return err - } - - secret := cephSecretData{ - UserID: cr.id, - Key: cr.key, - VolumeID: volID, - } - - err := secret.writeToFile() - return err +func pathExists(p string) bool { + _, err := os.Stat(p) + return err == nil } -// // Controller service request validation -// - func (cs *ControllerServer) validateCreateVolumeRequest(req *csi.CreateVolumeRequest) error { if err := cs.Driver.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME); err != nil { return fmt.Errorf("invalid CreateVolumeRequest: %v", err) @@ -132,10 +134,7 @@ func (cs *ControllerServer) validateDeleteVolumeRequest() error { return nil } -// // Node service request validation -// - func validateNodeStageVolumeRequest(req *csi.NodeStageVolumeRequest) error { if req.GetVolumeCapability() == nil { return errors.New("volume capability missing in request") @@ -178,7 +177,7 @@ func validateNodePublishVolumeRequest(req *csi.NodePublishVolumeRequest) error { } if req.GetTargetPath() == "" { - return errors.New("varget path missing in request") + return errors.New("target path missing in request") } return nil diff --git a/pkg/cephfs/volume.go b/pkg/cephfs/volume.go index e228892c8..a4d6f197d 100644 --- a/pkg/cephfs/volume.go +++ b/pkg/cephfs/volume.go @@ -25,14 +25,13 @@ import ( ) const ( - cephRootPrefix = PluginFolder + "/controller/volumes/root-" cephVolumesRoot = "csi-volumes" namespacePrefix = "ns-" ) func getCephRootPathLocal(volID volumeID) string { - return cephRootPrefix + string(volID) + return fmt.Sprintf("%s/controller/volumes/root-%s", PluginFolder, string(volID)) } func getCephRootVolumePathLocal(volID volumeID) string { @@ -48,83 +47,70 @@ func getVolumeNamespace(volID volumeID) string { } func setVolumeAttribute(root, attrName, attrValue string) error { - return execCommandAndValidate("setfattr", "-n", attrName, "-v", attrValue, root) + return execCommandErr("setfattr", "-n", attrName, "-v", attrValue, root) } func createVolume(volOptions *volumeOptions, adminCr *credentials, volID volumeID, bytesQuota int64) error { - cephRoot := getCephRootPathLocal(volID) - - if err := createMountPoint(cephRoot); err != nil { + if err := mountCephRoot(volID, volOptions, adminCr); err != nil { return err } + defer unmountCephRoot(volID) - // RootPath is not set for a dynamically provisioned volume - // Access to cephfs's / is required - volOptions.RootPath = "/" + var ( + volRoot = getCephRootVolumePathLocal(volID) + volRootCreating = volRoot + "-creating" + ) - m, err := newMounter(volOptions) - if err != nil { - return fmt.Errorf("failed to create mounter: %v", err) + if pathExists(volRoot) { + klog.V(4).Infof("cephfs: volume %s already exists, skipping creation", volID) + return nil } - if err = m.mount(cephRoot, adminCr, volOptions, volID); err != nil { - return fmt.Errorf("error mounting ceph root: %v", err) - } - - defer unmountAndRemove(cephRoot) - - volOptions.RootPath = getVolumeRootPathCeph(volID) - localVolRoot := getCephRootVolumePathLocal(volID) - - if err := createMountPoint(localVolRoot); err != nil { + if err := createMountPoint(volRootCreating); err != nil { return err } if bytesQuota > 0 { - if err := setVolumeAttribute(localVolRoot, "ceph.quota.max_bytes", fmt.Sprintf("%d", bytesQuota)); err != nil { + if err := setVolumeAttribute(volRootCreating, "ceph.quota.max_bytes", fmt.Sprintf("%d", bytesQuota)); err != nil { return err } } - if err := setVolumeAttribute(localVolRoot, "ceph.dir.layout.pool", volOptions.Pool); err != nil { + if err := setVolumeAttribute(volRootCreating, "ceph.dir.layout.pool", volOptions.Pool); err != nil { return fmt.Errorf("%v\ncephfs: Does pool '%s' exist?", err, volOptions.Pool) } - if err := setVolumeAttribute(localVolRoot, "ceph.dir.layout.pool_namespace", getVolumeNamespace(volID)); err != nil { + if err := setVolumeAttribute(volRootCreating, "ceph.dir.layout.pool_namespace", getVolumeNamespace(volID)); err != nil { return err } + if err := os.Rename(volRootCreating, volRoot); err != nil { + return fmt.Errorf("couldn't mark volume %s as created: %v", volID, err) + } + return nil } func purgeVolume(volID volumeID, adminCr *credentials, volOptions *volumeOptions) error { + if err := mountCephRoot(volID, volOptions, adminCr); err != nil { + return err + } + defer unmountCephRoot(volID) + var ( - cephRoot = getCephRootPathLocal(volID) volRoot = getCephRootVolumePathLocal(volID) volRootDeleting = volRoot + "-deleting" ) - if err := createMountPoint(cephRoot); err != nil { - return err - } - - // Root path is not set for dynamically provisioned volumes - // Access to cephfs's / is required - volOptions.RootPath = "/" - - m, err := newMounter(volOptions) - if err != nil { - return fmt.Errorf("failed to create mounter: %v", err) - } - - if err = m.mount(cephRoot, adminCr, volOptions, volID); err != nil { - return fmt.Errorf("error mounting ceph root: %v", err) - } - - defer unmountAndRemove(cephRoot) - - if err := os.Rename(volRoot, volRootDeleting); err != nil { - return fmt.Errorf("coudln't mark volume %s for deletion: %v", volID, err) + if pathExists(volRoot) { + if err := os.Rename(volRoot, volRootDeleting); err != nil { + return fmt.Errorf("couldn't mark volume %s for deletion: %v", volID, err) + } + } else { + if !pathExists(volRootDeleting) { + klog.V(4).Infof("cephfs: volume %s not found, assuming it to be already deleted", volID) + return nil + } } if err := os.RemoveAll(volRootDeleting); err != nil { @@ -134,13 +120,37 @@ func purgeVolume(volID volumeID, adminCr *credentials, volOptions *volumeOptions return nil } -func unmountAndRemove(mountPoint string) { - var err error - if err = unmountVolume(mountPoint); err != nil { - klog.Errorf("failed to unmount %s with error %s", mountPoint, err) +func mountCephRoot(volID volumeID, volOptions *volumeOptions, adminCr *credentials) error { + cephRoot := getCephRootPathLocal(volID) + + // Root path is not set for dynamically provisioned volumes + // Access to cephfs's / is required + volOptions.RootPath = "/" + + if err := createMountPoint(cephRoot); err != nil { + return err } - if err = os.Remove(mountPoint); err != nil { - klog.Errorf("failed to remove %s with error %s", mountPoint, err) + m, err := newMounter(volOptions) + if err != nil { + return fmt.Errorf("failed to create mounter: %v", err) + } + + if err = m.mount(cephRoot, adminCr, volOptions); err != nil { + return fmt.Errorf("error mounting ceph root: %v", err) + } + + return nil +} + +func unmountCephRoot(volID volumeID) { + cephRoot := getCephRootPathLocal(volID) + + if err := unmountVolume(cephRoot); err != nil { + klog.Errorf("failed to unmount %s with error %s", cephRoot, err) + } else { + if err := os.Remove(cephRoot); err != nil { + klog.Errorf("failed to remove %s with error %s", cephRoot, err) + } } } diff --git a/pkg/cephfs/volumemounter.go b/pkg/cephfs/volumemounter.go index 6a78f8266..6a91afafc 100644 --- a/pkg/cephfs/volumemounter.go +++ b/pkg/cephfs/volumemounter.go @@ -17,11 +17,15 @@ limitations under the License. package cephfs import ( - "bytes" "errors" "fmt" "os" "os/exec" + "regexp" + "strconv" + "sync" + + "k8s.io/klog" ) const ( @@ -31,6 +35,12 @@ const ( var ( availableMounters []string + + // maps a mountpoint to PID of its FUSE daemon + fusePidMap = make(map[string]int) + fusePidMapMtx sync.Mutex + + fusePidRx = regexp.MustCompile(`(?m)^ceph-fuse\[(.+)\]: starting fuse$`) ) // Load available ceph mounters installed on system into availableMounters @@ -57,7 +67,7 @@ func loadAvailableMounters() error { } type volumeMounter interface { - mount(mountPoint string, cr *credentials, volOptions *volumeOptions, volID volumeID) error + mount(mountPoint string, cr *credentials, volOptions *volumeOptions) error name() string } @@ -101,72 +111,84 @@ func newMounter(volOptions *volumeOptions) (volumeMounter, error) { type fuseMounter struct{} -func mountFuse(mountPoint string, cr *credentials, volOptions *volumeOptions, volID volumeID) error { +func mountFuse(mountPoint string, cr *credentials, volOptions *volumeOptions) error { args := [...]string{ mountPoint, "-m", volOptions.Monitors, "-c", cephConfigPath, - "-n", cephEntityClientPrefix + cr.id, - "--keyring", getCephKeyringPath(volID, cr.id), + "-n", cephEntityClientPrefix + cr.id, "--key=" + cr.key, "-r", volOptions.RootPath, "-o", "nonempty", } - out, err := execCommand("ceph-fuse", args[:]...) + _, stderr, err := execCommand("ceph-fuse", args[:]...) if err != nil { - return fmt.Errorf("cephfs: ceph-fuse failed with following error: %s\ncephfs: ceph-fuse output: %s", err, out) + return err } - if !bytes.Contains(out, []byte("starting fuse")) { - return fmt.Errorf("cephfs: ceph-fuse failed:\ncephfs: ceph-fuse output: %s", out) + // Parse the output: + // We need "starting fuse" meaning the mount is ok + // and PID of the ceph-fuse daemon for unmount + + match := fusePidRx.FindSubmatch(stderr) + if len(match) != 2 { + return fmt.Errorf("ceph-fuse failed: %s", stderr) } + pid, err := strconv.Atoi(string(match[1])) + if err != nil { + return fmt.Errorf("failed to parse FUSE daemon PID: %v", err) + } + + fusePidMapMtx.Lock() + fusePidMap[mountPoint] = pid + fusePidMapMtx.Unlock() + return nil } -func (m *fuseMounter) mount(mountPoint string, cr *credentials, volOptions *volumeOptions, volID volumeID) error { +func (m *fuseMounter) mount(mountPoint string, cr *credentials, volOptions *volumeOptions) error { if err := createMountPoint(mountPoint); err != nil { return err } - return mountFuse(mountPoint, cr, volOptions, volID) + return mountFuse(mountPoint, cr, volOptions) } func (m *fuseMounter) name() string { return "Ceph FUSE driver" } type kernelMounter struct{} -func mountKernel(mountPoint string, cr *credentials, volOptions *volumeOptions, volID volumeID) error { - if err := execCommandAndValidate("modprobe", "ceph"); err != nil { +func mountKernel(mountPoint string, cr *credentials, volOptions *volumeOptions) error { + if err := execCommandErr("modprobe", "ceph"); err != nil { return err } - return execCommandAndValidate("mount", + return execCommandErr("mount", "-t", "ceph", fmt.Sprintf("%s:%s", volOptions.Monitors, volOptions.RootPath), mountPoint, - "-o", - fmt.Sprintf("name=%s,secretfile=%s", cr.id, getCephSecretPath(volID, cr.id)), + "-o", fmt.Sprintf("name=%s,secret=%s", cr.id, cr.key), ) } -func (m *kernelMounter) mount(mountPoint string, cr *credentials, volOptions *volumeOptions, volID volumeID) error { +func (m *kernelMounter) mount(mountPoint string, cr *credentials, volOptions *volumeOptions) error { if err := createMountPoint(mountPoint); err != nil { return err } - return mountKernel(mountPoint, cr, volOptions, volID) + return mountKernel(mountPoint, cr, volOptions) } func (m *kernelMounter) name() string { return "Ceph kernel client" } func bindMount(from, to string, readOnly bool) error { - if err := execCommandAndValidate("mount", "--bind", from, to); err != nil { + if err := execCommandErr("mount", "--bind", from, to); err != nil { return fmt.Errorf("failed to bind-mount %s to %s: %v", from, to, err) } if readOnly { - if err := execCommandAndValidate("mount", "-o", "remount,ro,bind", to); err != nil { + if err := execCommandErr("mount", "-o", "remount,ro,bind", to); err != nil { return fmt.Errorf("failed read-only remount of %s: %v", to, err) } } @@ -175,7 +197,29 @@ func bindMount(from, to string, readOnly bool) error { } func unmountVolume(mountPoint string) error { - return execCommandAndValidate("umount", mountPoint) + if err := execCommandErr("umount", mountPoint); err != nil { + return err + } + + fusePidMapMtx.Lock() + pid, ok := fusePidMap[mountPoint] + if ok { + delete(fusePidMap, mountPoint) + } + fusePidMapMtx.Unlock() + + if ok { + p, err := os.FindProcess(pid) + if err != nil { + klog.Warningf("failed to find process %d: %v", pid, err) + } else { + if _, err = p.Wait(); err != nil { + klog.Warningf("%d is not a child process: %v", pid, err) + } + } + } + + return nil } func createMountPoint(root string) error { diff --git a/vendor/github.com/kubernetes-csi/drivers/pkg/csi-common/controllerserver-default.go b/pkg/csi-common/controllerserver-default.go similarity index 78% rename from vendor/github.com/kubernetes-csi/drivers/pkg/csi-common/controllerserver-default.go rename to pkg/csi-common/controllerserver-default.go index db72dc2e9..0424021b7 100644 --- a/vendor/github.com/kubernetes-csi/drivers/pkg/csi-common/controllerserver-default.go +++ b/pkg/csi-common/controllerserver-default.go @@ -18,40 +18,33 @@ package csicommon import ( "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/golang/glog" "golang.org/x/net/context" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "k8s.io/klog" ) +// DefaultControllerServer points to default driver type DefaultControllerServer struct { Driver *CSIDriver } -func (cs *DefaultControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) { - return nil, status.Error(codes.Unimplemented, "") -} - -func (cs *DefaultControllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) { - return nil, status.Error(codes.Unimplemented, "") -} - +// ControllerPublishVolume publish volume on node func (cs *DefaultControllerServer) ControllerPublishVolume(ctx context.Context, req *csi.ControllerPublishVolumeRequest) (*csi.ControllerPublishVolumeResponse, error) { return nil, status.Error(codes.Unimplemented, "") } +// ControllerUnpublishVolume unpublish on node func (cs *DefaultControllerServer) ControllerUnpublishVolume(ctx context.Context, req *csi.ControllerUnpublishVolumeRequest) (*csi.ControllerUnpublishVolumeResponse, error) { return nil, status.Error(codes.Unimplemented, "") } -func (cs *DefaultControllerServer) ValidateVolumeCapabilities(ctx context.Context, req *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error) { - return nil, status.Error(codes.Unimplemented, "") -} - +// ListVolumes lists volumes func (cs *DefaultControllerServer) ListVolumes(ctx context.Context, req *csi.ListVolumesRequest) (*csi.ListVolumesResponse, error) { return nil, status.Error(codes.Unimplemented, "") } +// GetCapacity get volume capacity func (cs *DefaultControllerServer) GetCapacity(ctx context.Context, req *csi.GetCapacityRequest) (*csi.GetCapacityResponse, error) { return nil, status.Error(codes.Unimplemented, "") } @@ -59,21 +52,24 @@ func (cs *DefaultControllerServer) GetCapacity(ctx context.Context, req *csi.Get // ControllerGetCapabilities implements the default GRPC callout. // Default supports all capabilities func (cs *DefaultControllerServer) ControllerGetCapabilities(ctx context.Context, req *csi.ControllerGetCapabilitiesRequest) (*csi.ControllerGetCapabilitiesResponse, error) { - glog.V(5).Infof("Using default ControllerGetCapabilities") + klog.V(5).Infof("Using default ControllerGetCapabilities") return &csi.ControllerGetCapabilitiesResponse{ Capabilities: cs.Driver.cap, }, nil } +// CreateSnapshot creates snapshot func (cs *DefaultControllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateSnapshotRequest) (*csi.CreateSnapshotResponse, error) { return nil, status.Error(codes.Unimplemented, "") } +// DeleteSnapshot deletes snapshot func (cs *DefaultControllerServer) DeleteSnapshot(ctx context.Context, req *csi.DeleteSnapshotRequest) (*csi.DeleteSnapshotResponse, error) { return nil, status.Error(codes.Unimplemented, "") } +// ListSnapshots lists snapshosts func (cs *DefaultControllerServer) ListSnapshots(ctx context.Context, req *csi.ListSnapshotsRequest) (*csi.ListSnapshotsResponse, error) { return nil, status.Error(codes.Unimplemented, "") } diff --git a/vendor/github.com/kubernetes-csi/drivers/pkg/csi-common/driver.go b/pkg/csi-common/driver.go similarity index 73% rename from vendor/github.com/kubernetes-csi/drivers/pkg/csi-common/driver.go rename to pkg/csi-common/driver.go index d20c6c808..f479fca9c 100644 --- a/vendor/github.com/kubernetes-csi/drivers/pkg/csi-common/driver.go +++ b/pkg/csi-common/driver.go @@ -19,13 +19,13 @@ package csicommon import ( "fmt" - "github.com/golang/glog" + "github.com/container-storage-interface/spec/lib/go/csi" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - - "github.com/container-storage-interface/spec/lib/go/csi" + "k8s.io/klog" ) +// CSIDriver stores driver information type CSIDriver struct { name string nodeID string @@ -34,21 +34,22 @@ type CSIDriver struct { vc []*csi.VolumeCapability_AccessMode } -// Creates a NewCSIDriver object. Assumes vendor version is equal to driver version & -// does not support optional driver plugin info manifest field. Refer to CSI spec for more details. +// NewCSIDriver Creates a NewCSIDriver object. Assumes vendor +// version is equal to driver version & does not support optional +// driver plugin info manifest field. Refer to CSI spec for more details. func NewCSIDriver(name string, v string, nodeID string) *CSIDriver { if name == "" { - glog.Errorf("Driver name missing") + klog.Errorf("Driver name missing") return nil } if nodeID == "" { - glog.Errorf("NodeID missing") + klog.Errorf("NodeID missing") return nil } // TODO version format and validation if len(v) == 0 { - glog.Errorf("Version argument missing") + klog.Errorf("Version argument missing") return nil } @@ -61,6 +62,8 @@ func NewCSIDriver(name string, v string, nodeID string) *CSIDriver { return &driver } +// ValidateControllerServiceRequest validates the controller +// plugin capabilities func (d *CSIDriver) ValidateControllerServiceRequest(c csi.ControllerServiceCapability_RPC_Type) error { if c == csi.ControllerServiceCapability_RPC_UNKNOWN { return nil @@ -71,32 +74,35 @@ func (d *CSIDriver) ValidateControllerServiceRequest(c csi.ControllerServiceCapa return nil } } - return status.Error(codes.InvalidArgument, fmt.Sprintf("%s", c)) + return status.Error(codes.InvalidArgument, fmt.Sprintf("%s", c)) //nolint } +// AddControllerServiceCapabilities stores the controller capabilities +// in driver object func (d *CSIDriver) AddControllerServiceCapabilities(cl []csi.ControllerServiceCapability_RPC_Type) { var csc []*csi.ControllerServiceCapability for _, c := range cl { - glog.Infof("Enabling controller service capability: %v", c.String()) + klog.Infof("Enabling controller service capability: %v", c.String()) csc = append(csc, NewControllerServiceCapability(c)) } d.cap = csc - return } +// AddVolumeCapabilityAccessModes stores volume access modes func (d *CSIDriver) AddVolumeCapabilityAccessModes(vc []csi.VolumeCapability_AccessMode_Mode) []*csi.VolumeCapability_AccessMode { var vca []*csi.VolumeCapability_AccessMode for _, c := range vc { - glog.Infof("Enabling volume access mode: %v", c.String()) + klog.Infof("Enabling volume access mode: %v", c.String()) vca = append(vca, NewVolumeCapabilityAccessMode(c)) } d.vc = vca return vca } +// GetVolumeCapabilityAccessModes returns access modes func (d *CSIDriver) GetVolumeCapabilityAccessModes() []*csi.VolumeCapability_AccessMode { return d.vc } diff --git a/vendor/github.com/kubernetes-csi/drivers/pkg/csi-common/identityserver-default.go b/pkg/csi-common/identityserver-default.go similarity index 86% rename from vendor/github.com/kubernetes-csi/drivers/pkg/csi-common/identityserver-default.go rename to pkg/csi-common/identityserver-default.go index b16b67c09..a206836ae 100644 --- a/vendor/github.com/kubernetes-csi/drivers/pkg/csi-common/identityserver-default.go +++ b/pkg/csi-common/identityserver-default.go @@ -18,18 +18,20 @@ package csicommon import ( "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/golang/glog" "golang.org/x/net/context" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "k8s.io/klog" ) +// DefaultIdentityServer stores driver object type DefaultIdentityServer struct { Driver *CSIDriver } +// GetPluginInfo returns plugin information func (ids *DefaultIdentityServer) GetPluginInfo(ctx context.Context, req *csi.GetPluginInfoRequest) (*csi.GetPluginInfoResponse, error) { - glog.V(5).Infof("Using default GetPluginInfo") + klog.V(5).Infof("Using default GetPluginInfo") if ids.Driver.name == "" { return nil, status.Error(codes.Unavailable, "Driver name not configured") @@ -45,12 +47,14 @@ func (ids *DefaultIdentityServer) GetPluginInfo(ctx context.Context, req *csi.Ge }, nil } +// Probe returns empty response func (ids *DefaultIdentityServer) Probe(ctx context.Context, req *csi.ProbeRequest) (*csi.ProbeResponse, error) { return &csi.ProbeResponse{}, nil } +// GetPluginCapabilities returns plugin capabilities func (ids *DefaultIdentityServer) GetPluginCapabilities(ctx context.Context, req *csi.GetPluginCapabilitiesRequest) (*csi.GetPluginCapabilitiesResponse, error) { - glog.V(5).Infof("Using default capabilities") + klog.V(5).Infof("Using default capabilities") return &csi.GetPluginCapabilitiesResponse{ Capabilities: []*csi.PluginCapability{ { diff --git a/vendor/github.com/kubernetes-csi/drivers/pkg/csi-common/nodeserver-default.go b/pkg/csi-common/nodeserver-default.go similarity index 72% rename from vendor/github.com/kubernetes-csi/drivers/pkg/csi-common/nodeserver-default.go rename to pkg/csi-common/nodeserver-default.go index cd2355bf2..f24b63d6b 100644 --- a/vendor/github.com/kubernetes-csi/drivers/pkg/csi-common/nodeserver-default.go +++ b/pkg/csi-common/nodeserver-default.go @@ -18,34 +18,39 @@ package csicommon import ( "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/golang/glog" "golang.org/x/net/context" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "k8s.io/klog" ) +// DefaultNodeServer stores driver object type DefaultNodeServer struct { Driver *CSIDriver } -func (ns *DefaultNodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) { +// NodeStageVolume returns unimplemented response +func (ns *DefaultNodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) { return nil, status.Error(codes.Unimplemented, "") } -func (ns *DefaultNodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) { +// NodeUnstageVolume returns unimplemented response +func (ns *DefaultNodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error) { return nil, status.Error(codes.Unimplemented, "") } +// NodeGetInfo returns node ID func (ns *DefaultNodeServer) NodeGetInfo(ctx context.Context, req *csi.NodeGetInfoRequest) (*csi.NodeGetInfoResponse, error) { - glog.V(5).Infof("Using default NodeGetInfo") + klog.V(5).Infof("Using default NodeGetInfo") return &csi.NodeGetInfoResponse{ NodeId: ns.Driver.nodeID, }, nil } +// NodeGetCapabilities returns RPC unknow capability func (ns *DefaultNodeServer) NodeGetCapabilities(ctx context.Context, req *csi.NodeGetCapabilitiesRequest) (*csi.NodeGetCapabilitiesResponse, error) { - glog.V(5).Infof("Using default NodeGetCapabilities") + klog.V(5).Infof("Using default NodeGetCapabilities") return &csi.NodeGetCapabilitiesResponse{ Capabilities: []*csi.NodeServiceCapability{ @@ -60,6 +65,7 @@ func (ns *DefaultNodeServer) NodeGetCapabilities(ctx context.Context, req *csi.N }, nil } +// NodeGetVolumeStats returns volume stats func (ns *DefaultNodeServer) NodeGetVolumeStats(ctx context.Context, in *csi.NodeGetVolumeStatsRequest) (*csi.NodeGetVolumeStatsResponse, error) { return nil, status.Error(codes.Unimplemented, "") } diff --git a/vendor/github.com/kubernetes-csi/drivers/pkg/csi-common/server.go b/pkg/csi-common/server.go similarity index 75% rename from vendor/github.com/kubernetes-csi/drivers/pkg/csi-common/server.go rename to pkg/csi-common/server.go index 9d3c99523..caaaa8bb8 100644 --- a/vendor/github.com/kubernetes-csi/drivers/pkg/csi-common/server.go +++ b/pkg/csi-common/server.go @@ -21,13 +21,12 @@ import ( "os" "sync" - "github.com/golang/glog" - "google.golang.org/grpc" - "github.com/container-storage-interface/spec/lib/go/csi" + "google.golang.org/grpc" + "k8s.io/klog" ) -// Defines Non blocking GRPC server interfaces +// NonBlockingGRPCServer defines Non blocking GRPC server interfaces type NonBlockingGRPCServer interface { // Start services at the endpoint Start(endpoint string, ids csi.IdentityServer, cs csi.ControllerServer, ns csi.NodeServer) @@ -39,6 +38,7 @@ type NonBlockingGRPCServer interface { ForceStop() } +// NewNonBlockingGRPCServer return non-blocking GRPC func NewNonBlockingGRPCServer() NonBlockingGRPCServer { return &nonBlockingGRPCServer{} } @@ -49,44 +49,45 @@ type nonBlockingGRPCServer struct { server *grpc.Server } +// Start start service on endpoint func (s *nonBlockingGRPCServer) Start(endpoint string, ids csi.IdentityServer, cs csi.ControllerServer, ns csi.NodeServer) { s.wg.Add(1) - go s.serve(endpoint, ids, cs, ns) - - return } +// Wait blocks until the WaitGroup counter func (s *nonBlockingGRPCServer) Wait() { s.wg.Wait() } +// GracefulStop stops the gRPC server gracefully. func (s *nonBlockingGRPCServer) Stop() { s.server.GracefulStop() } +// Stop stops the gRPC server. func (s *nonBlockingGRPCServer) ForceStop() { s.server.Stop() } func (s *nonBlockingGRPCServer) serve(endpoint string, ids csi.IdentityServer, cs csi.ControllerServer, ns csi.NodeServer) { - proto, addr, err := ParseEndpoint(endpoint) + proto, addr, err := parseEndpoint(endpoint) if err != nil { - glog.Fatal(err.Error()) + klog.Fatal(err.Error()) } if proto == "unix" { addr = "/" + addr - if err := os.Remove(addr); err != nil && !os.IsNotExist(err) { - glog.Fatalf("Failed to remove %s, error: %s", addr, err.Error()) + if e := os.Remove(addr); e != nil && !os.IsNotExist(e) { + klog.Fatalf("Failed to remove %s, error: %s", addr, e.Error()) } } listener, err := net.Listen(proto, addr) if err != nil { - glog.Fatalf("Failed to listen: %v", err) + klog.Fatalf("Failed to listen: %v", err) } opts := []grpc.ServerOption{ @@ -105,8 +106,10 @@ func (s *nonBlockingGRPCServer) serve(endpoint string, ids csi.IdentityServer, c csi.RegisterNodeServer(server, ns) } - glog.Infof("Listening for connections on address: %#v", listener.Addr()) - - server.Serve(listener) + klog.Infof("Listening for connections on address: %#v", listener.Addr()) + err = server.Serve(listener) + if err != nil { + klog.Fatalf("Failed to server: %v", err) + } } diff --git a/vendor/github.com/kubernetes-csi/drivers/pkg/csi-common/utils.go b/pkg/csi-common/utils.go similarity index 76% rename from vendor/github.com/kubernetes-csi/drivers/pkg/csi-common/utils.go rename to pkg/csi-common/utils.go index b39132e0d..c6ebb5ac8 100644 --- a/vendor/github.com/kubernetes-csi/drivers/pkg/csi-common/utils.go +++ b/pkg/csi-common/utils.go @@ -21,44 +21,49 @@ import ( "strings" "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/golang/glog" "github.com/kubernetes-csi/csi-lib-utils/protosanitizer" "golang.org/x/net/context" "google.golang.org/grpc" + "k8s.io/klog" ) -func ParseEndpoint(ep string) (string, string, error) { +func parseEndpoint(ep string) (string, string, error) { if strings.HasPrefix(strings.ToLower(ep), "unix://") || strings.HasPrefix(strings.ToLower(ep), "tcp://") { s := strings.SplitN(ep, "://", 2) if s[1] != "" { return s[0], s[1], nil } } - return "", "", fmt.Errorf("Invalid endpoint: %v", ep) + return "", "", fmt.Errorf("invalid endpoint: %v", ep) } +// NewVolumeCapabilityAccessMode returns volume access mode func NewVolumeCapabilityAccessMode(mode csi.VolumeCapability_AccessMode_Mode) *csi.VolumeCapability_AccessMode { return &csi.VolumeCapability_AccessMode{Mode: mode} } +// NewDefaultNodeServer initializes default node server func NewDefaultNodeServer(d *CSIDriver) *DefaultNodeServer { return &DefaultNodeServer{ Driver: d, } } +// NewDefaultIdentityServer initializes default identity servier func NewDefaultIdentityServer(d *CSIDriver) *DefaultIdentityServer { return &DefaultIdentityServer{ Driver: d, } } +// NewDefaultControllerServer initializes default controller server func NewDefaultControllerServer(d *CSIDriver) *DefaultControllerServer { return &DefaultControllerServer{ Driver: d, } } +// NewControllerServiceCapability returns controller capabilities func NewControllerServiceCapability(cap csi.ControllerServiceCapability_RPC_Type) *csi.ControllerServiceCapability { return &csi.ControllerServiceCapability{ Type: &csi.ControllerServiceCapability_Rpc{ @@ -69,6 +74,7 @@ func NewControllerServiceCapability(cap csi.ControllerServiceCapability_RPC_Type } } +// RunNodePublishServer starts node server func RunNodePublishServer(endpoint string, d *CSIDriver, ns csi.NodeServer) { ids := NewDefaultIdentityServer(d) @@ -77,6 +83,7 @@ func RunNodePublishServer(endpoint string, d *CSIDriver, ns csi.NodeServer) { s.Wait() } +// RunControllerPublishServer starts controller server func RunControllerPublishServer(endpoint string, d *CSIDriver, cs csi.ControllerServer) { ids := NewDefaultIdentityServer(d) @@ -85,6 +92,7 @@ func RunControllerPublishServer(endpoint string, d *CSIDriver, cs csi.Controller s.Wait() } +// RunControllerandNodePublishServer starts both controller and node server func RunControllerandNodePublishServer(endpoint string, d *CSIDriver, cs csi.ControllerServer, ns csi.NodeServer) { ids := NewDefaultIdentityServer(d) @@ -94,13 +102,13 @@ func RunControllerandNodePublishServer(endpoint string, d *CSIDriver, cs csi.Con } func logGRPC(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { - glog.V(3).Infof("GRPC call: %s", info.FullMethod) - glog.V(5).Infof("GRPC request: %s", protosanitizer.StripSecrets(req)) + klog.V(3).Infof("GRPC call: %s", info.FullMethod) + klog.V(5).Infof("GRPC request: %s", protosanitizer.StripSecrets(req)) resp, err := handler(ctx, req) if err != nil { - glog.Errorf("GRPC error: %v", err) + klog.Errorf("GRPC error: %v", err) } else { - glog.V(5).Infof("GRPC response: %s", protosanitizer.StripSecrets(resp)) + klog.V(5).Infof("GRPC response: %s", protosanitizer.StripSecrets(resp)) } return resp, err } diff --git a/pkg/rbd/controllerserver.go b/pkg/rbd/controllerserver.go index 616012095..27c407fdf 100644 --- a/pkg/rbd/controllerserver.go +++ b/pkg/rbd/controllerserver.go @@ -18,16 +18,18 @@ package rbd import ( "fmt" - "os" "os/exec" + "sort" + "strconv" "syscall" + csicommon "github.com/ceph/ceph-csi/pkg/csi-common" "github.com/ceph/ceph-csi/pkg/util" + "github.com/container-storage-interface/spec/lib/go/csi" "github.com/golang/protobuf/ptypes" "github.com/golang/protobuf/ptypes/timestamp" "github.com/kubernetes-csi/csi-lib-utils/protosanitizer" - "github.com/kubernetes-csi/drivers/pkg/csi-common" "github.com/pborman/uuid" "github.com/pkg/errors" "golang.org/x/net/context" @@ -91,17 +93,32 @@ func (cs *ControllerServer) validateVolumeReq(req *csi.CreateVolumeRequest) erro func parseVolCreateRequest(req *csi.CreateVolumeRequest) (*rbdVolume, error) { // TODO (sbezverk) Last check for not exceeding total storage capacity - rbdVol, err := getRBDVolumeOptions(req.GetParameters()) + isMultiNode := false + isBlock := false + for _, cap := range req.VolumeCapabilities { + // RO modes need to be handled indepedently (ie right now even if access mode is RO, they'll be RW upon attach) + if cap.GetAccessMode().GetMode() == csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER { + isMultiNode = true + } + if cap.GetBlock() != nil { + isBlock = true + } + } + + // We want to fail early if the user is trying to create a RWX on a non-block type device + if isMultiNode && !isBlock { + return nil, status.Error(codes.InvalidArgument, "multi node access modes are only supported on rbd `block` type volumes") + } + + // if it's NOT SINGLE_NODE_WRITER and it's BLOCK we'll set the parameter to ignore the in-use checks + rbdVol, err := getRBDVolumeOptions(req.GetParameters(), (isMultiNode && isBlock)) if err != nil { - return nil, err + return nil, status.Error(codes.InvalidArgument, err.Error()) } // Generating Volume Name and Volume ID, as according to CSI spec they MUST be different volName := req.GetName() uniqueID := uuid.NewUUID().String() - if len(volName) == 0 { - volName = rbdVol.Pool + "-dynamic-pvc-" + uniqueID - } rbdVol.VolName = volName volumeID := "csi-rbd-vol-" + uniqueID rbdVol.VolID = volumeID @@ -110,11 +127,21 @@ func parseVolCreateRequest(req *csi.CreateVolumeRequest) (*rbdVolume, error) { if req.GetCapacityRange() != nil { volSizeBytes = req.GetCapacityRange().GetRequiredBytes() } - rbdVol.VolSize = volSizeBytes + + rbdVol.VolSize = util.RoundUpToMiB(volSizeBytes) return rbdVol, nil } +func storeVolumeMetadata(vol *rbdVolume, cp util.CachePersister) error { + if err := cp.Create(vol.VolID, vol); err != nil { + klog.Errorf("failed to store metadata for volume %s: %v", vol.VolID, err) + return err + } + + return nil +} + // CreateVolume creates the volume in backend and store the volume metadata func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) { @@ -136,6 +163,11 @@ func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol // request if exVol.VolSize >= req.GetCapacityRange().GetRequiredBytes() { // existing volume is compatible with new request and should be reused. + + if err = storeVolumeMetadata(exVol, cs.MetadataStore); err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + // TODO (sbezverk) Do I need to make sure that RBD volume still exists? return &csi.CreateVolumeResponse{ Volume: &csi.Volume{ @@ -153,23 +185,21 @@ func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol return nil, err } - volSizeGB := int(rbdVol.VolSize / 1024 / 1024 / 1024) - // Check if there is already RBD image with requested name - err = cs.checkRBDStatus(rbdVol, req, volSizeGB) + err = cs.checkRBDStatus(rbdVol, req, int(rbdVol.VolSize)) if err != nil { return nil, err } - if createErr := cs.MetadataStore.Create(rbdVol.VolID, rbdVol); createErr != nil { - klog.Warningf("failed to store volume metadata with error: %v", err) - if err = deleteRBDImage(rbdVol, rbdVol.AdminID, req.GetSecrets()); err != nil { - klog.V(3).Infof("failed to delete rbd image: %s/%s with error: %v", rbdVol.Pool, rbdVol.VolName, err) - return nil, err - } - return nil, createErr - } + // store volume size in bytes (snapshot and check existing volume needs volume + // size in bytes) + rbdVol.VolSize = rbdVol.VolSize * util.MiB rbdVolumes[rbdVol.VolID] = rbdVol + + if err = storeVolumeMetadata(rbdVol, cs.MetadataStore); err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + return &csi.CreateVolumeResponse{ Volume: &csi.Volume{ VolumeId: rbdVol.VolID, @@ -179,10 +209,11 @@ func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol }, nil } -func (cs *ControllerServer) checkRBDStatus(rbdVol *rbdVolume, req *csi.CreateVolumeRequest, volSizeGB int) error { +func (cs *ControllerServer) checkRBDStatus(rbdVol *rbdVolume, req *csi.CreateVolumeRequest, volSizeMiB int) error { var err error // Check if there is already RBD image with requested name - found, _, _ := rbdStatus(rbdVol, rbdVol.UserID, req.GetSecrets()) // #nosec + //nolint + found, _, _ := rbdStatus(rbdVol, rbdVol.UserID, req.GetSecrets()) if !found { // if VolumeContentSource is not nil, this request is for snapshot if req.VolumeContentSource != nil { @@ -190,10 +221,10 @@ func (cs *ControllerServer) checkRBDStatus(rbdVol *rbdVolume, req *csi.CreateVol return err } } else { - err = createRBDImage(rbdVol, volSizeGB, rbdVol.AdminID, req.GetSecrets()) + err = createRBDImage(rbdVol, volSizeMiB, rbdVol.AdminID, req.GetSecrets()) if err != nil { klog.Warningf("failed to create volume: %v", err) - return err + return status.Error(codes.Internal, err.Error()) } klog.V(4).Infof("create volume %s", rbdVol.VolName) @@ -214,12 +245,12 @@ func (cs *ControllerServer) checkSnapshot(req *csi.CreateVolumeRequest, rbdVol * rbdSnap := &rbdSnapshot{} if err := cs.MetadataStore.Get(snapshotID, rbdSnap); err != nil { - return err + return status.Error(codes.NotFound, err.Error()) } err := restoreSnapshot(rbdVol, rbdSnap, rbdVol.AdminID, req.GetSecrets()) if err != nil { - return err + return status.Error(codes.Internal, err.Error()) } klog.V(4).Infof("create volume %s from snapshot %s", req.GetName(), rbdSnap.SnapName) return nil @@ -244,9 +275,11 @@ func (cs *ControllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVol rbdVol := &rbdVolume{} if err := cs.MetadataStore.Get(volumeID, rbdVol); err != nil { - if os.IsNotExist(errors.Cause(err)) { + if err, ok := err.(*util.CacheEntryNotFound); ok { + klog.V(3).Infof("metadata for volume %s not found, assuming the volume to be already deleted (%v)", volumeID, err) return &csi.DeleteVolumeResponse{}, nil } + return nil, err } @@ -256,17 +289,66 @@ func (cs *ControllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVol if err := deleteRBDImage(rbdVol, rbdVol.AdminID, req.GetSecrets()); err != nil { // TODO: can we detect "already deleted" situations here and proceed? klog.V(3).Infof("failed to delete rbd image: %s/%s with error: %v", rbdVol.Pool, volName, err) - return nil, err + return nil, status.Error(codes.Internal, err.Error()) } if err := cs.MetadataStore.Delete(volumeID); err != nil { - return nil, err + return nil, status.Error(codes.Internal, err.Error()) } delete(rbdVolumes, volumeID) return &csi.DeleteVolumeResponse{}, nil } +// ListVolumes returns a list of volumes stored in memory +func (cs *ControllerServer) ListVolumes(ctx context.Context, req *csi.ListVolumesRequest) (*csi.ListVolumesResponse, error) { + var startToken int + if err := cs.Driver.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_LIST_VOLUMES); err != nil { + klog.Warningf("invalid list volume req: %v", req) + return nil, err + } + + //validate starting token if present + if len(req.GetStartingToken()) > 0 { + i, parseErr := strconv.ParseUint(req.StartingToken, 10, 32) + if parseErr != nil { + return nil, status.Errorf(codes.Aborted, "invalid starting token %s", parseErr.Error()) + } + //check starting Token is greater than list of rbd volumes + if len(rbdVolumes) < int(i) { + return nil, status.Errorf(codes.Aborted, "invalid starting token %s", parseErr.Error()) + } + startToken = int(i) + } + + var entries []*csi.ListVolumesResponse_Entry + + keys := make([]string, 0) + for k := range rbdVolumes { + keys = append(keys, k) + } + sort.Strings(keys) + + for index, k := range keys { + if index < startToken { + continue + } + entries = append(entries, &csi.ListVolumesResponse_Entry{ + Volume: &csi.Volume{ + VolumeId: rbdVolumes[k].VolID, + CapacityBytes: rbdVolumes[k].VolSize, + VolumeContext: extractStoredVolOpt(rbdVolumes[k]), + }, + }) + } + + resp := &csi.ListVolumesResponse{ + Entries: entries, + } + + return resp, nil +} + // ValidateVolumeCapabilities checks whether the volume capabilities requested // are supported. func (cs *ControllerServer) ValidateVolumeCapabilities(ctx context.Context, req *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error) { @@ -294,6 +376,7 @@ func (cs *ControllerServer) ControllerPublishVolume(ctx context.Context, req *cs // CreateSnapshot creates the snapshot in backend and stores metadata // in store +// nolint: gocyclo func (cs *ControllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateSnapshotRequest) (*csi.CreateSnapshotResponse, error) { if err := cs.validateSnapshotReq(req); err != nil { @@ -311,6 +394,10 @@ func (cs *ControllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateS // check for the requested source volume id and already allocated source volume id if exSnap, err := getRBDSnapshotByName(req.GetName()); err == nil { if req.SourceVolumeId == exSnap.SourceVolumeID { + if err = storeSnapshotMetadata(exSnap, cs.MetadataStore); err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + return &csi.CreateSnapshotResponse{ Snapshot: &csi.Snapshot{ SizeBytes: exSnap.SizeBytes, @@ -328,7 +415,7 @@ func (cs *ControllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateS rbdSnap, err := getRBDSnapshotOptions(req.GetParameters()) if err != nil { - return nil, err + return nil, status.Error(codes.InvalidArgument, err.Error()) } // Generating Snapshot Name and Snapshot ID, as according to CSI spec they MUST be different @@ -339,7 +426,7 @@ func (cs *ControllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateS return nil, status.Errorf(codes.NotFound, "Source Volume ID %s cannot found", req.GetSourceVolumeId()) } if !hasSnapshotFeature(rbdVolume.ImageFeatures) { - return nil, fmt.Errorf("volume(%s) has not snapshot feature(layering)", req.GetSourceVolumeId()) + return nil, status.Errorf(codes.InvalidArgument, "volume(%s) has not snapshot feature(layering)", req.GetSourceVolumeId()) } rbdSnap.VolName = rbdVolume.VolName @@ -352,16 +439,17 @@ func (cs *ControllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateS err = cs.doSnapshot(rbdSnap, req.GetSecrets()) // if we already have the snapshot, return the snapshot if err != nil { - return nil, err + return nil, status.Error(codes.Internal, err.Error()) } rbdSnap.CreatedAt = ptypes.TimestampNow().GetSeconds() - if err = cs.storeSnapMetadata(rbdSnap, req.GetSecrets()); err != nil { - return nil, err + rbdSnapshots[snapshotID] = rbdSnap + + if err = storeSnapshotMetadata(rbdSnap, cs.MetadataStore); err != nil { + return nil, status.Error(codes.Internal, err.Error()) } - rbdSnapshots[snapshotID] = rbdSnap return &csi.CreateSnapshotResponse{ Snapshot: &csi.Snapshot{ SizeBytes: rbdSnap.SizeBytes, @@ -375,22 +463,13 @@ func (cs *ControllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateS }, nil } -func (cs *ControllerServer) storeSnapMetadata(rbdSnap *rbdSnapshot, secret map[string]string) error { - errCreate := cs.MetadataStore.Create(rbdSnap.SnapID, rbdSnap) - if errCreate != nil { - klog.Warningf("rbd: failed to store snapInfo with error: %v", errCreate) - // Unprotect snapshot - err := unprotectSnapshot(rbdSnap, rbdSnap.AdminID, secret) - if err != nil { - return status.Errorf(codes.Unknown, "This Snapshot should be removed but failed to unprotect snapshot: %s/%s with error: %v", rbdSnap.Pool, rbdSnap.SnapName, err) - } - // Deleting snapshot - klog.V(4).Infof("deleting Snaphot %s", rbdSnap.SnapName) - if err = deleteSnapshot(rbdSnap, rbdSnap.AdminID, secret); err != nil { - return status.Errorf(codes.Unknown, "This Snapshot should be removed but failed to delete snapshot: %s/%s with error: %v", rbdSnap.Pool, rbdSnap.SnapName, err) - } +func storeSnapshotMetadata(rbdSnap *rbdSnapshot, cp util.CachePersister) error { + if err := cp.Create(rbdSnap.SnapID, rbdSnap); err != nil { + klog.Errorf("failed to store metadata for snapshot %s: %v", rbdSnap.SnapID, err) + return err } - return errCreate + + return nil } func (cs *ControllerServer) validateSnapshotReq(req *csi.CreateSnapshotRequest) error { @@ -438,7 +517,7 @@ func (cs *ControllerServer) doSnapshot(rbdSnap *rbdSnapshot, secret map[string]s if err != nil { return fmt.Errorf("snapshot is created but failed to protect and delete snapshot: %v", err) } - return fmt.Errorf("snapshot is created but failed to protect snapshot") + return errors.New("snapshot is created but failed to protect snapshot") } } return nil @@ -466,6 +545,11 @@ func (cs *ControllerServer) DeleteSnapshot(ctx context.Context, req *csi.DeleteS rbdSnap := &rbdSnapshot{} if err := cs.MetadataStore.Get(snapshotID, rbdSnap); err != nil { + if err, ok := err.(*util.CacheEntryNotFound); ok { + klog.V(3).Infof("metadata for snapshot %s not found, assuming the snapshot to be already deleted (%v)", snapshotID, err) + return &csi.DeleteSnapshotResponse{}, nil + } + return nil, err } @@ -482,7 +566,7 @@ func (cs *ControllerServer) DeleteSnapshot(ctx context.Context, req *csi.DeleteS } if err := cs.MetadataStore.Delete(snapshotID); err != nil { - return nil, err + return nil, status.Error(codes.Internal, err.Error()) } delete(rbdSnapshots, snapshotID) diff --git a/pkg/rbd/identityserver.go b/pkg/rbd/identityserver.go index 856bc50a8..891759f7c 100644 --- a/pkg/rbd/identityserver.go +++ b/pkg/rbd/identityserver.go @@ -19,8 +19,9 @@ package rbd import ( "context" + csicommon "github.com/ceph/ceph-csi/pkg/csi-common" + "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/kubernetes-csi/drivers/pkg/csi-common" ) // IdentityServer struct of rbd CSI driver with supported methods of CSI diff --git a/pkg/rbd/nodeserver.go b/pkg/rbd/nodeserver.go index d1eb374c1..f31ec4cc5 100644 --- a/pkg/rbd/nodeserver.go +++ b/pkg/rbd/nodeserver.go @@ -23,16 +23,14 @@ import ( "regexp" "strings" - "golang.org/x/net/context" - "k8s.io/klog" + "github.com/ceph/ceph-csi/pkg/csi-common" "github.com/container-storage-interface/spec/lib/go/csi" + "golang.org/x/net/context" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - + "k8s.io/klog" "k8s.io/kubernetes/pkg/util/mount" - - "github.com/kubernetes-csi/drivers/pkg/csi-common" ) // NodeServer struct of ceph rbd driver with supported methods of CSI @@ -45,21 +43,12 @@ type NodeServer struct { //TODO remove both stage and unstage methods //once https://github.com/kubernetes-csi/drivers/pull/145 is merged -// NodeStageVolume returns unimplemented response -func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) { - return nil, status.Error(codes.Unimplemented, "") -} - -// NodeUnstageVolume returns unimplemented response -func (ns *NodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error) { - return nil, status.Error(codes.Unimplemented, "") -} - // NodePublishVolume mounts the volume mounted to the device path to the target // path func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) { targetPath := req.GetTargetPath() targetPathMutex.LockKey(targetPath) + disableInUseChecks := false defer func() { if err := targetPathMutex.UnlockKey(targetPath); err != nil { @@ -82,7 +71,18 @@ func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis if !notMnt { return &csi.NodePublishVolumeResponse{}, nil } - volOptions, err := getRBDVolumeOptions(req.GetVolumeContext()) + + // MULTI_NODE_MULTI_WRITER is supported by default for Block access type volumes + if req.VolumeCapability.AccessMode.Mode == csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER { + if isBlock { + disableInUseChecks = true + } else { + klog.Warningf("MULTI_NODE_MULTI_WRITER currently only supported with volumes of access type `block`, invalid AccessMode for volume: %v", req.GetVolumeId()) + return nil, status.Error(codes.InvalidArgument, "rbd: RWX access mode request is only valid for volumes with access type `block`") + } + } + + volOptions, err := getRBDVolumeOptions(req.GetVolumeContext(), disableInUseChecks) if err != nil { return nil, err } diff --git a/pkg/rbd/rbd.go b/pkg/rbd/rbd.go index 8740e8c26..bcd5c6c90 100644 --- a/pkg/rbd/rbd.go +++ b/pkg/rbd/rbd.go @@ -17,12 +17,11 @@ limitations under the License. package rbd import ( - "k8s.io/klog" - + csicommon "github.com/ceph/ceph-csi/pkg/csi-common" "github.com/ceph/ceph-csi/pkg/util" - "github.com/container-storage-interface/spec/lib/go/csi" - "github.com/kubernetes-csi/drivers/pkg/csi-common" + "github.com/container-storage-interface/spec/lib/go/csi" + "k8s.io/klog" "k8s.io/kubernetes/pkg/util/mount" "k8s.io/kubernetes/pkg/util/nsenter" "k8s.io/utils/exec" @@ -30,11 +29,13 @@ import ( // PluginFolder defines the location of rbdplugin const ( - PluginFolder = "/var/lib/kubelet/plugins/csi-rbdplugin" rbdDefaultAdminID = "admin" rbdDefaultUserID = rbdDefaultAdminID ) +// PluginFolder defines the location of ceph plugin +var PluginFolder = "/var/lib/kubelet/plugins/" + // Driver contains the default identity,node and controller struct type Driver struct { cd *csicommon.CSIDriver @@ -46,6 +47,8 @@ type Driver struct { var ( version = "1.0.0" + // confStore is the global config store + confStore *util.ConfigStore ) // NewDriver returns new rbd driver @@ -86,10 +89,16 @@ func NewNodeServer(d *csicommon.CSIDriver, containerized bool) (*NodeServer, err // Run start a non-blocking grpc controller,node and identityserver for // rbd CSI driver which can serve multiple parallel requests -func (r *Driver) Run(driverName, nodeID, endpoint string, containerized bool, cachePersister util.CachePersister) { +func (r *Driver) Run(driverName, nodeID, endpoint, configRoot string, containerized bool, cachePersister util.CachePersister) { var err error klog.Infof("Driver: %v version: %v", driverName, version) + // Initialize config store + confStore, err = util.NewConfigStore(configRoot) + if err != nil { + klog.Fatalln("Failed to initialize config store.") + } + // Initialize default library driver r.cd = csicommon.NewCSIDriver(driverName, version, nodeID) if r.cd == nil { @@ -98,11 +107,19 @@ func (r *Driver) Run(driverName, nodeID, endpoint string, containerized bool, ca r.cd.AddControllerServiceCapabilities([]csi.ControllerServiceCapability_RPC_Type{ csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME, csi.ControllerServiceCapability_RPC_PUBLISH_UNPUBLISH_VOLUME, + csi.ControllerServiceCapability_RPC_LIST_VOLUMES, csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT, csi.ControllerServiceCapability_RPC_LIST_SNAPSHOTS, csi.ControllerServiceCapability_RPC_CLONE_VOLUME, }) - r.cd.AddVolumeCapabilityAccessModes([]csi.VolumeCapability_AccessMode_Mode{csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER}) + + // We only support the multi-writer option when using block, but it's a supported capability for the plugin in general + // In addition, we want to add the remaining modes like MULTI_NODE_READER_ONLY, + // MULTI_NODE_SINGLE_WRITER etc, but need to do some verification of RO modes first + // will work those as follow up features + r.cd.AddVolumeCapabilityAccessModes( + []csi.VolumeCapability_AccessMode_Mode{csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, + csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER}) // Create GRPC servers r.ids = NewIdentityServer(r.cd) diff --git a/pkg/rbd/rbd_attach.go b/pkg/rbd/rbd_attach.go index 354554d12..d56ac74f3 100644 --- a/pkg/rbd/rbd_attach.go +++ b/pkg/rbd/rbd_attach.go @@ -258,6 +258,7 @@ func attachRBDImage(volOptions *rbdVolume, userID string, credentials map[string Factor: rbdImageWatcherFactor, Steps: rbdImageWatcherSteps, } + err = waitForrbdImage(backoff, volOptions, userID, credentials) if err != nil { @@ -279,7 +280,7 @@ func createPath(volOpt *rbdVolume, userID string, creds map[string]string) (stri } klog.V(5).Infof("rbd: map mon %s", mon) - key, err := getRBDKey(userID, creds) + key, err := getRBDKey(volOpt.ClusterID, userID, creds) if err != nil { return "", err } @@ -313,6 +314,10 @@ func waitForrbdImage(backoff wait.Backoff, volOptions *rbdVolume, userID string, if err != nil { return false, fmt.Errorf("fail to check rbd image status with: (%v), rbd output: (%s)", err, rbdOutput) } + if (volOptions.DisableInUseChecks) && (used) { + klog.V(2).Info("valid multi-node attach requested, ignoring watcher in-use result") + return used, nil + } return !used, nil }) // return error if rbd image has not become available for the specified timeout diff --git a/pkg/rbd/rbd_util.go b/pkg/rbd/rbd_util.go index 7df546553..14fcc0ebf 100644 --- a/pkg/rbd/rbd_util.go +++ b/pkg/rbd/rbd_util.go @@ -51,6 +51,8 @@ type rbdVolume struct { AdminID string `json:"adminId"` UserID string `json:"userId"` Mounter string `json:"mounter"` + DisableInUseChecks bool `json:"disableInUseChecks"` + ClusterID string `json:"clusterId"` } type rbdSnapshot struct { @@ -65,6 +67,7 @@ type rbdSnapshot struct { SizeBytes int64 `json:"sizeBytes"` AdminID string `json:"adminId"` UserID string `json:"userId"` + ClusterID string `json:"clusterId"` } var ( @@ -84,12 +87,25 @@ var ( supportedFeatures = sets.NewString("layering") ) -func getRBDKey(id string, credentials map[string]string) (string, error) { +func getRBDKey(clusterid, id string, credentials map[string]string) (string, error) { + var ( + ok bool + err error + key string + ) - if key, ok := credentials[id]; ok { - return key, nil + if key, ok = credentials[id]; !ok { + if clusterid != "" { + key, err = confStore.KeyForUser(clusterid, id) + if err != nil { + return "", fmt.Errorf("RBD key for ID: %s not found in config store of clusterID (%s)", id, clusterid) + } + } else { + return "", fmt.Errorf("RBD key for ID: %s not found", id) + } } - return "", fmt.Errorf("RBD key for ID: %s not found", id) + + return key, nil } func getMon(pOpts *rbdVolume, credentials map[string]string) (string, error) { @@ -120,18 +136,18 @@ func createRBDImage(pOpts *rbdVolume, volSz int, adminID string, credentials map } image := pOpts.VolName - volSzGB := fmt.Sprintf("%dG", volSz) + volSzMiB := fmt.Sprintf("%dM", volSz) - key, err := getRBDKey(adminID, credentials) + key, err := getRBDKey(pOpts.ClusterID, adminID, credentials) if err != nil { return err } if pOpts.ImageFormat == rbdImageFormat2 { - klog.V(4).Infof("rbd: create %s size %s format %s (features: %s) using mon %s, pool %s ", image, volSzGB, pOpts.ImageFormat, pOpts.ImageFeatures, mon, pOpts.Pool) + klog.V(4).Infof("rbd: create %s size %s format %s (features: %s) using mon %s, pool %s ", image, volSzMiB, pOpts.ImageFormat, pOpts.ImageFeatures, mon, pOpts.Pool) } else { - klog.V(4).Infof("rbd: create %s size %s format %s using mon %s, pool %s", image, volSzGB, pOpts.ImageFormat, mon, pOpts.Pool) + klog.V(4).Infof("rbd: create %s size %s format %s using mon %s, pool %s", image, volSzMiB, pOpts.ImageFormat, mon, pOpts.Pool) } - args := []string{"create", image, "--size", volSzGB, "--pool", pOpts.Pool, "--id", adminID, "-m", mon, "--key=" + key, "--image-format", pOpts.ImageFormat} + args := []string{"create", image, "--size", volSzMiB, "--pool", pOpts.Pool, "--id", adminID, "-m", mon, "--key=" + key, "--image-format", pOpts.ImageFormat} if pOpts.ImageFormat == rbdImageFormat2 { args = append(args, "--image-feature", pOpts.ImageFeatures) } @@ -153,7 +169,7 @@ func rbdStatus(pOpts *rbdVolume, userID string, credentials map[string]string) ( image := pOpts.VolName // If we don't have admin id/secret (e.g. attaching), fallback to user id/secret. - key, err := getRBDKey(userID, credentials) + key, err := getRBDKey(pOpts.ClusterID, userID, credentials) if err != nil { return false, "", err } @@ -201,7 +217,7 @@ func deleteRBDImage(pOpts *rbdVolume, adminID string, credentials map[string]str klog.Info("rbd is still being used ", image) return fmt.Errorf("rbd %s is still being used", image) } - key, err := getRBDKey(adminID, credentials) + key, err := getRBDKey(pOpts.ClusterID, adminID, credentials) if err != nil { return err } @@ -226,24 +242,82 @@ func execCommand(command string, args []string) ([]byte, error) { return cmd.CombinedOutput() } -func getRBDVolumeOptions(volOptions map[string]string) (*rbdVolume, error) { +func getMonsAndClusterID(options map[string]string) (monitors, clusterID, monInSecret string, err error) { var ok bool + + monitors, ok = options["monitors"] + if !ok { + // if mons are not set in options, check if they are set in secret + if monInSecret, ok = options["monValueFromSecret"]; !ok { + // if mons are not in secret, check if we have a cluster-id + if clusterID, ok = options["clusterID"]; !ok { + err = errors.New("either monitors or monValueFromSecret or clusterID must be set") + return + } + + if monitors, err = confStore.Mons(clusterID); err != nil { + klog.Errorf("failed getting mons (%s)", err) + err = fmt.Errorf("failed to fetch monitor list using clusterID (%s)", clusterID) + return + } + } + } + + return +} + +func getIDs(options map[string]string, clusterID string) (adminID, userID string, err error) { + var ok bool + + adminID, ok = options["adminid"] + switch { + case ok: + case clusterID != "": + if adminID, err = confStore.AdminID(clusterID); err != nil { + klog.Errorf("failed getting adminID (%s)", err) + return "", "", fmt.Errorf("failed to fetch adminID for clusterID (%s)", clusterID) + } + default: + adminID = rbdDefaultAdminID + } + + userID, ok = options["userid"] + switch { + case ok: + case clusterID != "": + if userID, err = confStore.UserID(clusterID); err != nil { + klog.Errorf("failed getting userID (%s)", err) + return "", "", fmt.Errorf("failed to fetch userID using clusterID (%s)", clusterID) + } + default: + userID = rbdDefaultUserID + } + + return adminID, userID, err +} + +func getRBDVolumeOptions(volOptions map[string]string, disableInUseChecks bool) (*rbdVolume, error) { + var ( + ok bool + err error + ) + rbdVol := &rbdVolume{} rbdVol.Pool, ok = volOptions["pool"] if !ok { - return nil, fmt.Errorf("missing required parameter pool") + return nil, errors.New("missing required parameter pool") } - rbdVol.Monitors, ok = volOptions["monitors"] - if !ok { - // if mons are not set in options, check if they are set in secret - if rbdVol.MonValueFromSecret, ok = volOptions["monValueFromSecret"]; !ok { - return nil, fmt.Errorf("either monitors or monValueFromSecret must be set") - } + + rbdVol.Monitors, rbdVol.ClusterID, rbdVol.MonValueFromSecret, err = getMonsAndClusterID(volOptions) + if err != nil { + return nil, err } + rbdVol.ImageFormat, ok = volOptions["imageFormat"] if !ok { rbdVol.ImageFormat = rbdImageFormat2 } + if rbdVol.ImageFormat == rbdImageFormat2 { // if no image features is provided, it results in empty string // which disable all RBD image format 2 features as we expected @@ -259,48 +333,58 @@ func getRBDVolumeOptions(volOptions map[string]string) (*rbdVolume, error) { } } - getCredsFromVol(rbdVol, volOptions) + + klog.V(3).Infof("setting disableInUseChecks on rbd volume to: %v", disableInUseChecks) + rbdVol.DisableInUseChecks = disableInUseChecks + + err = getCredsFromVol(rbdVol, volOptions) + if err != nil { + return nil, err + } + return rbdVol, nil } -func getCredsFromVol(rbdVol *rbdVolume, volOptions map[string]string) { - var ok bool - rbdVol.AdminID, ok = volOptions["adminid"] - if !ok { - rbdVol.AdminID = rbdDefaultAdminID - } - rbdVol.UserID, ok = volOptions["userid"] - if !ok { - rbdVol.UserID = rbdDefaultUserID +func getCredsFromVol(rbdVol *rbdVolume, volOptions map[string]string) error { + var ( + ok bool + err error + ) + + rbdVol.AdminID, rbdVol.UserID, err = getIDs(volOptions, rbdVol.ClusterID) + if err != nil { + return err } + rbdVol.Mounter, ok = volOptions["mounter"] if !ok { rbdVol.Mounter = rbdDefaultMounter } + + return err } + func getRBDSnapshotOptions(snapOptions map[string]string) (*rbdSnapshot, error) { - var ok bool + var ( + ok bool + err error + ) + rbdSnap := &rbdSnapshot{} rbdSnap.Pool, ok = snapOptions["pool"] if !ok { - return nil, fmt.Errorf("missing required parameter pool") - } - rbdSnap.Monitors, ok = snapOptions["monitors"] - if !ok { - // if mons are not set in options, check if they are set in secret - if rbdSnap.MonValueFromSecret, ok = snapOptions["monValueFromSecret"]; !ok { - return nil, fmt.Errorf("either monitors or monValueFromSecret must be set") - } - } - rbdSnap.AdminID, ok = snapOptions["adminid"] - if !ok { - rbdSnap.AdminID = rbdDefaultAdminID - } - rbdSnap.UserID, ok = snapOptions["userid"] - if !ok { - rbdSnap.UserID = rbdDefaultUserID + return nil, errors.New("missing required parameter pool") } + rbdSnap.Monitors, rbdSnap.ClusterID, rbdSnap.MonValueFromSecret, err = getMonsAndClusterID(snapOptions) + if err != nil { + return nil, err + } + + rbdSnap.AdminID, rbdSnap.UserID, err = getIDs(snapOptions, rbdSnap.ClusterID) + if err != nil { + return nil, err + } return rbdSnap, nil } @@ -362,7 +446,7 @@ func protectSnapshot(pOpts *rbdSnapshot, adminID string, credentials map[string] image := pOpts.VolName snapID := pOpts.SnapID - key, err := getRBDKey(adminID, credentials) + key, err := getRBDKey(pOpts.ClusterID, adminID, credentials) if err != nil { return err } @@ -383,6 +467,37 @@ func protectSnapshot(pOpts *rbdSnapshot, adminID string, credentials map[string] return nil } +func extractStoredVolOpt(r *rbdVolume) map[string]string { + volOptions := make(map[string]string) + volOptions["pool"] = r.Pool + + if len(r.Monitors) > 0 { + volOptions["monitors"] = r.Monitors + } + + if len(r.MonValueFromSecret) > 0 { + volOptions["monValueFromSecret"] = r.MonValueFromSecret + } + + volOptions["imageFormat"] = r.ImageFormat + + if len(r.ImageFeatures) > 0 { + volOptions["imageFeatures"] = r.ImageFeatures + } + + if len(r.AdminID) > 0 { + volOptions["adminId"] = r.AdminID + } + + if len(r.UserID) > 0 { + volOptions["userId"] = r.UserID + } + if len(r.Mounter) > 0 { + volOptions["mounter"] = r.Mounter + } + return volOptions +} + func createSnapshot(pOpts *rbdSnapshot, adminID string, credentials map[string]string) error { var output []byte @@ -394,7 +509,7 @@ func createSnapshot(pOpts *rbdSnapshot, adminID string, credentials map[string]s image := pOpts.VolName snapID := pOpts.SnapID - key, err := getRBDKey(adminID, credentials) + key, err := getRBDKey(pOpts.ClusterID, adminID, credentials) if err != nil { return err } @@ -421,7 +536,7 @@ func unprotectSnapshot(pOpts *rbdSnapshot, adminID string, credentials map[strin image := pOpts.VolName snapID := pOpts.SnapID - key, err := getRBDKey(adminID, credentials) + key, err := getRBDKey(pOpts.ClusterID, adminID, credentials) if err != nil { return err } @@ -448,7 +563,7 @@ func deleteSnapshot(pOpts *rbdSnapshot, adminID string, credentials map[string]s image := pOpts.VolName snapID := pOpts.SnapID - key, err := getRBDKey(adminID, credentials) + key, err := getRBDKey(pOpts.ClusterID, adminID, credentials) if err != nil { return err } @@ -475,7 +590,7 @@ func restoreSnapshot(pVolOpts *rbdVolume, pSnapOpts *rbdSnapshot, adminID string image := pVolOpts.VolName snapID := pSnapOpts.SnapID - key, err := getRBDKey(adminID, credentials) + key, err := getRBDKey(pVolOpts.ClusterID, adminID, credentials) if err != nil { return err } diff --git a/pkg/util/cachepersister.go b/pkg/util/cachepersister.go index e72c28f16..4faf6366c 100644 --- a/pkg/util/cachepersister.go +++ b/pkg/util/cachepersister.go @@ -23,13 +23,19 @@ import ( ) const ( - //PluginFolder defines location of plugins + // PluginFolder defines location of plugins PluginFolder = "/var/lib/kubelet/plugins" ) -// ForAllFunc stores metadata with identifier +// ForAllFunc is a unary predicate for visiting all cache entries +// matching the `pattern' in CachePersister's ForAll function. type ForAllFunc func(identifier string) error +// CacheEntryNotFound is an error type for "Not Found" cache errors +type CacheEntryNotFound struct { + error +} + // CachePersister interface implemented for store type CachePersister interface { Create(identifier string, data interface{}) error @@ -50,6 +56,7 @@ func NewCachePersister(metadataStore, driverName string) (CachePersister, error) klog.Infof("cache-persister: using node as metadata cache persister") nc := &NodeCache{} nc.BasePath = PluginFolder + "/" + driverName + nc.CacheDir = "controller" return nc, nil } return nil, errors.New("cache-persister: couldn't parse metadatastorage flag") diff --git a/pkg/util/configstore.go b/pkg/util/configstore.go new file mode 100644 index 000000000..a1194efb6 --- /dev/null +++ b/pkg/util/configstore.go @@ -0,0 +1,137 @@ +/* +Copyright 2019 The Ceph-CSI Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "errors" + "fmt" + "k8s.io/klog" + "path" + "strings" +) + +// StoreReader interface enables plugging different stores, that contain the +// keys and data. (e.g k8s secrets or local files) +type StoreReader interface { + DataForKey(clusterID string, key string) (string, error) +} + +/* ConfigKeys contents and format, +- csMonitors: MON list, comma separated +- csAdminID: adminID, used for provisioning +- csUserID: userID, used for publishing +- csAdminKey: key, for adminID in csProvisionerUser +- csUserKey: key, for userID in csPublisherUser +- csPools: Pool list, comma separated +*/ + +// Constants for various ConfigKeys +const ( + csMonitors = "monitors" + csAdminID = "adminid" + csUserID = "userid" + csAdminKey = "adminkey" + csUserKey = "userkey" + csPools = "pools" +) + +// ConfigStore provides various gettors for ConfigKeys +type ConfigStore struct { + StoreReader +} + +// dataForKey returns data from the config store for the provided key +func (dc *ConfigStore) dataForKey(clusterID, key string) (string, error) { + if dc.StoreReader != nil { + return dc.StoreReader.DataForKey(clusterID, key) + } + + return "", errors.New("config store location uninitialized") +} + +// Mons returns a comma separated MON list from the cluster config represented by clusterID +func (dc *ConfigStore) Mons(clusterID string) (string, error) { + return dc.dataForKey(clusterID, csMonitors) +} + +// Pools returns a list of pool names from the cluster config represented by clusterID +func (dc *ConfigStore) Pools(clusterID string) ([]string, error) { + content, err := dc.dataForKey(clusterID, csPools) + if err != nil { + return nil, err + } + + return strings.Split(content, ","), nil +} + +// AdminID returns the admin ID from the cluster config represented by clusterID +func (dc *ConfigStore) AdminID(clusterID string) (string, error) { + return dc.dataForKey(clusterID, csAdminID) +} + +// UserID returns the user ID from the cluster config represented by clusterID +func (dc *ConfigStore) UserID(clusterID string) (string, error) { + return dc.dataForKey(clusterID, csUserID) +} + +// KeyForUser returns the key for the requested user ID from the cluster config +// represented by clusterID +func (dc *ConfigStore) KeyForUser(clusterID, userID string) (data string, err error) { + var fetchKey string + user, err := dc.AdminID(clusterID) + if err != nil { + return + } + + if user == userID { + fetchKey = csAdminKey + } else { + user, err = dc.UserID(clusterID) + if err != nil { + return + } + + if user != userID { + err = fmt.Errorf("requested user (%s) not found in cluster configuration of (%s)", userID, clusterID) + return + } + + fetchKey = csUserKey + } + + return dc.dataForKey(clusterID, fetchKey) +} + +// NewConfigStore returns a config store based on value of configRoot. If +// configRoot is not "k8s_objects" then it is assumed to be a path to a +// directory, under which the configuration files can be found +func NewConfigStore(configRoot string) (*ConfigStore, error) { + if configRoot != "k8s_objects" { + klog.Infof("cache-store: using files in path (%s) as config store", configRoot) + fc := &FileConfig{} + fc.BasePath = path.Clean(configRoot) + dc := &ConfigStore{fc} + return dc, nil + } + + klog.Infof("cache-store: using k8s objects as config store") + kc := &K8sConfig{} + kc.Client = NewK8sClient() + kc.Namespace = GetK8sNamespace() + dc := &ConfigStore{kc} + return dc, nil +} diff --git a/pkg/util/configstore_test.go b/pkg/util/configstore_test.go new file mode 100644 index 000000000..92bd8b4a9 --- /dev/null +++ b/pkg/util/configstore_test.go @@ -0,0 +1,160 @@ +/* +Copyright 2019 ceph-csi authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "io/ioutil" + "os" + "strings" + "testing" +) + +var basePath = "./test_artifacts" +var clusterID = "testclusterid" +var cs *ConfigStore + +func cleanupTestData() { + os.RemoveAll(basePath) +} + +// nolint: gocyclo +func TestConfigStore(t *testing.T) { + var err error + var data string + var content string + var testDir string + + defer cleanupTestData() + + cs, err = NewConfigStore(basePath) + if err != nil { + t.Errorf("Fatal, failed to get a new config store") + } + + err = os.MkdirAll(basePath, 0700) + if err != nil { + t.Errorf("Test setup error %s", err) + } + + // TEST: Should fail as clusterid directory is missing + _, err = cs.Mons(clusterID) + if err == nil { + t.Errorf("Failed: expected error due to missing parent directory") + } + + testDir = basePath + "/" + "ceph-cluster-" + clusterID + err = os.MkdirAll(testDir, 0700) + if err != nil { + t.Errorf("Test setup error %s", err) + } + + // TEST: Should fail as mons file is missing + _, err = cs.Mons(clusterID) + if err == nil { + t.Errorf("Failed: expected error due to missing mons file") + } + + data = "" + err = ioutil.WriteFile(testDir+"/"+csMonitors, []byte(data), 0644) + if err != nil { + t.Errorf("Test setup error %s", err) + } + + // TEST: Should fail as MONs is an empty string + content, err = cs.Mons(clusterID) + if err == nil { + t.Errorf("Failed: want (%s), got (%s)", data, content) + } + + data = "mon1,mon2,mon3" + err = ioutil.WriteFile(testDir+"/"+csMonitors, []byte(data), 0644) + if err != nil { + t.Errorf("Test setup error %s", err) + } + + // TEST: Fetching MONs should succeed + content, err = cs.Mons(clusterID) + if err != nil || content != data { + t.Errorf("Failed: want (%s), got (%s), err (%s)", data, content, err) + } + + data = "pool1,pool2" + err = ioutil.WriteFile(testDir+"/"+csPools, []byte(data), 0644) + if err != nil { + t.Errorf("Test setup error %s", err) + } + + // TEST: Fetching MONs should succeed + listContent, err := cs.Pools(clusterID) + if err != nil || strings.Join(listContent, ",") != data { + t.Errorf("Failed: want (%s), got (%s), err (%s)", data, content, err) + } + + data = "provuser" + err = ioutil.WriteFile(testDir+"/"+csAdminID, []byte(data), 0644) + if err != nil { + t.Errorf("Test setup error %s", err) + } + + // TEST: Fetching provuser should succeed + content, err = cs.AdminID(clusterID) + if err != nil || content != data { + t.Errorf("Failed: want (%s), got (%s), err (%s)", data, content, err) + } + + data = "pubuser" + err = ioutil.WriteFile(testDir+"/"+csUserID, []byte(data), 0644) + if err != nil { + t.Errorf("Test setup error %s", err) + } + + // TEST: Fetching pubuser should succeed + content, err = cs.UserID(clusterID) + if err != nil || content != data { + t.Errorf("Failed: want (%s), got (%s), err (%s)", data, content, err) + } + + data = "provkey" + err = ioutil.WriteFile(testDir+"/"+csAdminKey, []byte(data), 0644) + if err != nil { + t.Errorf("Test setup error %s", err) + } + + // TEST: Fetching provkey should succeed + content, err = cs.KeyForUser(clusterID, "provuser") + if err != nil || content != data { + t.Errorf("Failed: want (%s), got (%s), err (%s)", data, content, err) + } + + data = "pubkey" + err = ioutil.WriteFile(testDir+"/"+csUserKey, []byte(data), 0644) + if err != nil { + t.Errorf("Test setup error %s", err) + } + + // TEST: Fetching pubkey should succeed + content, err = cs.KeyForUser(clusterID, "pubuser") + if err != nil || content != data { + t.Errorf("Failed: want (%s), got (%s), err (%s)", data, content, err) + } + + // TEST: Fetching random user key should fail + _, err = cs.KeyForUser(clusterID, "random") + if err == nil { + t.Errorf("Failed: Expected to fail fetching random user key") + } +} diff --git a/pkg/util/fileconfig.go b/pkg/util/fileconfig.go new file mode 100644 index 000000000..4d156a603 --- /dev/null +++ b/pkg/util/fileconfig.go @@ -0,0 +1,57 @@ +/* +Copyright 2019 ceph-csi authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "fmt" + "io/ioutil" + "path" +) + +/* +FileConfig is a ConfigStore interface implementation that reads configuration +information from files. + +BasePath defines the directory under which FileConfig will attempt to open and +read contents of various Ceph cluster configurations. + +Each Ceph cluster configuration is stored under a directory named, +BasePath/ceph-cluster-, where uniquely identifies and +separates the each Ceph cluster configuration. + +Under each Ceph cluster configuration directory, individual files named as per +the ConfigKeys constants in the ConfigStore interface, store the required +configuration information. +*/ +type FileConfig struct { + BasePath string +} + +// DataForKey reads the appropriate config file, named using key, and returns +// the contents of the file to the caller +func (fc *FileConfig) DataForKey(clusterid, key string) (data string, err error) { + pathToKey := path.Join(fc.BasePath, "ceph-cluster-"+clusterid, key) + // #nosec + content, err := ioutil.ReadFile(pathToKey) + if err != nil || string(content) == "" { + err = fmt.Errorf("error fetching configuration for cluster ID (%s). (%s)", clusterid, err) + return + } + + data = string(content) + return +} diff --git a/pkg/util/k8scmcache.go b/pkg/util/k8scmcache.go index 570857f89..5982602e2 100644 --- a/pkg/util/k8scmcache.go +++ b/pkg/util/k8scmcache.go @@ -25,7 +25,7 @@ import ( "github.com/pkg/errors" "k8s.io/klog" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" apierrs "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" k8s "k8s.io/client-go/kubernetes" @@ -110,7 +110,7 @@ func (k8scm *K8sCMCache) ForAll(pattern string, destObj interface{}, f ForAllFun continue } if err = json.Unmarshal([]byte(data), destObj); err != nil { - return errors.Wrap(err, "k8s-cm-cache: unmarshal error") + return errors.Wrapf(err, "k8s-cm-cache: JSON unmarshaling failed for configmap %s", cm.ObjectMeta.Name) } if err = f(cm.ObjectMeta.Name); err != nil { return err @@ -123,12 +123,12 @@ func (k8scm *K8sCMCache) ForAll(pattern string, destObj interface{}, f ForAllFun func (k8scm *K8sCMCache) Create(identifier string, data interface{}) error { cm, err := k8scm.getMetadataCM(identifier) if cm != nil && err == nil { - klog.V(4).Infof("k8s-cm-cache: configmap already exists, skipping configmap creation") + klog.V(4).Infof("k8s-cm-cache: configmap %s already exists, skipping configmap creation", identifier) return nil } dataJSON, err := json.Marshal(data) if err != nil { - return errors.Wrap(err, "k8s-cm-cache: marshal error") + return errors.Wrapf(err, "k8s-cm-cache: JSON marshaling failed for configmap %s", identifier) } cm = &v1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ @@ -145,13 +145,13 @@ func (k8scm *K8sCMCache) Create(identifier string, data interface{}) error { _, err = k8scm.Client.CoreV1().ConfigMaps(k8scm.Namespace).Create(cm) if err != nil { if apierrs.IsAlreadyExists(err) { - klog.V(4).Infof("k8s-cm-cache: configmap already exists") + klog.V(4).Infof("k8s-cm-cache: configmap %s already exists", identifier) return nil } return errors.Wrapf(err, "k8s-cm-cache: couldn't persist %s metadata as configmap", identifier) } - klog.V(4).Infof("k8s-cm-cache: configmap %s successfully created\n", identifier) + klog.V(4).Infof("k8s-cm-cache: configmap %s successfully created", identifier) return nil } @@ -159,11 +159,15 @@ func (k8scm *K8sCMCache) Create(identifier string, data interface{}) error { func (k8scm *K8sCMCache) Get(identifier string, data interface{}) error { cm, err := k8scm.getMetadataCM(identifier) if err != nil { + if apierrs.IsNotFound(err) { + return &CacheEntryNotFound{err} + } + return err } err = json.Unmarshal([]byte(cm.Data[cmDataKey]), data) if err != nil { - return errors.Wrap(err, "k8s-cm-cache: unmarshal error") + return errors.Wrapf(err, "k8s-cm-cache: JSON unmarshaling failed for configmap %s", identifier) } return nil } @@ -172,6 +176,11 @@ func (k8scm *K8sCMCache) Get(identifier string, data interface{}) error { func (k8scm *K8sCMCache) Delete(identifier string) error { err := k8scm.Client.CoreV1().ConfigMaps(k8scm.Namespace).Delete(identifier, nil) if err != nil { + if apierrs.IsNotFound(err) { + klog.V(4).Infof("k8s-cm-cache: cannot delete missing metadata configmap %s, assuming it's already deleted", identifier) + return nil + } + return errors.Wrapf(err, "k8s-cm-cache: couldn't delete metadata configmap %s", identifier) } klog.V(4).Infof("k8s-cm-cache: successfully deleted metadata configmap %s", identifier) diff --git a/pkg/util/k8sconfig.go b/pkg/util/k8sconfig.go new file mode 100644 index 000000000..d00ad71b0 --- /dev/null +++ b/pkg/util/k8sconfig.go @@ -0,0 +1,58 @@ +/* +Copyright 2019 ceph-csi authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "fmt" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8s "k8s.io/client-go/kubernetes" +) + +/* +K8sConfig is a ConfigStore interface implementation that reads configuration +information from k8s secrets. + +Each Ceph cluster configuration secret is expected to be named, +ceph-cluster-, where uniquely identifies and +separates the each Ceph cluster configuration. + +The secret is expected to contain keys, as defined by the ConfigKeys constants +in the ConfigStore interface. +*/ +type K8sConfig struct { + Client *k8s.Clientset + Namespace string +} + +// DataForKey reads the appropriate k8s secret, named using clusterid, and +// returns the contents of key within the secret +func (kc *K8sConfig) DataForKey(clusterid, key string) (data string, err error) { + secret, err := kc.Client.CoreV1().Secrets(kc.Namespace).Get("ceph-cluster-"+clusterid, metav1.GetOptions{}) + if err != nil { + err = fmt.Errorf("error fetching configuration for cluster ID (%s). (%s)", clusterid, err) + return + } + + content, ok := secret.Data[key] + if !ok { + err = fmt.Errorf("missing data for key (%s) in cluster configuration of (%s)", key, clusterid) + return + } + + data = string(content) + return +} diff --git a/pkg/util/log.go b/pkg/util/log.go deleted file mode 100644 index 3d500aaeb..000000000 --- a/pkg/util/log.go +++ /dev/null @@ -1,45 +0,0 @@ -/* -Copyright 2018 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package util - -import ( - "flag" - "os" - - "k8s.io/klog" -) - -// InitLogging initializes klog alongside glog -// XXX: This is just a temporary solution till all deps move to klog -func InitLogging() { - if err := flag.Set("logtostderr", "true"); err != nil { - klog.Errorf("failed to set logtostderr flag: %v", err) - os.Exit(1) - } - - flag.Parse() - - klogFlags := flag.NewFlagSet("klog", flag.ExitOnError) - klog.InitFlags(klogFlags) - - // Sync klog flags with glog - flag.CommandLine.VisitAll(func(f1 *flag.Flag) { - if f2 := klogFlags.Lookup(f1.Name); f2 != nil { - f2.Value.Set(f1.Value.String()) // nolint: errcheck, gosec - } - }) -} diff --git a/pkg/util/nodecache.go b/pkg/util/nodecache.go index 947375b00..bcb94debc 100644 --- a/pkg/util/nodecache.go +++ b/pkg/util/nodecache.go @@ -32,10 +32,9 @@ import ( // NodeCache to store metadata type NodeCache struct { BasePath string + CacheDir string } -var cacheDir = "controller" - var errDec = errors.New("file not found") // EnsureCacheDirectory creates cache directory if not present @@ -44,7 +43,7 @@ func (nc *NodeCache) EnsureCacheDirectory(cacheDir string) error { if _, err := os.Stat(fullPath); os.IsNotExist(err) { // #nosec if err := os.Mkdir(fullPath, 0755); err != nil { - return errors.Wrapf(err, "node-cache: failed to create %s folder with error: %v", fullPath, err) + return errors.Wrapf(err, "node-cache: failed to create %s folder", fullPath) } } return nil @@ -52,15 +51,15 @@ func (nc *NodeCache) EnsureCacheDirectory(cacheDir string) error { //ForAll list the metadata in Nodecache and filters outs based on the pattern func (nc *NodeCache) ForAll(pattern string, destObj interface{}, f ForAllFunc) error { - err := nc.EnsureCacheDirectory(cacheDir) + err := nc.EnsureCacheDirectory(nc.CacheDir) if err != nil { return errors.Wrap(err, "node-cache: couldn't ensure cache directory exists") } - files, err := ioutil.ReadDir(path.Join(nc.BasePath, cacheDir)) + files, err := ioutil.ReadDir(path.Join(nc.BasePath, nc.CacheDir)) if err != nil { return errors.Wrapf(err, "node-cache: failed to read %s folder", nc.BasePath) } - path := path.Join(nc.BasePath, cacheDir) + path := path.Join(nc.BasePath, nc.CacheDir) for _, file := range files { err = decodeObj(path, pattern, file, destObj) if err == errDec { @@ -104,7 +103,7 @@ func decodeObj(filepath, pattern string, file os.FileInfo, destObj interface{}) // Create creates the metadata file in cache directory with identifier name func (nc *NodeCache) Create(identifier string, data interface{}) error { - file := path.Join(nc.BasePath, cacheDir, identifier+".json") + file := path.Join(nc.BasePath, nc.CacheDir, identifier+".json") fp, err := os.Create(file) if err != nil { return errors.Wrapf(err, "node-cache: failed to create metadata storage file %s\n", file) @@ -126,10 +125,14 @@ func (nc *NodeCache) Create(identifier string, data interface{}) error { // Get retrieves the metadata from cache directory with identifier name func (nc *NodeCache) Get(identifier string, data interface{}) error { - file := path.Join(nc.BasePath, cacheDir, identifier+".json") + file := path.Join(nc.BasePath, nc.CacheDir, identifier+".json") // #nosec fp, err := os.Open(file) if err != nil { + if os.IsNotExist(errors.Cause(err)) { + return &CacheEntryNotFound{err} + } + return errors.Wrapf(err, "node-cache: open error for %s", file) } @@ -149,12 +152,16 @@ func (nc *NodeCache) Get(identifier string, data interface{}) error { // Delete deletes the metadata file from cache directory with identifier name func (nc *NodeCache) Delete(identifier string) error { - file := path.Join(nc.BasePath, cacheDir, identifier+".json") + file := path.Join(nc.BasePath, nc.CacheDir, identifier+".json") err := os.Remove(file) if err != nil { - if err != os.ErrNotExist { - return errors.Wrapf(err, "node-cache: error removing file %s", file) + if err == os.ErrNotExist { + klog.V(4).Infof("node-cache: cannot delete missing metadata storage file %s, assuming it's already deleted", file) + return nil } + + return errors.Wrapf(err, "node-cache: error removing file %s", file) + } klog.V(4).Infof("node-cache: successfully deleted metadata storage file at: %+v\n", file) return nil diff --git a/pkg/util/stripsecrets.go b/pkg/util/stripsecrets.go new file mode 100644 index 000000000..8b818de9d --- /dev/null +++ b/pkg/util/stripsecrets.go @@ -0,0 +1,76 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "strings" +) + +const ( + keyArg = "--key=" + secretArg = "secret=" + optionsArgSeparator = ',' + strippedKey = "--key=***stripped***" + strippedSecret = "secret=***stripped***" +) + +// StripSecretInArgs strips values of either "--key" or "secret=". +// `args` is left unchanged. +// Expects only one occurrence of either "--key" or "secret=". +func StripSecretInArgs(args []string) []string { + out := make([]string, len(args)) + copy(out, args) + + if !stripKey(out) { + stripSecret(out) + } + + return out +} + +func stripKey(out []string) bool { + for i := range out { + if strings.HasPrefix(out[i], keyArg) { + out[i] = strippedKey + return true + } + } + + return false +} + +func stripSecret(out []string) bool { + for i := range out { + arg := out[i] + begin := strings.Index(arg, secretArg) + + if begin == -1 { + continue + } + + end := strings.IndexByte(arg[begin+len(secretArg):], optionsArgSeparator) + + out[i] = arg[:begin] + strippedSecret + if end != -1 { + out[i] += arg[end+len(secretArg):] + } + + return true + } + + return false +} diff --git a/pkg/util/util.go b/pkg/util/util.go new file mode 100644 index 000000000..6c298bc99 --- /dev/null +++ b/pkg/util/util.go @@ -0,0 +1,93 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "os" + "path" + "strings" + + "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/klog" +) + +// remove this once kubernetes v1.14.0 release is done +// https://github.com/kubernetes/cloud-provider/blob/master/volume/helpers/rounding.go +const ( + // MiB - MebiByte size + MiB = 1024 * 1024 +) + +// RoundUpToMiB rounds up given quantity upto chunks of MiB +func RoundUpToMiB(size int64) int64 { + requestBytes := size + return roundUpSize(requestBytes, MiB) +} + +func roundUpSize(volumeSizeBytes int64, allocationUnitBytes int64) int64 { + roundedUp := volumeSizeBytes / allocationUnitBytes + if volumeSizeBytes%allocationUnitBytes > 0 { + roundedUp++ + } + return roundedUp +} + +// CreatePersistanceStorage creates storage path and initializes new cache +func CreatePersistanceStorage(sPath, metaDataStore, driverName string) (CachePersister, error) { + var err error + if err = createPersistentStorage(path.Join(sPath, "controller")); err != nil { + klog.Errorf("failed to create persistent storage for controller: %v", err) + return nil, err + } + + if err = createPersistentStorage(path.Join(sPath, "node")); err != nil { + klog.Errorf("failed to create persistent storage for node: %v", err) + return nil, err + } + + cp, err := NewCachePersister(metaDataStore, driverName) + if err != nil { + klog.Errorf("failed to define cache persistence method: %v", err) + return nil, err + } + return cp, err +} + +func createPersistentStorage(persistentStoragePath string) error { + return os.MkdirAll(persistentStoragePath, os.FileMode(0755)) +} + +// ValidateDriverName validates the driver name +func ValidateDriverName(driverName string) error { + if len(driverName) == 0 { + return errors.New("driver name is empty") + } + + if len(driverName) > 63 { + return errors.New("driver name length should be less than 63 chars") + } + var err error + for _, msg := range validation.IsDNS1123Subdomain(strings.ToLower(driverName)) { + if err == nil { + err = errors.New(msg) + continue + } + err = errors.Wrap(err, msg) + } + return err +} diff --git a/scripts/golangci.yml b/scripts/golangci.yml new file mode 100644 index 000000000..1aa82525b --- /dev/null +++ b/scripts/golangci.yml @@ -0,0 +1,153 @@ +--- +# https://github.com/golangci/golangci-lint/blob/master/.golangci.example.yml +# This file contains all available configuration options +# with their default values. + +# options for analysis running +run: + # default concurrency is a available CPU number + concurrency: 4 + + # timeout for analysis, e.g. 30s, 5m, default is 1m + deadline: 10m + + # exit code when at least one issue was found, default is 1 + issues-exit-code: 1 + + # include test files or not, default is true + tests: true + + # which dirs to skip: they won't be analyzed; + # can use regexp here: generated.*, regexp is applied on full path; + # default value is empty list, but next dirs are always skipped independently + # from this option's value: + # vendor$, third_party$, testdata$, examples$, Godeps$, builtin$ + skip-dirs: + - vendor$ + + # which files to skip: they will be analyzed, but issues from them + # won't be reported. Default value is empty list, but there is + # no need to include all autogenerated files, we confidently recognize + # autogenerated files. If it's not please let us know. + skip-files: + +# output configuration options +output: + # colored-line-number|line-number|json|tab|checkstyle|code-climate, + # default is "colored-line-number" + format: colored-line-number + + # print lines of code with issue, default is true + print-issued-lines: true + + # print linter name in the end of issue text, default is true + print-linter-name: true + +# all available settings of specific linters +linters-settings: + errcheck: + # report about not checking of errors in type assetions: + # `a := b.(MyStruct)`; + # default is false: such cases aren't reported by default. + check-type-assertions: true + + # report about assignment of errors to blank identifier: + # `num, _ := strconv.Atoi(numStr)`; + # default is false: such cases aren't reported by default. + check-blank: true + + # path to a file containing a list of functions to exclude from checking + # see https://github.com/kisielk/errcheck#excluding-functions for details + # exclude: /path/to/file.txt + govet: + # report about shadowed variables + check-shadowing: true + golint: + # minimal confidence for issues, default is 0.8 + min-confidence: 0 + gofmt: + # simplify code: gofmt with `-s` option, true by default + simplify: true + goimports: + # put imports beginning with prefix after 3rd-party packages; + # it's a comma-separated list of prefixes + local-prefixes: github.com/ceph/csph-csi + gocyclo: + # minimal code complexity to report, 30 by default (but we recommend 10-20) + min-complexity: 20 + maligned: + # print struct with more effective memory layout or not, false by default + suggest-new: true + dupl: + # tokens count to trigger issue, 150 by default + threshold: 100 + goconst: + # minimal length of string constant, 3 by default + min-len: 3 + # minimal occurrences count to trigger, 3 by default + min-occurrences: 3 + depguard: + list-type: blacklist + include-go-root: false + packages: + - github.com/davecgh/go-spew/spew + misspell: + # Correct spellings using locale preferences for US or UK. + # Default is to use a neutral variety of English. + # Setting locale to US will correct the British spelling of 'colour' to + # 'color'. + locale: US + ignore-words: + - someword + lll: + # max line length, lines longer will be reported. Default is 120. + # '\t' is counted as 1 character by default, and can be changed with the + # tab-width option + # TODO make line length to 120 char + line-length: 180 + # tab width in spaces. Default to 1. + tab-width: 1 + unused: + # treat code as a program (not a library) and report unused exported + # identifiers; default is false. + # XXX: if you enable this setting, unused will report a lot of + # false-positives in text editors: + # if it's called for subdir of a project it can't find funcs usages. + # All text editor integrations + # with golangci-lint call it on a directory with the changed file. + check-exported: false + unparam: + # Inspect exported functions, default is false. Set to true if no external + # program/library imports your code. + # XXX: if you enable this setting, unparam will report a lot of + # false-positives in text editors: + # if it's called for subdir of a project it can't find external + # interfaces. All text editor integrations + # with golangci-lint call it on a directory with the changed file. + check-exported: false + nakedret: + # make an issue if func has more lines of code than this setting and + # it has naked returns; default is 30 + max-func-lines: 30 + +linters: + enable: + - megacheck + - govet + - golint + - stylecheck + - interfacer + - unconvert + - gofmt + - gocyclo + - maligned + - lll + - nakedret + enable-all: false + disable: + - prealloc + disable-all: false + presets: + - bugs + - unused + fast: false diff --git a/scripts/lint-go.sh b/scripts/lint-go.sh index 0f9d49d18..c55112749 100755 --- a/scripts/lint-go.sh +++ b/scripts/lint-go.sh @@ -2,11 +2,8 @@ set -o pipefail -if [[ -x "$(command -v gometalinter)" ]]; then - gometalinter -j "${GO_METALINTER_THREADS:-1}" \ - --sort path --sort line --sort column --deadline=10m \ - --enable=misspell --enable=staticcheck \ - --vendor "${@-./...}" +if [[ -x "$(command -v golangci-lint)" ]]; then + golangci-lint --config=scripts/golangci.yml run ./... -v else - echo "WARNING: gometalinter not found, skipping lint tests" >&2 + echo "WARNING: golangci-lint not found, skipping lint tests" >&2 fi diff --git a/scripts/lint-text.sh b/scripts/lint-text.sh index 066c8d382..c7bcf2f9f 100755 --- a/scripts/lint-text.sh +++ b/scripts/lint-text.sh @@ -44,6 +44,6 @@ run_check '.*\.(ba)?sh' bash -n # Install via: pip install yamllint # disable yamlint chekck for helm chats -run_check '.*\.ya?ml' yamllint -s -d "{extends: default, rules: {line-length: {allow-non-breakable-inline-mappings: true}},ignore: deploy/rbd/helm/templates/*.yaml}" +run_check '.*\.ya?ml' yamllint -s -d "{extends: default, rules: {line-length: {allow-non-breakable-inline-mappings: true}},ignore: deploy/*/helm/templates/*.yaml}" echo "ALL OK." diff --git a/vendor/github.com/golang/glog/LICENSE b/vendor/github.com/golang/glog/LICENSE deleted file mode 100644 index 37ec93a14..000000000 --- a/vendor/github.com/golang/glog/LICENSE +++ /dev/null @@ -1,191 +0,0 @@ -Apache License -Version 2.0, January 2004 -http://www.apache.org/licenses/ - -TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - -1. Definitions. - -"License" shall mean the terms and conditions for use, reproduction, and -distribution as defined by Sections 1 through 9 of this document. - -"Licensor" shall mean the copyright owner or entity authorized by the copyright -owner that is granting the License. - -"Legal Entity" shall mean the union of the acting entity and all other entities -that control, are controlled by, or are under common control with that entity. -For the purposes of this definition, "control" means (i) the power, direct or -indirect, to cause the direction or management of such entity, whether by -contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the -outstanding shares, or (iii) beneficial ownership of such entity. - -"You" (or "Your") shall mean an individual or Legal Entity exercising -permissions granted by this License. - -"Source" form shall mean the preferred form for making modifications, including -but not limited to software source code, documentation source, and configuration -files. - -"Object" form shall mean any form resulting from mechanical transformation or -translation of a Source form, including but not limited to compiled object code, -generated documentation, and conversions to other media types. - -"Work" shall mean the work of authorship, whether in Source or Object form, made -available under the License, as indicated by a copyright notice that is included -in or attached to the work (an example is provided in the Appendix below). - -"Derivative Works" shall mean any work, whether in Source or Object form, that -is based on (or derived from) the Work and for which the editorial revisions, -annotations, elaborations, or other modifications represent, as a whole, an -original work of authorship. For the purposes of this License, Derivative Works -shall not include works that remain separable from, or merely link (or bind by -name) to the interfaces of, the Work and Derivative Works thereof. - -"Contribution" shall mean any work of authorship, including the original version -of the Work and any modifications or additions to that Work or Derivative Works -thereof, that is intentionally submitted to Licensor for inclusion in the Work -by the copyright owner or by an individual or Legal Entity authorized to submit -on behalf of the copyright owner. For the purposes of this definition, -"submitted" means any form of electronic, verbal, or written communication sent -to the Licensor or its representatives, including but not limited to -communication on electronic mailing lists, source code control systems, and -issue tracking systems that are managed by, or on behalf of, the Licensor for -the purpose of discussing and improving the Work, but excluding communication -that is conspicuously marked or otherwise designated in writing by the copyright -owner as "Not a Contribution." - -"Contributor" shall mean Licensor and any individual or Legal Entity on behalf -of whom a Contribution has been received by Licensor and subsequently -incorporated within the Work. - -2. Grant of Copyright License. - -Subject to the terms and conditions of this License, each Contributor hereby -grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, -irrevocable copyright license to reproduce, prepare Derivative Works of, -publicly display, publicly perform, sublicense, and distribute the Work and such -Derivative Works in Source or Object form. - -3. Grant of Patent License. - -Subject to the terms and conditions of this License, each Contributor hereby -grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, -irrevocable (except as stated in this section) patent license to make, have -made, use, offer to sell, sell, import, and otherwise transfer the Work, where -such license applies only to those patent claims licensable by such Contributor -that are necessarily infringed by their Contribution(s) alone or by combination -of their Contribution(s) with the Work to which such Contribution(s) was -submitted. If You institute patent litigation against any entity (including a -cross-claim or counterclaim in a lawsuit) alleging that the Work or a -Contribution incorporated within the Work constitutes direct or contributory -patent infringement, then any patent licenses granted to You under this License -for that Work shall terminate as of the date such litigation is filed. - -4. Redistribution. - -You may reproduce and distribute copies of the Work or Derivative Works thereof -in any medium, with or without modifications, and in Source or Object form, -provided that You meet the following conditions: - -You must give any other recipients of the Work or Derivative Works a copy of -this License; and -You must cause any modified files to carry prominent notices stating that You -changed the files; and -You must retain, in the Source form of any Derivative Works that You distribute, -all copyright, patent, trademark, and attribution notices from the Source form -of the Work, excluding those notices that do not pertain to any part of the -Derivative Works; and -If the Work includes a "NOTICE" text file as part of its distribution, then any -Derivative Works that You distribute must include a readable copy of the -attribution notices contained within such NOTICE file, excluding those notices -that do not pertain to any part of the Derivative Works, in at least one of the -following places: within a NOTICE text file distributed as part of the -Derivative Works; within the Source form or documentation, if provided along -with the Derivative Works; or, within a display generated by the Derivative -Works, if and wherever such third-party notices normally appear. The contents of -the NOTICE file are for informational purposes only and do not modify the -License. You may add Your own attribution notices within Derivative Works that -You distribute, alongside or as an addendum to the NOTICE text from the Work, -provided that such additional attribution notices cannot be construed as -modifying the License. -You may add Your own copyright statement to Your modifications and may provide -additional or different license terms and conditions for use, reproduction, or -distribution of Your modifications, or for any such Derivative Works as a whole, -provided Your use, reproduction, and distribution of the Work otherwise complies -with the conditions stated in this License. - -5. Submission of Contributions. - -Unless You explicitly state otherwise, any Contribution intentionally submitted -for inclusion in the Work by You to the Licensor shall be under the terms and -conditions of this License, without any additional terms or conditions. -Notwithstanding the above, nothing herein shall supersede or modify the terms of -any separate license agreement you may have executed with Licensor regarding -such Contributions. - -6. Trademarks. - -This License does not grant permission to use the trade names, trademarks, -service marks, or product names of the Licensor, except as required for -reasonable and customary use in describing the origin of the Work and -reproducing the content of the NOTICE file. - -7. Disclaimer of Warranty. - -Unless required by applicable law or agreed to in writing, Licensor provides the -Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, -including, without limitation, any warranties or conditions of TITLE, -NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are -solely responsible for determining the appropriateness of using or -redistributing the Work and assume any risks associated with Your exercise of -permissions under this License. - -8. Limitation of Liability. - -In no event and under no legal theory, whether in tort (including negligence), -contract, or otherwise, unless required by applicable law (such as deliberate -and grossly negligent acts) or agreed to in writing, shall any Contributor be -liable to You for damages, including any direct, indirect, special, incidental, -or consequential damages of any character arising as a result of this License or -out of the use or inability to use the Work (including but not limited to -damages for loss of goodwill, work stoppage, computer failure or malfunction, or -any and all other commercial damages or losses), even if such Contributor has -been advised of the possibility of such damages. - -9. Accepting Warranty or Additional Liability. - -While redistributing the Work or Derivative Works thereof, You may choose to -offer, and charge a fee for, acceptance of support, warranty, indemnity, or -other liability obligations and/or rights consistent with this License. However, -in accepting such obligations, You may act only on Your own behalf and on Your -sole responsibility, not on behalf of any other Contributor, and only if You -agree to indemnify, defend, and hold each Contributor harmless for any liability -incurred by, or claims asserted against, such Contributor by reason of your -accepting any such warranty or additional liability. - -END OF TERMS AND CONDITIONS - -APPENDIX: How to apply the Apache License to your work - -To apply the Apache License to your work, attach the following boilerplate -notice, with the fields enclosed by brackets "[]" replaced with your own -identifying information. (Don't include the brackets!) The text should be -enclosed in the appropriate comment syntax for the file format. We also -recommend that a file or class name and description of purpose be included on -the same "printed page" as the copyright notice for easier identification within -third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/vendor/github.com/golang/glog/glog.go b/vendor/github.com/golang/glog/glog.go deleted file mode 100644 index 54bd7afdc..000000000 --- a/vendor/github.com/golang/glog/glog.go +++ /dev/null @@ -1,1180 +0,0 @@ -// Go support for leveled logs, analogous to https://code.google.com/p/google-glog/ -// -// Copyright 2013 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package glog implements logging analogous to the Google-internal C++ INFO/ERROR/V setup. -// It provides functions Info, Warning, Error, Fatal, plus formatting variants such as -// Infof. It also provides V-style logging controlled by the -v and -vmodule=file=2 flags. -// -// Basic examples: -// -// glog.Info("Prepare to repel boarders") -// -// glog.Fatalf("Initialization failed: %s", err) -// -// See the documentation for the V function for an explanation of these examples: -// -// if glog.V(2) { -// glog.Info("Starting transaction...") -// } -// -// glog.V(2).Infoln("Processed", nItems, "elements") -// -// Log output is buffered and written periodically using Flush. Programs -// should call Flush before exiting to guarantee all log output is written. -// -// By default, all log statements write to files in a temporary directory. -// This package provides several flags that modify this behavior. -// As a result, flag.Parse must be called before any logging is done. -// -// -logtostderr=false -// Logs are written to standard error instead of to files. -// -alsologtostderr=false -// Logs are written to standard error as well as to files. -// -stderrthreshold=ERROR -// Log events at or above this severity are logged to standard -// error as well as to files. -// -log_dir="" -// Log files will be written to this directory instead of the -// default temporary directory. -// -// Other flags provide aids to debugging. -// -// -log_backtrace_at="" -// When set to a file and line number holding a logging statement, -// such as -// -log_backtrace_at=gopherflakes.go:234 -// a stack trace will be written to the Info log whenever execution -// hits that statement. (Unlike with -vmodule, the ".go" must be -// present.) -// -v=0 -// Enable V-leveled logging at the specified level. -// -vmodule="" -// The syntax of the argument is a comma-separated list of pattern=N, -// where pattern is a literal file name (minus the ".go" suffix) or -// "glob" pattern and N is a V level. For instance, -// -vmodule=gopher*=3 -// sets the V level to 3 in all Go files whose names begin "gopher". -// -package glog - -import ( - "bufio" - "bytes" - "errors" - "flag" - "fmt" - "io" - stdLog "log" - "os" - "path/filepath" - "runtime" - "strconv" - "strings" - "sync" - "sync/atomic" - "time" -) - -// severity identifies the sort of log: info, warning etc. It also implements -// the flag.Value interface. The -stderrthreshold flag is of type severity and -// should be modified only through the flag.Value interface. The values match -// the corresponding constants in C++. -type severity int32 // sync/atomic int32 - -// These constants identify the log levels in order of increasing severity. -// A message written to a high-severity log file is also written to each -// lower-severity log file. -const ( - infoLog severity = iota - warningLog - errorLog - fatalLog - numSeverity = 4 -) - -const severityChar = "IWEF" - -var severityName = []string{ - infoLog: "INFO", - warningLog: "WARNING", - errorLog: "ERROR", - fatalLog: "FATAL", -} - -// get returns the value of the severity. -func (s *severity) get() severity { - return severity(atomic.LoadInt32((*int32)(s))) -} - -// set sets the value of the severity. -func (s *severity) set(val severity) { - atomic.StoreInt32((*int32)(s), int32(val)) -} - -// String is part of the flag.Value interface. -func (s *severity) String() string { - return strconv.FormatInt(int64(*s), 10) -} - -// Get is part of the flag.Value interface. -func (s *severity) Get() interface{} { - return *s -} - -// Set is part of the flag.Value interface. -func (s *severity) Set(value string) error { - var threshold severity - // Is it a known name? - if v, ok := severityByName(value); ok { - threshold = v - } else { - v, err := strconv.Atoi(value) - if err != nil { - return err - } - threshold = severity(v) - } - logging.stderrThreshold.set(threshold) - return nil -} - -func severityByName(s string) (severity, bool) { - s = strings.ToUpper(s) - for i, name := range severityName { - if name == s { - return severity(i), true - } - } - return 0, false -} - -// OutputStats tracks the number of output lines and bytes written. -type OutputStats struct { - lines int64 - bytes int64 -} - -// Lines returns the number of lines written. -func (s *OutputStats) Lines() int64 { - return atomic.LoadInt64(&s.lines) -} - -// Bytes returns the number of bytes written. -func (s *OutputStats) Bytes() int64 { - return atomic.LoadInt64(&s.bytes) -} - -// Stats tracks the number of lines of output and number of bytes -// per severity level. Values must be read with atomic.LoadInt64. -var Stats struct { - Info, Warning, Error OutputStats -} - -var severityStats = [numSeverity]*OutputStats{ - infoLog: &Stats.Info, - warningLog: &Stats.Warning, - errorLog: &Stats.Error, -} - -// Level is exported because it appears in the arguments to V and is -// the type of the v flag, which can be set programmatically. -// It's a distinct type because we want to discriminate it from logType. -// Variables of type level are only changed under logging.mu. -// The -v flag is read only with atomic ops, so the state of the logging -// module is consistent. - -// Level is treated as a sync/atomic int32. - -// Level specifies a level of verbosity for V logs. *Level implements -// flag.Value; the -v flag is of type Level and should be modified -// only through the flag.Value interface. -type Level int32 - -// get returns the value of the Level. -func (l *Level) get() Level { - return Level(atomic.LoadInt32((*int32)(l))) -} - -// set sets the value of the Level. -func (l *Level) set(val Level) { - atomic.StoreInt32((*int32)(l), int32(val)) -} - -// String is part of the flag.Value interface. -func (l *Level) String() string { - return strconv.FormatInt(int64(*l), 10) -} - -// Get is part of the flag.Value interface. -func (l *Level) Get() interface{} { - return *l -} - -// Set is part of the flag.Value interface. -func (l *Level) Set(value string) error { - v, err := strconv.Atoi(value) - if err != nil { - return err - } - logging.mu.Lock() - defer logging.mu.Unlock() - logging.setVState(Level(v), logging.vmodule.filter, false) - return nil -} - -// moduleSpec represents the setting of the -vmodule flag. -type moduleSpec struct { - filter []modulePat -} - -// modulePat contains a filter for the -vmodule flag. -// It holds a verbosity level and a file pattern to match. -type modulePat struct { - pattern string - literal bool // The pattern is a literal string - level Level -} - -// match reports whether the file matches the pattern. It uses a string -// comparison if the pattern contains no metacharacters. -func (m *modulePat) match(file string) bool { - if m.literal { - return file == m.pattern - } - match, _ := filepath.Match(m.pattern, file) - return match -} - -func (m *moduleSpec) String() string { - // Lock because the type is not atomic. TODO: clean this up. - logging.mu.Lock() - defer logging.mu.Unlock() - var b bytes.Buffer - for i, f := range m.filter { - if i > 0 { - b.WriteRune(',') - } - fmt.Fprintf(&b, "%s=%d", f.pattern, f.level) - } - return b.String() -} - -// Get is part of the (Go 1.2) flag.Getter interface. It always returns nil for this flag type since the -// struct is not exported. -func (m *moduleSpec) Get() interface{} { - return nil -} - -var errVmoduleSyntax = errors.New("syntax error: expect comma-separated list of filename=N") - -// Syntax: -vmodule=recordio=2,file=1,gfs*=3 -func (m *moduleSpec) Set(value string) error { - var filter []modulePat - for _, pat := range strings.Split(value, ",") { - if len(pat) == 0 { - // Empty strings such as from a trailing comma can be ignored. - continue - } - patLev := strings.Split(pat, "=") - if len(patLev) != 2 || len(patLev[0]) == 0 || len(patLev[1]) == 0 { - return errVmoduleSyntax - } - pattern := patLev[0] - v, err := strconv.Atoi(patLev[1]) - if err != nil { - return errors.New("syntax error: expect comma-separated list of filename=N") - } - if v < 0 { - return errors.New("negative value for vmodule level") - } - if v == 0 { - continue // Ignore. It's harmless but no point in paying the overhead. - } - // TODO: check syntax of filter? - filter = append(filter, modulePat{pattern, isLiteral(pattern), Level(v)}) - } - logging.mu.Lock() - defer logging.mu.Unlock() - logging.setVState(logging.verbosity, filter, true) - return nil -} - -// isLiteral reports whether the pattern is a literal string, that is, has no metacharacters -// that require filepath.Match to be called to match the pattern. -func isLiteral(pattern string) bool { - return !strings.ContainsAny(pattern, `\*?[]`) -} - -// traceLocation represents the setting of the -log_backtrace_at flag. -type traceLocation struct { - file string - line int -} - -// isSet reports whether the trace location has been specified. -// logging.mu is held. -func (t *traceLocation) isSet() bool { - return t.line > 0 -} - -// match reports whether the specified file and line matches the trace location. -// The argument file name is the full path, not the basename specified in the flag. -// logging.mu is held. -func (t *traceLocation) match(file string, line int) bool { - if t.line != line { - return false - } - if i := strings.LastIndex(file, "/"); i >= 0 { - file = file[i+1:] - } - return t.file == file -} - -func (t *traceLocation) String() string { - // Lock because the type is not atomic. TODO: clean this up. - logging.mu.Lock() - defer logging.mu.Unlock() - return fmt.Sprintf("%s:%d", t.file, t.line) -} - -// Get is part of the (Go 1.2) flag.Getter interface. It always returns nil for this flag type since the -// struct is not exported -func (t *traceLocation) Get() interface{} { - return nil -} - -var errTraceSyntax = errors.New("syntax error: expect file.go:234") - -// Syntax: -log_backtrace_at=gopherflakes.go:234 -// Note that unlike vmodule the file extension is included here. -func (t *traceLocation) Set(value string) error { - if value == "" { - // Unset. - t.line = 0 - t.file = "" - } - fields := strings.Split(value, ":") - if len(fields) != 2 { - return errTraceSyntax - } - file, line := fields[0], fields[1] - if !strings.Contains(file, ".") { - return errTraceSyntax - } - v, err := strconv.Atoi(line) - if err != nil { - return errTraceSyntax - } - if v <= 0 { - return errors.New("negative or zero value for level") - } - logging.mu.Lock() - defer logging.mu.Unlock() - t.line = v - t.file = file - return nil -} - -// flushSyncWriter is the interface satisfied by logging destinations. -type flushSyncWriter interface { - Flush() error - Sync() error - io.Writer -} - -func init() { - flag.BoolVar(&logging.toStderr, "logtostderr", false, "log to standard error instead of files") - flag.BoolVar(&logging.alsoToStderr, "alsologtostderr", false, "log to standard error as well as files") - flag.Var(&logging.verbosity, "v", "log level for V logs") - flag.Var(&logging.stderrThreshold, "stderrthreshold", "logs at or above this threshold go to stderr") - flag.Var(&logging.vmodule, "vmodule", "comma-separated list of pattern=N settings for file-filtered logging") - flag.Var(&logging.traceLocation, "log_backtrace_at", "when logging hits line file:N, emit a stack trace") - - // Default stderrThreshold is ERROR. - logging.stderrThreshold = errorLog - - logging.setVState(0, nil, false) - go logging.flushDaemon() -} - -// Flush flushes all pending log I/O. -func Flush() { - logging.lockAndFlushAll() -} - -// loggingT collects all the global state of the logging setup. -type loggingT struct { - // Boolean flags. Not handled atomically because the flag.Value interface - // does not let us avoid the =true, and that shorthand is necessary for - // compatibility. TODO: does this matter enough to fix? Seems unlikely. - toStderr bool // The -logtostderr flag. - alsoToStderr bool // The -alsologtostderr flag. - - // Level flag. Handled atomically. - stderrThreshold severity // The -stderrthreshold flag. - - // freeList is a list of byte buffers, maintained under freeListMu. - freeList *buffer - // freeListMu maintains the free list. It is separate from the main mutex - // so buffers can be grabbed and printed to without holding the main lock, - // for better parallelization. - freeListMu sync.Mutex - - // mu protects the remaining elements of this structure and is - // used to synchronize logging. - mu sync.Mutex - // file holds writer for each of the log types. - file [numSeverity]flushSyncWriter - // pcs is used in V to avoid an allocation when computing the caller's PC. - pcs [1]uintptr - // vmap is a cache of the V Level for each V() call site, identified by PC. - // It is wiped whenever the vmodule flag changes state. - vmap map[uintptr]Level - // filterLength stores the length of the vmodule filter chain. If greater - // than zero, it means vmodule is enabled. It may be read safely - // using sync.LoadInt32, but is only modified under mu. - filterLength int32 - // traceLocation is the state of the -log_backtrace_at flag. - traceLocation traceLocation - // These flags are modified only under lock, although verbosity may be fetched - // safely using atomic.LoadInt32. - vmodule moduleSpec // The state of the -vmodule flag. - verbosity Level // V logging level, the value of the -v flag/ -} - -// buffer holds a byte Buffer for reuse. The zero value is ready for use. -type buffer struct { - bytes.Buffer - tmp [64]byte // temporary byte array for creating headers. - next *buffer -} - -var logging loggingT - -// setVState sets a consistent state for V logging. -// l.mu is held. -func (l *loggingT) setVState(verbosity Level, filter []modulePat, setFilter bool) { - // Turn verbosity off so V will not fire while we are in transition. - logging.verbosity.set(0) - // Ditto for filter length. - atomic.StoreInt32(&logging.filterLength, 0) - - // Set the new filters and wipe the pc->Level map if the filter has changed. - if setFilter { - logging.vmodule.filter = filter - logging.vmap = make(map[uintptr]Level) - } - - // Things are consistent now, so enable filtering and verbosity. - // They are enabled in order opposite to that in V. - atomic.StoreInt32(&logging.filterLength, int32(len(filter))) - logging.verbosity.set(verbosity) -} - -// getBuffer returns a new, ready-to-use buffer. -func (l *loggingT) getBuffer() *buffer { - l.freeListMu.Lock() - b := l.freeList - if b != nil { - l.freeList = b.next - } - l.freeListMu.Unlock() - if b == nil { - b = new(buffer) - } else { - b.next = nil - b.Reset() - } - return b -} - -// putBuffer returns a buffer to the free list. -func (l *loggingT) putBuffer(b *buffer) { - if b.Len() >= 256 { - // Let big buffers die a natural death. - return - } - l.freeListMu.Lock() - b.next = l.freeList - l.freeList = b - l.freeListMu.Unlock() -} - -var timeNow = time.Now // Stubbed out for testing. - -/* -header formats a log header as defined by the C++ implementation. -It returns a buffer containing the formatted header and the user's file and line number. -The depth specifies how many stack frames above lives the source line to be identified in the log message. - -Log lines have this form: - Lmmdd hh:mm:ss.uuuuuu threadid file:line] msg... -where the fields are defined as follows: - L A single character, representing the log level (eg 'I' for INFO) - mm The month (zero padded; ie May is '05') - dd The day (zero padded) - hh:mm:ss.uuuuuu Time in hours, minutes and fractional seconds - threadid The space-padded thread ID as returned by GetTID() - file The file name - line The line number - msg The user-supplied message -*/ -func (l *loggingT) header(s severity, depth int) (*buffer, string, int) { - _, file, line, ok := runtime.Caller(3 + depth) - if !ok { - file = "???" - line = 1 - } else { - slash := strings.LastIndex(file, "/") - if slash >= 0 { - file = file[slash+1:] - } - } - return l.formatHeader(s, file, line), file, line -} - -// formatHeader formats a log header using the provided file name and line number. -func (l *loggingT) formatHeader(s severity, file string, line int) *buffer { - now := timeNow() - if line < 0 { - line = 0 // not a real line number, but acceptable to someDigits - } - if s > fatalLog { - s = infoLog // for safety. - } - buf := l.getBuffer() - - // Avoid Fprintf, for speed. The format is so simple that we can do it quickly by hand. - // It's worth about 3X. Fprintf is hard. - _, month, day := now.Date() - hour, minute, second := now.Clock() - // Lmmdd hh:mm:ss.uuuuuu threadid file:line] - buf.tmp[0] = severityChar[s] - buf.twoDigits(1, int(month)) - buf.twoDigits(3, day) - buf.tmp[5] = ' ' - buf.twoDigits(6, hour) - buf.tmp[8] = ':' - buf.twoDigits(9, minute) - buf.tmp[11] = ':' - buf.twoDigits(12, second) - buf.tmp[14] = '.' - buf.nDigits(6, 15, now.Nanosecond()/1000, '0') - buf.tmp[21] = ' ' - buf.nDigits(7, 22, pid, ' ') // TODO: should be TID - buf.tmp[29] = ' ' - buf.Write(buf.tmp[:30]) - buf.WriteString(file) - buf.tmp[0] = ':' - n := buf.someDigits(1, line) - buf.tmp[n+1] = ']' - buf.tmp[n+2] = ' ' - buf.Write(buf.tmp[:n+3]) - return buf -} - -// Some custom tiny helper functions to print the log header efficiently. - -const digits = "0123456789" - -// twoDigits formats a zero-prefixed two-digit integer at buf.tmp[i]. -func (buf *buffer) twoDigits(i, d int) { - buf.tmp[i+1] = digits[d%10] - d /= 10 - buf.tmp[i] = digits[d%10] -} - -// nDigits formats an n-digit integer at buf.tmp[i], -// padding with pad on the left. -// It assumes d >= 0. -func (buf *buffer) nDigits(n, i, d int, pad byte) { - j := n - 1 - for ; j >= 0 && d > 0; j-- { - buf.tmp[i+j] = digits[d%10] - d /= 10 - } - for ; j >= 0; j-- { - buf.tmp[i+j] = pad - } -} - -// someDigits formats a zero-prefixed variable-width integer at buf.tmp[i]. -func (buf *buffer) someDigits(i, d int) int { - // Print into the top, then copy down. We know there's space for at least - // a 10-digit number. - j := len(buf.tmp) - for { - j-- - buf.tmp[j] = digits[d%10] - d /= 10 - if d == 0 { - break - } - } - return copy(buf.tmp[i:], buf.tmp[j:]) -} - -func (l *loggingT) println(s severity, args ...interface{}) { - buf, file, line := l.header(s, 0) - fmt.Fprintln(buf, args...) - l.output(s, buf, file, line, false) -} - -func (l *loggingT) print(s severity, args ...interface{}) { - l.printDepth(s, 1, args...) -} - -func (l *loggingT) printDepth(s severity, depth int, args ...interface{}) { - buf, file, line := l.header(s, depth) - fmt.Fprint(buf, args...) - if buf.Bytes()[buf.Len()-1] != '\n' { - buf.WriteByte('\n') - } - l.output(s, buf, file, line, false) -} - -func (l *loggingT) printf(s severity, format string, args ...interface{}) { - buf, file, line := l.header(s, 0) - fmt.Fprintf(buf, format, args...) - if buf.Bytes()[buf.Len()-1] != '\n' { - buf.WriteByte('\n') - } - l.output(s, buf, file, line, false) -} - -// printWithFileLine behaves like print but uses the provided file and line number. If -// alsoLogToStderr is true, the log message always appears on standard error; it -// will also appear in the log file unless --logtostderr is set. -func (l *loggingT) printWithFileLine(s severity, file string, line int, alsoToStderr bool, args ...interface{}) { - buf := l.formatHeader(s, file, line) - fmt.Fprint(buf, args...) - if buf.Bytes()[buf.Len()-1] != '\n' { - buf.WriteByte('\n') - } - l.output(s, buf, file, line, alsoToStderr) -} - -// output writes the data to the log files and releases the buffer. -func (l *loggingT) output(s severity, buf *buffer, file string, line int, alsoToStderr bool) { - l.mu.Lock() - if l.traceLocation.isSet() { - if l.traceLocation.match(file, line) { - buf.Write(stacks(false)) - } - } - data := buf.Bytes() - if !flag.Parsed() { - os.Stderr.Write([]byte("ERROR: logging before flag.Parse: ")) - os.Stderr.Write(data) - } else if l.toStderr { - os.Stderr.Write(data) - } else { - if alsoToStderr || l.alsoToStderr || s >= l.stderrThreshold.get() { - os.Stderr.Write(data) - } - if l.file[s] == nil { - if err := l.createFiles(s); err != nil { - os.Stderr.Write(data) // Make sure the message appears somewhere. - l.exit(err) - } - } - switch s { - case fatalLog: - l.file[fatalLog].Write(data) - fallthrough - case errorLog: - l.file[errorLog].Write(data) - fallthrough - case warningLog: - l.file[warningLog].Write(data) - fallthrough - case infoLog: - l.file[infoLog].Write(data) - } - } - if s == fatalLog { - // If we got here via Exit rather than Fatal, print no stacks. - if atomic.LoadUint32(&fatalNoStacks) > 0 { - l.mu.Unlock() - timeoutFlush(10 * time.Second) - os.Exit(1) - } - // Dump all goroutine stacks before exiting. - // First, make sure we see the trace for the current goroutine on standard error. - // If -logtostderr has been specified, the loop below will do that anyway - // as the first stack in the full dump. - if !l.toStderr { - os.Stderr.Write(stacks(false)) - } - // Write the stack trace for all goroutines to the files. - trace := stacks(true) - logExitFunc = func(error) {} // If we get a write error, we'll still exit below. - for log := fatalLog; log >= infoLog; log-- { - if f := l.file[log]; f != nil { // Can be nil if -logtostderr is set. - f.Write(trace) - } - } - l.mu.Unlock() - timeoutFlush(10 * time.Second) - os.Exit(255) // C++ uses -1, which is silly because it's anded with 255 anyway. - } - l.putBuffer(buf) - l.mu.Unlock() - if stats := severityStats[s]; stats != nil { - atomic.AddInt64(&stats.lines, 1) - atomic.AddInt64(&stats.bytes, int64(len(data))) - } -} - -// timeoutFlush calls Flush and returns when it completes or after timeout -// elapses, whichever happens first. This is needed because the hooks invoked -// by Flush may deadlock when glog.Fatal is called from a hook that holds -// a lock. -func timeoutFlush(timeout time.Duration) { - done := make(chan bool, 1) - go func() { - Flush() // calls logging.lockAndFlushAll() - done <- true - }() - select { - case <-done: - case <-time.After(timeout): - fmt.Fprintln(os.Stderr, "glog: Flush took longer than", timeout) - } -} - -// stacks is a wrapper for runtime.Stack that attempts to recover the data for all goroutines. -func stacks(all bool) []byte { - // We don't know how big the traces are, so grow a few times if they don't fit. Start large, though. - n := 10000 - if all { - n = 100000 - } - var trace []byte - for i := 0; i < 5; i++ { - trace = make([]byte, n) - nbytes := runtime.Stack(trace, all) - if nbytes < len(trace) { - return trace[:nbytes] - } - n *= 2 - } - return trace -} - -// logExitFunc provides a simple mechanism to override the default behavior -// of exiting on error. Used in testing and to guarantee we reach a required exit -// for fatal logs. Instead, exit could be a function rather than a method but that -// would make its use clumsier. -var logExitFunc func(error) - -// exit is called if there is trouble creating or writing log files. -// It flushes the logs and exits the program; there's no point in hanging around. -// l.mu is held. -func (l *loggingT) exit(err error) { - fmt.Fprintf(os.Stderr, "log: exiting because of error: %s\n", err) - // If logExitFunc is set, we do that instead of exiting. - if logExitFunc != nil { - logExitFunc(err) - return - } - l.flushAll() - os.Exit(2) -} - -// syncBuffer joins a bufio.Writer to its underlying file, providing access to the -// file's Sync method and providing a wrapper for the Write method that provides log -// file rotation. There are conflicting methods, so the file cannot be embedded. -// l.mu is held for all its methods. -type syncBuffer struct { - logger *loggingT - *bufio.Writer - file *os.File - sev severity - nbytes uint64 // The number of bytes written to this file -} - -func (sb *syncBuffer) Sync() error { - return sb.file.Sync() -} - -func (sb *syncBuffer) Write(p []byte) (n int, err error) { - if sb.nbytes+uint64(len(p)) >= MaxSize { - if err := sb.rotateFile(time.Now()); err != nil { - sb.logger.exit(err) - } - } - n, err = sb.Writer.Write(p) - sb.nbytes += uint64(n) - if err != nil { - sb.logger.exit(err) - } - return -} - -// rotateFile closes the syncBuffer's file and starts a new one. -func (sb *syncBuffer) rotateFile(now time.Time) error { - if sb.file != nil { - sb.Flush() - sb.file.Close() - } - var err error - sb.file, _, err = create(severityName[sb.sev], now) - sb.nbytes = 0 - if err != nil { - return err - } - - sb.Writer = bufio.NewWriterSize(sb.file, bufferSize) - - // Write header. - var buf bytes.Buffer - fmt.Fprintf(&buf, "Log file created at: %s\n", now.Format("2006/01/02 15:04:05")) - fmt.Fprintf(&buf, "Running on machine: %s\n", host) - fmt.Fprintf(&buf, "Binary: Built with %s %s for %s/%s\n", runtime.Compiler, runtime.Version(), runtime.GOOS, runtime.GOARCH) - fmt.Fprintf(&buf, "Log line format: [IWEF]mmdd hh:mm:ss.uuuuuu threadid file:line] msg\n") - n, err := sb.file.Write(buf.Bytes()) - sb.nbytes += uint64(n) - return err -} - -// bufferSize sizes the buffer associated with each log file. It's large -// so that log records can accumulate without the logging thread blocking -// on disk I/O. The flushDaemon will block instead. -const bufferSize = 256 * 1024 - -// createFiles creates all the log files for severity from sev down to infoLog. -// l.mu is held. -func (l *loggingT) createFiles(sev severity) error { - now := time.Now() - // Files are created in decreasing severity order, so as soon as we find one - // has already been created, we can stop. - for s := sev; s >= infoLog && l.file[s] == nil; s-- { - sb := &syncBuffer{ - logger: l, - sev: s, - } - if err := sb.rotateFile(now); err != nil { - return err - } - l.file[s] = sb - } - return nil -} - -const flushInterval = 30 * time.Second - -// flushDaemon periodically flushes the log file buffers. -func (l *loggingT) flushDaemon() { - for _ = range time.NewTicker(flushInterval).C { - l.lockAndFlushAll() - } -} - -// lockAndFlushAll is like flushAll but locks l.mu first. -func (l *loggingT) lockAndFlushAll() { - l.mu.Lock() - l.flushAll() - l.mu.Unlock() -} - -// flushAll flushes all the logs and attempts to "sync" their data to disk. -// l.mu is held. -func (l *loggingT) flushAll() { - // Flush from fatal down, in case there's trouble flushing. - for s := fatalLog; s >= infoLog; s-- { - file := l.file[s] - if file != nil { - file.Flush() // ignore error - file.Sync() // ignore error - } - } -} - -// CopyStandardLogTo arranges for messages written to the Go "log" package's -// default logs to also appear in the Google logs for the named and lower -// severities. Subsequent changes to the standard log's default output location -// or format may break this behavior. -// -// Valid names are "INFO", "WARNING", "ERROR", and "FATAL". If the name is not -// recognized, CopyStandardLogTo panics. -func CopyStandardLogTo(name string) { - sev, ok := severityByName(name) - if !ok { - panic(fmt.Sprintf("log.CopyStandardLogTo(%q): unrecognized severity name", name)) - } - // Set a log format that captures the user's file and line: - // d.go:23: message - stdLog.SetFlags(stdLog.Lshortfile) - stdLog.SetOutput(logBridge(sev)) -} - -// logBridge provides the Write method that enables CopyStandardLogTo to connect -// Go's standard logs to the logs provided by this package. -type logBridge severity - -// Write parses the standard logging line and passes its components to the -// logger for severity(lb). -func (lb logBridge) Write(b []byte) (n int, err error) { - var ( - file = "???" - line = 1 - text string - ) - // Split "d.go:23: message" into "d.go", "23", and "message". - if parts := bytes.SplitN(b, []byte{':'}, 3); len(parts) != 3 || len(parts[0]) < 1 || len(parts[2]) < 1 { - text = fmt.Sprintf("bad log format: %s", b) - } else { - file = string(parts[0]) - text = string(parts[2][1:]) // skip leading space - line, err = strconv.Atoi(string(parts[1])) - if err != nil { - text = fmt.Sprintf("bad line number: %s", b) - line = 1 - } - } - // printWithFileLine with alsoToStderr=true, so standard log messages - // always appear on standard error. - logging.printWithFileLine(severity(lb), file, line, true, text) - return len(b), nil -} - -// setV computes and remembers the V level for a given PC -// when vmodule is enabled. -// File pattern matching takes the basename of the file, stripped -// of its .go suffix, and uses filepath.Match, which is a little more -// general than the *? matching used in C++. -// l.mu is held. -func (l *loggingT) setV(pc uintptr) Level { - fn := runtime.FuncForPC(pc) - file, _ := fn.FileLine(pc) - // The file is something like /a/b/c/d.go. We want just the d. - if strings.HasSuffix(file, ".go") { - file = file[:len(file)-3] - } - if slash := strings.LastIndex(file, "/"); slash >= 0 { - file = file[slash+1:] - } - for _, filter := range l.vmodule.filter { - if filter.match(file) { - l.vmap[pc] = filter.level - return filter.level - } - } - l.vmap[pc] = 0 - return 0 -} - -// Verbose is a boolean type that implements Infof (like Printf) etc. -// See the documentation of V for more information. -type Verbose bool - -// V reports whether verbosity at the call site is at least the requested level. -// The returned value is a boolean of type Verbose, which implements Info, Infoln -// and Infof. These methods will write to the Info log if called. -// Thus, one may write either -// if glog.V(2) { glog.Info("log this") } -// or -// glog.V(2).Info("log this") -// The second form is shorter but the first is cheaper if logging is off because it does -// not evaluate its arguments. -// -// Whether an individual call to V generates a log record depends on the setting of -// the -v and --vmodule flags; both are off by default. If the level in the call to -// V is at least the value of -v, or of -vmodule for the source file containing the -// call, the V call will log. -func V(level Level) Verbose { - // This function tries hard to be cheap unless there's work to do. - // The fast path is two atomic loads and compares. - - // Here is a cheap but safe test to see if V logging is enabled globally. - if logging.verbosity.get() >= level { - return Verbose(true) - } - - // It's off globally but it vmodule may still be set. - // Here is another cheap but safe test to see if vmodule is enabled. - if atomic.LoadInt32(&logging.filterLength) > 0 { - // Now we need a proper lock to use the logging structure. The pcs field - // is shared so we must lock before accessing it. This is fairly expensive, - // but if V logging is enabled we're slow anyway. - logging.mu.Lock() - defer logging.mu.Unlock() - if runtime.Callers(2, logging.pcs[:]) == 0 { - return Verbose(false) - } - v, ok := logging.vmap[logging.pcs[0]] - if !ok { - v = logging.setV(logging.pcs[0]) - } - return Verbose(v >= level) - } - return Verbose(false) -} - -// Info is equivalent to the global Info function, guarded by the value of v. -// See the documentation of V for usage. -func (v Verbose) Info(args ...interface{}) { - if v { - logging.print(infoLog, args...) - } -} - -// Infoln is equivalent to the global Infoln function, guarded by the value of v. -// See the documentation of V for usage. -func (v Verbose) Infoln(args ...interface{}) { - if v { - logging.println(infoLog, args...) - } -} - -// Infof is equivalent to the global Infof function, guarded by the value of v. -// See the documentation of V for usage. -func (v Verbose) Infof(format string, args ...interface{}) { - if v { - logging.printf(infoLog, format, args...) - } -} - -// Info logs to the INFO log. -// Arguments are handled in the manner of fmt.Print; a newline is appended if missing. -func Info(args ...interface{}) { - logging.print(infoLog, args...) -} - -// InfoDepth acts as Info but uses depth to determine which call frame to log. -// InfoDepth(0, "msg") is the same as Info("msg"). -func InfoDepth(depth int, args ...interface{}) { - logging.printDepth(infoLog, depth, args...) -} - -// Infoln logs to the INFO log. -// Arguments are handled in the manner of fmt.Println; a newline is appended if missing. -func Infoln(args ...interface{}) { - logging.println(infoLog, args...) -} - -// Infof logs to the INFO log. -// Arguments are handled in the manner of fmt.Printf; a newline is appended if missing. -func Infof(format string, args ...interface{}) { - logging.printf(infoLog, format, args...) -} - -// Warning logs to the WARNING and INFO logs. -// Arguments are handled in the manner of fmt.Print; a newline is appended if missing. -func Warning(args ...interface{}) { - logging.print(warningLog, args...) -} - -// WarningDepth acts as Warning but uses depth to determine which call frame to log. -// WarningDepth(0, "msg") is the same as Warning("msg"). -func WarningDepth(depth int, args ...interface{}) { - logging.printDepth(warningLog, depth, args...) -} - -// Warningln logs to the WARNING and INFO logs. -// Arguments are handled in the manner of fmt.Println; a newline is appended if missing. -func Warningln(args ...interface{}) { - logging.println(warningLog, args...) -} - -// Warningf logs to the WARNING and INFO logs. -// Arguments are handled in the manner of fmt.Printf; a newline is appended if missing. -func Warningf(format string, args ...interface{}) { - logging.printf(warningLog, format, args...) -} - -// Error logs to the ERROR, WARNING, and INFO logs. -// Arguments are handled in the manner of fmt.Print; a newline is appended if missing. -func Error(args ...interface{}) { - logging.print(errorLog, args...) -} - -// ErrorDepth acts as Error but uses depth to determine which call frame to log. -// ErrorDepth(0, "msg") is the same as Error("msg"). -func ErrorDepth(depth int, args ...interface{}) { - logging.printDepth(errorLog, depth, args...) -} - -// Errorln logs to the ERROR, WARNING, and INFO logs. -// Arguments are handled in the manner of fmt.Println; a newline is appended if missing. -func Errorln(args ...interface{}) { - logging.println(errorLog, args...) -} - -// Errorf logs to the ERROR, WARNING, and INFO logs. -// Arguments are handled in the manner of fmt.Printf; a newline is appended if missing. -func Errorf(format string, args ...interface{}) { - logging.printf(errorLog, format, args...) -} - -// Fatal logs to the FATAL, ERROR, WARNING, and INFO logs, -// including a stack trace of all running goroutines, then calls os.Exit(255). -// Arguments are handled in the manner of fmt.Print; a newline is appended if missing. -func Fatal(args ...interface{}) { - logging.print(fatalLog, args...) -} - -// FatalDepth acts as Fatal but uses depth to determine which call frame to log. -// FatalDepth(0, "msg") is the same as Fatal("msg"). -func FatalDepth(depth int, args ...interface{}) { - logging.printDepth(fatalLog, depth, args...) -} - -// Fatalln logs to the FATAL, ERROR, WARNING, and INFO logs, -// including a stack trace of all running goroutines, then calls os.Exit(255). -// Arguments are handled in the manner of fmt.Println; a newline is appended if missing. -func Fatalln(args ...interface{}) { - logging.println(fatalLog, args...) -} - -// Fatalf logs to the FATAL, ERROR, WARNING, and INFO logs, -// including a stack trace of all running goroutines, then calls os.Exit(255). -// Arguments are handled in the manner of fmt.Printf; a newline is appended if missing. -func Fatalf(format string, args ...interface{}) { - logging.printf(fatalLog, format, args...) -} - -// fatalNoStacks is non-zero if we are to exit without dumping goroutine stacks. -// It allows Exit and relatives to use the Fatal logs. -var fatalNoStacks uint32 - -// Exit logs to the FATAL, ERROR, WARNING, and INFO logs, then calls os.Exit(1). -// Arguments are handled in the manner of fmt.Print; a newline is appended if missing. -func Exit(args ...interface{}) { - atomic.StoreUint32(&fatalNoStacks, 1) - logging.print(fatalLog, args...) -} - -// ExitDepth acts as Exit but uses depth to determine which call frame to log. -// ExitDepth(0, "msg") is the same as Exit("msg"). -func ExitDepth(depth int, args ...interface{}) { - atomic.StoreUint32(&fatalNoStacks, 1) - logging.printDepth(fatalLog, depth, args...) -} - -// Exitln logs to the FATAL, ERROR, WARNING, and INFO logs, then calls os.Exit(1). -func Exitln(args ...interface{}) { - atomic.StoreUint32(&fatalNoStacks, 1) - logging.println(fatalLog, args...) -} - -// Exitf logs to the FATAL, ERROR, WARNING, and INFO logs, then calls os.Exit(1). -// Arguments are handled in the manner of fmt.Printf; a newline is appended if missing. -func Exitf(format string, args ...interface{}) { - atomic.StoreUint32(&fatalNoStacks, 1) - logging.printf(fatalLog, format, args...) -} diff --git a/vendor/github.com/golang/glog/glog_file.go b/vendor/github.com/golang/glog/glog_file.go deleted file mode 100644 index 65075d281..000000000 --- a/vendor/github.com/golang/glog/glog_file.go +++ /dev/null @@ -1,124 +0,0 @@ -// Go support for leveled logs, analogous to https://code.google.com/p/google-glog/ -// -// Copyright 2013 Google Inc. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// File I/O for logs. - -package glog - -import ( - "errors" - "flag" - "fmt" - "os" - "os/user" - "path/filepath" - "strings" - "sync" - "time" -) - -// MaxSize is the maximum size of a log file in bytes. -var MaxSize uint64 = 1024 * 1024 * 1800 - -// logDirs lists the candidate directories for new log files. -var logDirs []string - -// If non-empty, overrides the choice of directory in which to write logs. -// See createLogDirs for the full list of possible destinations. -var logDir = flag.String("log_dir", "", "If non-empty, write log files in this directory") - -func createLogDirs() { - if *logDir != "" { - logDirs = append(logDirs, *logDir) - } - logDirs = append(logDirs, os.TempDir()) -} - -var ( - pid = os.Getpid() - program = filepath.Base(os.Args[0]) - host = "unknownhost" - userName = "unknownuser" -) - -func init() { - h, err := os.Hostname() - if err == nil { - host = shortHostname(h) - } - - current, err := user.Current() - if err == nil { - userName = current.Username - } - - // Sanitize userName since it may contain filepath separators on Windows. - userName = strings.Replace(userName, `\`, "_", -1) -} - -// shortHostname returns its argument, truncating at the first period. -// For instance, given "www.google.com" it returns "www". -func shortHostname(hostname string) string { - if i := strings.Index(hostname, "."); i >= 0 { - return hostname[:i] - } - return hostname -} - -// logName returns a new log file name containing tag, with start time t, and -// the name for the symlink for tag. -func logName(tag string, t time.Time) (name, link string) { - name = fmt.Sprintf("%s.%s.%s.log.%s.%04d%02d%02d-%02d%02d%02d.%d", - program, - host, - userName, - tag, - t.Year(), - t.Month(), - t.Day(), - t.Hour(), - t.Minute(), - t.Second(), - pid) - return name, program + "." + tag -} - -var onceLogDirs sync.Once - -// create creates a new log file and returns the file and its filename, which -// contains tag ("INFO", "FATAL", etc.) and t. If the file is created -// successfully, create also attempts to update the symlink for that tag, ignoring -// errors. -func create(tag string, t time.Time) (f *os.File, filename string, err error) { - onceLogDirs.Do(createLogDirs) - if len(logDirs) == 0 { - return nil, "", errors.New("log: no log dirs") - } - name, link := logName(tag, t) - var lastErr error - for _, dir := range logDirs { - fname := filepath.Join(dir, name) - f, err := os.Create(fname) - if err == nil { - symlink := filepath.Join(dir, link) - os.Remove(symlink) // ignore err - os.Symlink(name, symlink) // ignore err - return f, fname, nil - } - lastErr = err - } - return nil, "", fmt.Errorf("log: cannot create log: %v", lastErr) -} diff --git a/vendor/github.com/kubernetes-csi/drivers/LICENSE b/vendor/github.com/kubernetes-csi/drivers/LICENSE deleted file mode 100644 index 261eeb9e9..000000000 --- a/vendor/github.com/kubernetes-csi/drivers/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go b/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go index 91fd4ed4f..a60a7c041 100644 --- a/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go +++ b/vendor/k8s.io/apimachinery/pkg/runtime/serializer/streaming/streaming.go @@ -64,7 +64,7 @@ func NewDecoder(r io.ReadCloser, d runtime.Decoder) Decoder { reader: r, decoder: d, buf: make([]byte, 1024), - maxBytes: 1024 * 1024, + maxBytes: 16 * 1024 * 1024, } }