mirrors/ceph-csi
1
0
Fork 0
mirror of https://github.com/ceph/ceph-csi.git synced 2024-11-09 16:00:22 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #297 from Madhu-1/master

Browse Source 
merge csiv1.0 branch to master
This commit is contained in:
mergify[bot] 2019-04-03 19:14:21 +00:00 committed by GitHub
commit 426bf67966
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1033 changed files with 134885 additions and 32339 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
.github/ISSUE_TEMPLATE/bug_report.md vendored
View File

@ -1,23 +0,0 @@
---
name: Bug report
about: Create a report to help us improve
title: ''
labels: ''
assignees: ''
---
**Describe the bug**
A clear and concise description of what the bug is.
**Kubernetes and Ceph CSI Versions**
Kubernetes: 1.12 and below or 1.13 and above
Ceph CSI: 0.3.0 or 1.0.0
**Ceph CSI Driver logs**
Post logs from rbdplugin/cephfs plugin, provisioner, etc.
**To Reproduce**
**Expected behavior**
A clear and concise description of what you expected to happen.

20
.github/ISSUE_TEMPLATE/feature_request.md vendored
View File

@ -1,20 +0,0 @@
---
name: Feature request
about: Suggest an idea for this project
title: ''
labels: ''
assignees: ''
---
**Is your feature request related to a problem? Please describe.**
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
**Describe the solution you'd like**
A clear and concise description of what you want to happen.
**Describe alternatives you've considered**
A clear and concise description of any alternative solutions or features you've considered.
**Additional context**
Add any other context or screenshots about the feature request here.

1
.mergify.yml
View File

@ -1,3 +1,4 @@
---
pull_request_rules:
- name: automatic merge
conditions:

53
.travis.yml
View File

@ -1,24 +1,55 @@
---
# need for docker build
sudo: true
addons:
apt:
packages:
- realpath
- ruby
language: go
branches:
only:
- csi-v0.3
- master
- csi-v1.0 # remove this once csi-v1.0 becomes master
go: 1.9.x
go: 1.11.x
before_script:
- GO_FILES=$(find . -iname '*.go' -type f | grep -v /vendor/)
- go get -u golang.org/x/lint/golint #go get github.com/golang/lint/golint
env:
global:
- GOLANGCI_VERSION="v1.15.0"
- TEST_COVERAGE=stdout
- GO_METALINTER_THREADS=1
- GO_COVER_DIR=_output
script:
- test -z $(gofmt -s -l $GO_FILES)
- go vet -v $(go list ./... | grep -v /vendor/)
- make rbdplugin
- make cephfsplugin
jobs:
include:
- name: Linter
install:
- gem install mdl
- pip install --user --upgrade pip
- pip install --user yamllint
# install golangci-lint
- curl -sf
"https://install.goreleaser.com/github.com/golangci/golangci-lint.sh"
| bash -s -- -b $GOPATH/bin "${GOLANGCI_VERSION}"
script:
- scripts/lint-text.sh --require-all
- scripts/lint-go.sh
- scripts/test-go.sh
- name: rbdplugin
script:
- make rbdplugin
- name: cephfsplugin
script:
- make cephfsplugin
deploy:
- provider: script
script:
- ./deploy.sh
on: # yamllint disable-line rule:truthy
all_branches: true
script: ./deploy.sh

283
Gopkg.lock generated
View File

@ -2,145 +2,217 @@
[[projects]]
digest = "1:94ffc0947c337d618b6ff5ed9abaddc1217b090c1b3a1ae4739b35b7b25851d5"
name = "github.com/container-storage-interface/spec"
packages = ["lib/go/csi/v0"]
revision = "2178fdeea87f1150a17a63252eee28d4d8141f72"
version = "v0.3.0"
[[projects]]
name = "github.com/ghodss/yaml"
packages = ["."]
revision = "0ca9ea5df5451ffdf184b4428c902747c2c11cd7"
packages = ["lib/go/csi"]
pruneopts = "NUT"
revision = "ed0bb0e1557548aa028307f48728767cfe8f6345"
version = "v1.0.0"
[[projects]]
digest = "1:abea725bcf0210887f5da19d804fffa1dd45a42a56bdf5f02322345e3fee4f0d"
name = "github.com/gogo/protobuf"
packages = [
"proto",
"sortkeys"
"sortkeys",
]
pruneopts = "NUT"
revision = "4cbf7e384e768b4e01799441fdf2a706a5635ae7"
version = "v1.2.0"
[[projects]]
branch = "master"
name = "github.com/golang/glog"
packages = ["."]
revision = "23def4e6c14b4da8ac2ed8007337bc5eb5007998"
[[projects]]
digest = "1:bff0ce7c8e3d6357fa5a8549bbe4bdb620bddc13c11ae569aa7248ea92e2139f"
name = "github.com/golang/protobuf"
packages = [
"descriptor",
"proto",
"protoc-gen-go/descriptor",
"ptypes",
"ptypes/any",
"ptypes/duration",
"ptypes/timestamp",
"ptypes/wrappers"
"ptypes/wrappers",
]
revision = "b4deda0973fb4c70b50d226b1af49f3da59f5265"
version = "v1.1.0"
pruneopts = "NUT"
revision = "aa810b61a9c79d51363740d207bb46cf8e620ed5"
version = "v1.2.0"
[[projects]]
branch = "master"
digest = "1:05f95ffdfcf651bdb0f05b40b69e7f5663047f8da75c72d58728acb59b5cc107"
name = "github.com/google/btree"
packages = ["."]
pruneopts = "NUT"
revision = "4030bb1f1f0c35b30ca7009e9ebd06849dd45306"
[[projects]]
branch = "master"
digest = "1:52c5834e2bebac9030c97cc0798ac11c3aa8a39f098aeb419f142533da6cd3cc"
name = "github.com/google/gofuzz"
packages = ["."]
pruneopts = "NUT"
revision = "24818f796faf91cd76ec7bddd72458fbced7a6c1"
[[projects]]
digest = "1:56a1f3949ebb7fa22fa6b4e4ac0fe0f77cc4faee5b57413e6fa9199a8458faf1"
name = "github.com/google/uuid"
packages = ["."]
pruneopts = "NUT"
revision = "9b3b1e0f5f99ae461456d768e7d301a7acdaa2d8"
version = "v1.1.0"
[[projects]]
digest = "1:06a7dadb7b760767341ffb6c8d377238d68a1226f2b21b5d497d2e3f6ecf6b4e"
name = "github.com/googleapis/gnostic"
packages = [
"OpenAPIv2",
"compiler",
"extensions"
"extensions",
]
pruneopts = "NUT"
revision = "7c663266750e7d82587642f65e60bc4083f1f84e"
version = "v0.2.0"
[[projects]]
branch = "master"
name = "github.com/howeyc/gopass"
packages = ["."]
revision = "bf9dde6d0d2c004a008c27aaee91170c786f6db8"
digest = "1:7fdf3223c7372d1ced0b98bf53457c5e89d89aecbad9a77ba9fcc6e01f9e5621"
name = "github.com/gregjones/httpcache"
packages = [
".",
"diskcache",
]
pruneopts = "NUT"
revision = "c63ab54fda8f77302f8d414e19933f2b6026a089"
[[projects]]
digest = "1:9a52adf44086cead3b384e5d0dbf7a1c1cce65e67552ee3383a8561c42a18cd3"
name = "github.com/imdario/mergo"
packages = ["."]
pruneopts = "NUT"
revision = "9f23e2d6bd2a77f959b2bf6acdbefd708a83a4a4"
version = "v0.3.6"
[[projects]]
digest = "1:8e36686e8b139f8fe240c1d5cf3a145bc675c22ff8e707857cdd3ae17b00d728"
name = "github.com/json-iterator/go"
packages = ["."]
pruneopts = "NUT"
revision = "1624edc4454b8682399def8740d46db5e4362ba4"
version = "v1.1.5"
[[projects]]
branch = "master"
name = "github.com/kubernetes-csi/drivers"
packages = ["pkg/csi-common"]
revision = "d8f283cd941f1e24e20e62653476a3722633f43e"
digest = "1:2b060bb1a39127e592baf9ab62ec1e94100dc22107f915183f3cd1f6d1cd579a"
name = "github.com/kubernetes-csi/csi-lib-utils"
packages = ["protosanitizer"]
pruneopts = "NUT"
revision = "5853414e1d4771302e0df10d1870c444c2135799"
version = "v0.2.0"
[[projects]]
digest = "1:2f42fa12d6911c7b7659738758631bec870b7e9b4c6be5444f963cdcfccc191f"
name = "github.com/modern-go/concurrent"
packages = ["."]
pruneopts = "NUT"
revision = "bacd9c7ef1dd9b15be4a9909b8ac7a4e313eec94"
version = "1.0.3"
[[projects]]
digest = "1:c6aca19413b13dc59c220ad7430329e2ec454cc310bc6d8de2c7e2b93c18a0f6"
name = "github.com/modern-go/reflect2"
packages = ["."]
pruneopts = "NUT"
revision = "4b7aa43c6742a2c18fdef89dd197aaae7dac7ccd"
version = "1.0.1"
[[projects]]
digest = "1:93b1d84c5fa6d1ea52f4114c37714cddd84d5b78f151b62bb101128dd51399bf"
name = "github.com/pborman/uuid"
packages = ["."]
revision = "e790cca94e6cc75c7064b1332e63811d4aae1a53"
version = "v1.1"
pruneopts = "NUT"
revision = "adf5a7427709b9deb95d29d3fa8a2bf9cfd388f1"
version = "v1.2"
[[projects]]
branch = "master"
digest = "1:3bf17a6e6eaa6ad24152148a631d18662f7212e21637c2699bff3369b7f00fa2"
name = "github.com/petar/GoLLRB"
packages = ["llrb"]
pruneopts = "NUT"
revision = "53be0d36a84c2a886ca057d34b6aa4468df9ccb4"
[[projects]]
digest = "1:6c6d91dc326ed6778783cff869c49fb2f61303cdd2ebbcf90abe53505793f3b6"
name = "github.com/peterbourgon/diskv"
packages = ["."]
pruneopts = "NUT"
revision = "5f041e8faa004a95c88a202771f4cc3e991971e6"
version = "v2.0.1"
[[projects]]
digest = "1:14715f705ff5dfe0ffd6571d7d201dd8e921030f8070321a79380d8ca4ec1a24"
name = "github.com/pkg/errors"
packages = ["."]
revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
version = "v0.8.0"
pruneopts = "NUT"
revision = "ba968bfe8b2f7e042a574c888954fccecfa385b4"
version = "v0.8.1"
[[projects]]
digest = "1:9d8420bbf131d1618bde6530af37c3799340d3762cc47210c1d9532a4c3a2779"
name = "github.com/spf13/pflag"
packages = ["."]
pruneopts = "NUT"
revision = "298182f68c66c05229eb03ac171abe6e309ee79a"
version = "v1.0.3"
[[projects]]
branch = "master"
digest = "1:38f553aff0273ad6f367cb0a0f8b6eecbaef8dc6cb8b50e57b6a81c1d5b1e332"
name = "golang.org/x/crypto"
packages = ["ssh/terminal"]
revision = "505ab145d0a99da450461ae2c1a9f6cd10d1f447"
pruneopts = "NUT"
revision = "ff983b9c42bc9fbf91556e191cc8efb585c16908"
[[projects]]
branch = "master"
digest = "1:d4e37d487310720926343302a747f3f9e8f020e5fe961190c57ce437159a6e63"
name = "golang.org/x/net"
packages = [
"context",
"context/ctxhttp",
"http/httpguts",
"http2",
"http2/hpack",
"idna",
"internal/timeseries",
"trace"
"trace",
]
revision = "3673e40ba22529d22c3fd7c93e97b0ce50fa7bdd"
pruneopts = "NUT"
revision = "915654e7eabcea33ae277abbecf52f0d8b7a9fdc"
[[projects]]
branch = "master"
digest = "1:293b8e3359faf71cc5f85f3c144115ca6540396f4e1ff0fa30cd014c04258c30"
name = "golang.org/x/oauth2"
packages = [
".",
"internal",
]
pruneopts = "NUT"
revision = "36a7019397c4c86cf59eeab3bc0d188bac444277"
[[projects]]
branch = "master"
digest = "1:0f0298002380ddbc31230516fc1dc354ff466e607823e9122d69cce79310bdc9"
name = "golang.org/x/sys"
packages = [
"unix",
"windows"
"windows",
]
revision = "e072cadbbdc8dd3d3ffa82b8b4b9304c261d9311"
pruneopts = "NUT"
revision = "a457fd036447854c0c02e89ea439481bdcf941a2"
[[projects]]
digest = "1:e7071ed636b5422cc51c0e3a6cebc229d6c9fffc528814b519a980641422d619"
name = "golang.org/x/text"
packages = [
"collate",
@ -156,40 +228,69 @@
"unicode/bidi",
"unicode/cldr",
"unicode/norm",
"unicode/rangetable"
"unicode/rangetable",
]
pruneopts = "NUT"
revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0"
version = "v0.3.0"
[[projects]]
branch = "master"
digest = "1:9fdc2b55e8e0fafe4b41884091e51e77344f7dc511c5acedcfd98200003bff90"
name = "golang.org/x/time"
packages = ["rate"]
pruneopts = "NUT"
revision = "85acf8d2951cb2a3bde7632f9ff273ef0379bcbd"
[[projects]]
branch = "master"
name = "google.golang.org/genproto"
packages = ["googleapis/rpc/status"]
revision = "2a72893556e4d1f6c795a4c039314c9fa751eedb"
digest = "1:34c10243da5972105edd1b4b883e2bd918fbb3f73fbe14d6af6929e547173494"
name = "google.golang.org/appengine"
packages = [
"internal",
"internal/base",
"internal/datastore",
"internal/log",
"internal/remote_api",
"internal/urlfetch",
"urlfetch",
]
pruneopts = "NUT"
revision = "e9657d882bb81064595ca3b56cbe2546bbabf7b1"
version = "v1.4.0"
[[projects]]
branch = "master"
digest = "1:077c1c599507b3b3e9156d17d36e1e61928ee9b53a5b420f10f28ebd4a0b275c"
name = "google.golang.org/genproto"
packages = ["googleapis/rpc/status"]
pruneopts = "NUT"
revision = "db91494dd46c1fdcbbde05e5ff5eb56df8f7d79a"
[[projects]]
digest = "1:638e6e596d67d0a0c8aeb76ebdcf73561b701ea43f21963b1db231d96ed7db68"
name = "google.golang.org/grpc"
packages = [
".",
"balancer",
"balancer/base",
"balancer/roundrobin",
"binarylog/grpc_binarylog_v1",
"codes",
"connectivity",
"credentials",
"credentials/internal",
"encoding",
"encoding/proto",
"grpclog",
"internal",
"internal/backoff",
"internal/binarylog",
"internal/channelz",
"internal/envconfig",
"internal/grpcrand",
"internal/grpcsync",
"internal/syscall",
"internal/transport",
"keepalive",
"metadata",
"naming",
@ -200,24 +301,29 @@
"stats",
"status",
"tap",
"transport"
]
revision = "168a6198bcb0ef175f7dacec0b8691fc141dc9b8"
version = "v1.13.0"
pruneopts = "NUT"
revision = "df014850f6dee74ba2fc94874043a9f3f75fbfd8"
version = "v1.17.0"
[[projects]]
digest = "1:2d1fbdc6777e5408cabeb02bf336305e724b925ff4546ded0fa8715a7267922a"
name = "gopkg.in/inf.v0"
packages = ["."]
pruneopts = "NUT"
revision = "d2d2541c53f18d2a059457998ce2876cc8e67cbf"
version = "v0.9.1"
[[projects]]
digest = "1:18108594151654e9e696b27b181b953f9a90b16bf14d253dd1b397b025a1487f"
name = "gopkg.in/yaml.v2"
packages = ["."]
pruneopts = "NUT"
revision = "51d6538a90f86fe93ac480b35f37b2be17fef232"
version = "v2.2.2"
[[projects]]
digest = "1:c453ddc26bdab1e4267683a588ad9046e48d803a73f124fe2927adbab6ff02a5"
name = "k8s.io/api"
packages = [
"admissionregistration/v1alpha1",
@ -225,16 +331,19 @@
"apps/v1",
"apps/v1beta1",
"apps/v1beta2",
"auditregistration/v1alpha1",
"authentication/v1",
"authentication/v1beta1",
"authorization/v1",
"authorization/v1beta1",
"autoscaling/v1",
"autoscaling/v2beta1",
"autoscaling/v2beta2",
"batch/v1",
"batch/v1beta1",
"batch/v2alpha1",
"certificates/v1beta1",
"coordination/v1beta1",
"core/v1",
"events/v1beta1",
"extensions/v1beta1",
@ -244,15 +353,18 @@
"rbac/v1alpha1",
"rbac/v1beta1",
"scheduling/v1alpha1",
"scheduling/v1beta1",
"settings/v1alpha1",
"storage/v1",
"storage/v1alpha1",
"storage/v1beta1"
"storage/v1beta1",
]
revision = "7aac3e00a1b32fa476b83078cebaaca606b2fb48"
version = "kubernetes-1.10.0-beta.1"
pruneopts = "NUT"
revision = "74b699b93c15473932b89e3d1818ba8282f3b5ab"
version = "kubernetes-1.13.3"
[[projects]]
digest = "1:09dee8b7c6cb2fc9c6bee525de3b95199a82a8647a189e153d072a1dfce17de7"
name = "k8s.io/apimachinery"
packages = [
"pkg/api/errors",
@ -280,6 +392,7 @@
"pkg/util/framer",
"pkg/util/intstr",
"pkg/util/json",
"pkg/util/naming",
"pkg/util/net",
"pkg/util/runtime",
"pkg/util/sets",
@ -289,12 +402,14 @@
"pkg/util/yaml",
"pkg/version",
"pkg/watch",
"third_party/forked/golang/reflect"
"third_party/forked/golang/reflect",
]
revision = "302974c03f7e50f16561ba237db776ab93594ef6"
version = "kubernetes-1.10.0-beta.1"
pruneopts = "NUT"
revision = "572dfc7bdfcb4531361a17d27b92851f59acf0dc"
version = "kubernetes-1.13.3"
[[projects]]
digest = "1:638623327cb201b425a328d0bddb3379b05eb05ef4cab589380f0be07ac1dc17"
name = "k8s.io/client-go"
packages = [
"discovery",
@ -305,16 +420,19 @@
"kubernetes/typed/apps/v1",
"kubernetes/typed/apps/v1beta1",
"kubernetes/typed/apps/v1beta2",
"kubernetes/typed/auditregistration/v1alpha1",
"kubernetes/typed/authentication/v1",
"kubernetes/typed/authentication/v1beta1",
"kubernetes/typed/authorization/v1",
"kubernetes/typed/authorization/v1beta1",
"kubernetes/typed/autoscaling/v1",
"kubernetes/typed/autoscaling/v2beta1",
"kubernetes/typed/autoscaling/v2beta2",
"kubernetes/typed/batch/v1",
"kubernetes/typed/batch/v1beta1",
"kubernetes/typed/batch/v2alpha1",
"kubernetes/typed/certificates/v1beta1",
"kubernetes/typed/coordination/v1beta1",
"kubernetes/typed/core/v1",
"kubernetes/typed/events/v1beta1",
"kubernetes/typed/extensions/v1beta1",
@ -324,11 +442,16 @@
"kubernetes/typed/rbac/v1alpha1",
"kubernetes/typed/rbac/v1beta1",
"kubernetes/typed/scheduling/v1alpha1",
"kubernetes/typed/scheduling/v1beta1",
"kubernetes/typed/settings/v1alpha1",
"kubernetes/typed/storage/v1",
"kubernetes/typed/storage/v1alpha1",
"kubernetes/typed/storage/v1beta1",
"pkg/apis/clientauthentication",
"pkg/apis/clientauthentication/v1alpha1",
"pkg/apis/clientauthentication/v1beta1",
"pkg/version",
"plugin/pkg/client/auth/exec",
"rest",
"rest/watch",
"tools/auth",
@ -340,34 +463,80 @@
"tools/reference",
"transport",
"util/cert",
"util/connrotation",
"util/flowcontrol",
"util/homedir",
"util/integer"
"util/integer",
]
revision = "82eadfdc39007c2eb47e3ddeb7ed7d96365e409d"
version = "kubernetes-1.10.0-beta.1"
pruneopts = "NUT"
revision = "6e4752048fde21176ab35eb54ec1117359830d8a"
version = "kubernetes-1.13.3"
[[projects]]
digest = "1:9cc257b3c9ff6a0158c9c661ab6eebda1fe8a4a4453cd5c4044dc9a2ebfb992b"
name = "k8s.io/klog"
packages = ["."]
pruneopts = "NUT"
revision = "a5bc97fbc634d635061f3146511332c7e313a55a"
version = "v0.1.0"
[[projects]]
digest = "1:cf54450b967dcae4f270dfa44395b01175e5358bbd79fe8a6073b13f220f1c2e"
name = "k8s.io/kubernetes"
packages = [
"pkg/util/file",
"pkg/util/io",
"pkg/util/keymutex",
"pkg/util/mount",
"pkg/util/nsenter"
"pkg/util/nsenter",
]
revision = "b1b29978270dc22fecc592ac55d903350454310a"
version = "v1.11.1"
pruneopts = "NUT"
revision = "721bfa751924da8d1680787490c54b9179b1fed0"
version = "v1.13.3"
[[projects]]
branch = "master"
digest = "1:381323c2fe2e890a3dd3b5d6dc6f2199068408cca89b24f6b7ca1c60f32644a5"
name = "k8s.io/utils"
packages = ["exec"]
revision = "66066c83e385e385ccc3c964b44fd7dcd413d0ed"
pruneopts = "NUT"
revision = "8a16e7dd8fb6d97d1331b0c79a16722f934b00b1"
[[projects]]
digest = "1:8730e0150dfb2b7e173890c8b9868e7a273082ef8e39f4940e3506a481cf895c"
name = "sigs.k8s.io/yaml"
packages = ["."]
pruneopts = "NUT"
revision = "fd68e9863619f6ec2fdd8625fe1f02e7c877e480"
version = "v1.1.0"
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
inputs-digest = "6e997f1f66bb80e4e145da05f1872070947725cba04d3da2521bcdc0e33d7150"
input-imports = [
"github.com/container-storage-interface/spec/lib/go/csi",
"github.com/golang/protobuf/ptypes",
"github.com/golang/protobuf/ptypes/timestamp",
"github.com/kubernetes-csi/csi-lib-utils/protosanitizer",
"github.com/pborman/uuid",
"github.com/pkg/errors",
"golang.org/x/net/context",
"google.golang.org/grpc",
"google.golang.org/grpc/codes",
"google.golang.org/grpc/status",
"k8s.io/api/core/v1",
"k8s.io/apimachinery/pkg/api/errors",
"k8s.io/apimachinery/pkg/apis/meta/v1",
"k8s.io/apimachinery/pkg/util/sets",
"k8s.io/apimachinery/pkg/util/wait",
"k8s.io/client-go/kubernetes",
"k8s.io/client-go/rest",
"k8s.io/client-go/tools/clientcmd",
"k8s.io/klog",
"k8s.io/kubernetes/pkg/util/keymutex",
"k8s.io/kubernetes/pkg/util/mount",
"k8s.io/kubernetes/pkg/util/nsenter",
"k8s.io/utils/exec",
]
solver-name = "gps-cdcl"
solver-version = 1

18
Gopkg.toml
View File

@ -1,14 +1,6 @@
[[constraint]]
name = "github.com/container-storage-interface/spec"
version = "~0.3.0"
[[constraint]]
branch = "master"
name = "github.com/kubernetes-csi/drivers"
[[constraint]]
branch = "master"
name = "github.com/golang/glog"
version = "~1.0.0"
[[override]]
revision = "5db89f0ca68677abc5eefce8f2a0a772c98ba52d"
@ -19,15 +11,15 @@
version = "1.10.0"
[[constraint]]
version = "kubernetes-1.10.0-beta.1"
version = "kubernetes-1.13.3"
name = "k8s.io/apimachinery"
[[constraint]]
name = "k8s.io/kubernetes"
version = "v1.10.0-beta.1"
version = "v1.13.3"
[[override]]
version = "kubernetes-1.10.0-beta.1"
version = "kubernetes-1.13.3"
name = "k8s.io/api"
[[override]]
@ -36,7 +28,7 @@
[[constraint]]
name = "k8s.io/client-go"
version = "kubernetes-1.10.0-beta.1"
version = "kubernetes-1.13.3"
[prune]
go-tests = true

33
Makefile
View File

@ -14,46 +14,53 @@
.PHONY: all rbdplugin cephfsplugin
CONTAINER_CMD?=docker
RBD_IMAGE_NAME=$(if $(ENV_RBD_IMAGE_NAME),$(ENV_RBD_IMAGE_NAME),quay.io/cephcsi/rbdplugin)
RBD_IMAGE_VERSION=$(if $(ENV_RBD_IMAGE_VERSION),$(ENV_RBD_IMAGE_VERSION),v0.3.0)
RBD_IMAGE_VERSION=$(if $(ENV_RBD_IMAGE_VERSION),$(ENV_RBD_IMAGE_VERSION),v1.0.0)
CEPHFS_IMAGE_NAME=$(if $(ENV_CEPHFS_IMAGE_NAME),$(ENV_CEPHFS_IMAGE_NAME),quay.io/cephcsi/cephfsplugin)
CEPHFS_IMAGE_VERSION=$(if $(ENV_CEPHFS_IMAGE_VERSION),$(ENV_CEPHFS_IMAGE_VERSION),v0.3.0)
CEPHFS_IMAGE_VERSION=$(if $(ENV_CEPHFS_IMAGE_VERSION),$(ENV_CEPHFS_IMAGE_VERSION),v1.0.0)
$(info rbd image settings: $(RBD_IMAGE_NAME) version $(RBD_IMAGE_VERSION))
$(info cephfs image settings: $(CEPHFS_IMAGE_NAME) version $(CEPHFS_IMAGE_VERSION))
all: rbdplugin cephfsplugin
test:
go test github.com/ceph/ceph-csi/pkg/... -cover
go vet github.com/ceph/ceph-csi/pkg/...
test: go-test static-check
go-test:
./scripts/test-go.sh
static-check:
./scripts/lint-go.sh
./scripts/lint-text.sh
rbdplugin:
if [ ! -d ./vendor ]; then dep ensure -vendor-only; fi
CGO_ENABLED=0 GOOS=linux go build -a -ldflags '-extldflags "-static"' -o _output/rbdplugin ./rbd
CGO_ENABLED=0 GOOS=linux go build -a -ldflags '-extldflags "-static"' -o _output/rbdplugin ./cmd/rbd
image-rbdplugin: rbdplugin
cp _output/rbdplugin deploy/rbd/docker
docker build -t $(RBD_IMAGE_NAME):$(RBD_IMAGE_VERSION) deploy/rbd/docker
$(CONTAINER_CMD) build -t $(RBD_IMAGE_NAME):$(RBD_IMAGE_VERSION) deploy/rbd/docker
cephfsplugin:
if [ ! -d ./vendor ]; then dep ensure -vendor-only; fi
CGO_ENABLED=0 GOOS=linux go build -a -ldflags '-extldflags "-static"' -o _output/cephfsplugin ./cephfs
CGO_ENABLED=0 GOOS=linux go build -a -ldflags '-extldflags "-static"' -o _output/cephfsplugin ./cmd/cephfs
image-cephfsplugin: cephfsplugin
cp _output/cephfsplugin deploy/cephfs/docker
docker build -t $(CEPHFS_IMAGE_NAME):$(CEPHFS_IMAGE_VERSION) deploy/cephfs/docker
$(CONTAINER_CMD) build -t $(CEPHFS_IMAGE_NAME):$(CEPHFS_IMAGE_VERSION) deploy/cephfs/docker
push-image-rbdplugin: image-rbdplugin
docker push $(RBD_IMAGE_NAME):$(RBD_IMAGE_VERSION)
$(CONTAINER_CMD) push $(RBD_IMAGE_NAME):$(RBD_IMAGE_VERSION)
push-image-cephfsplugin: image-cephfsplugin
docker push $(CEPHFS_IMAGE_NAME):$(CEPHFS_IMAGE_VERSION)
$(CONTAINER_CMD) push $(CEPHFS_IMAGE_NAME):$(CEPHFS_IMAGE_VERSION)
clean:
go clean -r -x
rm -f deploy/rbd/docker/rbdplugin
rm -f deploy/cephfs/docker/cephfsplugin
rm -f _output/rbdplugin
rm -f _output/cephfsplugin
rm -f _output/rbdplugin
rm -f _output/cephfsplugin

24
README.md
View File

@ -1,18 +1,22 @@
# Ceph CSI
# Ceph CSI 1.0.0
# Supported platforms
For Kubernetes 1.12 and below, please use [0.3 images and deployments](https://github.com/ceph/ceph-csi/tree/master/deploy/rbd/kubernetes).
For Kubernetes 1.13 and above, please use [1.0 images and deployments](https://github.com/ceph/ceph-csi/tree/csi-v1.0/deploy/rbd/kubernetes).
[Container Storage Interface (CSI)](https://github.com/container-storage-interface/) driver, provisioner, and attacher for Ceph RBD and CephFS.
[Container Storage Interface
(CSI)](https://github.com/container-storage-interface/) driver, provisioner,
and attacher for Ceph RBD and CephFS.
## Overview
Ceph CSI plugins implement an interface between CSI enabled Container Orchestrator (CO) and CEPH cluster. It allows dynamically provisioning CEPH volumes and attaching them to workloads. Current implementation of Ceph CSI plugins was tested in Kubernetes environment (requires Kubernetes 1.11+), but the code does not rely on any Kubernetes specific calls (WIP to make it k8s agnostic) and should be able to run with any CSI enabled CO.
Ceph CSI plugins implement an interface between CSI enabled Container
Orchestrator (CO) and CEPH cluster.
It allows dynamically provisioning CEPH volumes and attaching them to
workloads.
Current implementation of Ceph CSI plugins was tested in Kubernetes
environment (requires Kubernetes 1.13+), but the code does not rely on
any Kubernetes specific calls (WIP to make it k8s agnostic) and
should be able to run with any CSI enabled CO.
For details about configuration and deployment of RBD and CephFS CSI plugins, see documentation in `docs/`.
For details about configuration and deployment of RBD and
CephFS CSI plugins, see documentation in `docs/`.
For example usage of RBD and CephFS CSI plugins, see examples in `examples/`.

45
cephfs/main.go → cmd/cephfs/main.go
View File

@ -19,58 +19,45 @@ package main
import (
"flag"
"os"
"path"
"github.com/ceph/ceph-csi/pkg/cephfs"
"github.com/ceph/ceph-csi/pkg/util"
"github.com/golang/glog"
"k8s.io/klog"
)
func init() {
flag.Set("logtostderr", "true")
}
var (
endpoint = flag.String("endpoint", "unix://tmp/csi.sock", "CSI endpoint")
driverName = flag.String("drivername", "cephfs.csi.ceph.com", "name of the driver")
nodeId = flag.String("nodeid", "", "node id")
nodeID = flag.String("nodeid", "", "node id")
volumeMounter = flag.String("volumemounter", "", "default volume mounter (possible options are 'kernel', 'fuse')")
metadataStorage = flag.String("metadatastorage", "node", "metadata persistence method [node|k8s_configmap]")
metadataStorage = flag.String("metadatastorage", "", "metadata persistence method [node|k8s_configmap]")
mountCacheDir = flag.String("mountcachedir", "", "mount info cache save dir")
)
func main() {
func init() {
klog.InitFlags(nil)
if err := flag.Set("logtostderr", "true"); err != nil {
klog.Exitf("failed to set logtostderr flag: %v", err)
}
flag.Parse()
}
func main() {
err := util.ValidateDriverName(*driverName)
if err != nil {
glog.Errorf("failed to validate driver name: %v", err)
os.Exit(1)
klog.Fatalln(err)
}
//update plugin name
cephfs.PluginFolder = cephfs.PluginFolder + *driverName
if err := createPersistentStorage(path.Join(cephfs.PluginFolder, "controller")); err != nil {
glog.Errorf("failed to create persistent storage for controller: %v", err)
os.Exit(1)
}
if err := createPersistentStorage(path.Join(cephfs.PluginFolder, "node")); err != nil {
glog.Errorf("failed to create persistent storage for node: %v", err)
os.Exit(1)
}
cp, err := util.NewCachePersister(*metadataStorage, *driverName)
cp, err := util.CreatePersistanceStorage(cephfs.PluginFolder, *metadataStorage, *driverName)
if err != nil {
glog.Errorf("failed to define cache persistence method: %v", err)
os.Exit(1)
}
driver := cephfs.NewCephFSDriver()
driver.Run(*driverName, *nodeId, *endpoint, *volumeMounter, cp)
driver := cephfs.NewDriver()
driver.Run(*driverName, *nodeID, *endpoint, *volumeMounter, *mountCacheDir, cp)
os.Exit(0)
}
func createPersistentStorage(persistentStoragePath string) error {
return os.MkdirAll(persistentStoragePath, os.FileMode(0755))
}

49
rbd/main.go → cmd/rbd/main.go
View File

@ -19,63 +19,46 @@ package main
import (
"flag"
"os"
"path"
"github.com/ceph/ceph-csi/pkg/rbd"
"github.com/ceph/ceph-csi/pkg/util"
"github.com/golang/glog"
"k8s.io/klog"
)
func init() {
flag.Set("logtostderr", "true")
}
var (
endpoint = flag.String("endpoint", "unix://tmp/csi.sock", "CSI endpoint")
driverName = flag.String("drivername", "rbd.csi.ceph.com", "name of the driver")
nodeID = flag.String("nodeid", "", "node id")
containerized = flag.Bool("containerized", true, "whether run as containerized")
metadataStorage = flag.String("metadatastorage", "node", "metadata persistence method [node|k8s_configmap]")
metadataStorage = flag.String("metadatastorage", "", "metadata persistence method [node|k8s_configmap]")
configRoot = flag.String("configroot", "/etc/csi-config", "directory in which CSI specific Ceph"+
" cluster configurations are present, OR the value \"k8s_objects\" if present as kubernetes secrets")
)
func main() {
func init() {
klog.InitFlags(nil)
if err := flag.Set("logtostderr", "true"); err != nil {
klog.Exitf("failed to set logtostderr flag: %v", err)
}
flag.Parse()
}
func main() {
err := util.ValidateDriverName(*driverName)
if err != nil {
glog.Errorf("failed to validate driver name: %v", err)
os.Exit(1)
klog.Fatalln(err)
}
//update plugin name
rbd.PluginFolder = rbd.PluginFolder + *driverName
if err := createPersistentStorage(path.Join(rbd.PluginFolder, "controller")); err != nil {
glog.Errorf("failed to create persistent storage for controller %v", err)
os.Exit(1)
}
if err := createPersistentStorage(path.Join(rbd.PluginFolder, "node")); err != nil {
glog.Errorf("failed to create persistent storage for node %v", err)
os.Exit(1)
}
cp, err := util.NewCachePersister(*metadataStorage, *driverName)
cp, err := util.CreatePersistanceStorage(rbd.PluginFolder, *metadataStorage, *driverName)
if err != nil {
glog.Errorf("failed to define cache persistence method: %v", err)
os.Exit(1)
}
driver := rbd.GetRBDDriver()
driver.Run(*driverName, *nodeID, *endpoint, *containerized, cp)
driver := rbd.NewDriver()
driver.Run(*driverName, *nodeID, *endpoint, *configRoot, *containerized, cp)
os.Exit(0)
}
func createPersistentStorage(persistentStoragePath string) error {
if _, err := os.Stat(persistentStoragePath); os.IsNotExist(err) {
if err := os.MkdirAll(persistentStoragePath, os.FileMode(0755)); err != nil {
return err
}
} else {
}
return nil
}

58
deploy.sh
View File

@ -1,6 +1,58 @@
#!/bin/bash
if [ "${TRAVIS_BRANCH}" == "master" ] && [ "${TRAVIS_PULL_REQUEST}" == "false" ]; then
docker login -u "${QUAY_IO_USERNAME}" -p "${QUAY_IO_PASSWORD}" quay.io
make push-image-rbdplugin push-image-cephfsplugin
push_helm_chats() {
PACKAGE=$1
CHANGED=0
VERSION=$(grep 'version:' deploy/"$PACKAGE"/helm/Chart.yaml | awk '{print $2}')
if [ ! -f "tmp/csi-charts/docs/$PACKAGE/ceph-csi-$PACKAGE-$VERSION.tgz" ]; then
CHANGED=1
ln -s helm deploy/"$PACKAGE"/ceph-csi-"$PACKAGE"
mkdir -p tmp/csi-charts/docs/"$PACKAGE"
pushd tmp/csi-charts/docs/"$PACKAGE" >/dev/null
helm init --client-only
helm package ../../../../deploy/"$PACKAGE"/ceph-csi-"$PACKAGE"
popd >/dev/null
fi
if [ $CHANGED -eq 1 ]; then
pushd tmp/csi-charts/docs >/dev/null
helm repo index .
git add --all :/ && git commit -m "Update repo"
git push https://"$GITHUB_TOKEN"@github.com/ceph/csi-charts
popd >/dev/null
fi
}
if [ "${TRAVIS_BRANCH}" == 'csi-v0.3' ]; then
export RBD_IMAGE_VERSION='v0.3.0'
export CEPHFS_IMAGE_VERSION='v0.3.0'
elif [ "${TRAVIS_BRANCH}" == 'csi-v1.0' ]; then
export RBD_IMAGE_VERSION='v1.0.0'
export CEPHFS_IMAGE_VERSION='v1.0.0'
else
echo "!!! Branch ${TRAVIS_BRANCH} is not a deployable branch; exiting"
exit 0 # Exiting 0 so that this isn't marked as failing
fi
if [ "${TRAVIS_PULL_REQUEST}" == "false" ]; then
"${CONTAINER_CMD:-docker}" login -u "${QUAY_IO_USERNAME}" -p "${QUAY_IO_PASSWORD}" quay.io
make push-image-rbdplugin push-image-cephfsplugin
set -xe
mkdir -p tmp
pushd tmp >/dev/null
curl https://raw.githubusercontent.com/helm/helm/master/scripts/get >get_helm.sh
chmod 700 get_helm.sh
./get_helm.sh
git clone https://github.com/ceph/csi-charts
mkdir -p csi-charts/docs
popd >/dev/null
push_helm_chats rbd
push_helm_chats cephfs
fi

21
deploy/cephfs/helm/.helmignore Normal file
View File

@ -0,0 +1,21 @@
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*~
# Various IDEs
.project
.idea/
*.tmproj

14
deploy/cephfs/helm/Chart.yaml Normal file
View File

@ -0,0 +1,14 @@
---
apiVersion: v1
appVersion: "1.0.0"
description: "Container Storage Interface (CSI) driver,
provisioner, and attacher for Ceph cephfs"
name: ceph-csi-cephfs
version: 0.5.1
keywords:
- ceph
- cephfs
- ceph-csi
home: https://github.com/ceph/ceph-csi
sources:
- https://github.com/ceph/ceph-csi/tree/csi-v1.0/deploy/cephfs/helm

29
deploy/cephfs/helm/README.md Normal file
View File

@ -0,0 +1,29 @@
# ceph-csi-cephfs
The ceph-csi-cephfs chart adds cephfs volume support to your cluster.
## Install Chart
To install the Chart into your Kubernetes cluster
```bash
helm install --namespace "ceph-csi-cephfs" --name "ceph-csi-cephfs" ceph-csi/ceph-csi-cephfs
```
After installation succeeds, you can get a status of Chart
```bash
helm status "ceph-csi-cephfs"
```
If you want to delete your Chart, use this command
```bash
helm delete --purge "ceph-csi-cephfs"
```
If you want to delete the namespace, use this command
```bash
kubectl delete namespace ceph-csi-rbd
```

2
deploy/cephfs/helm/templates/NOTES.txt Normal file
View File

@ -0,0 +1,2 @@
Examples on how to configure a storage class and start using the driver are here:
https://github.com/ceph/ceph-csi/tree/csi-v1.0/examples/cephfs

119
deploy/cephfs/helm/templates/_helpers.tpl Normal file
View File

@ -0,0 +1,119 @@
{{/* vim: set filetype=mustache: */}}
{{/*
Expand the name of the chart.
*/}}
{{- define "ceph-csi-cephfs.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "ceph-csi-cephfs.fullname" -}}
{{- if .Values.fullnameOverride -}}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- $name := default .Chart.Name .Values.nameOverride -}}
{{- if contains $name .Release.Name -}}
{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- end -}}
{{- end -}}
{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "ceph-csi-cephfs.attacher.fullname" -}}
{{- if .Values.attacher.fullnameOverride -}}
{{- .Values.attacher.fullnameOverride | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- $name := default .Chart.Name .Values.nameOverride -}}
{{- if contains $name .Release.Name -}}
{{- printf "%s-%s" .Release.Name .Values.attacher.name | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- printf "%s-%s-%s" .Release.Name $name .Values.attacher.name | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- end -}}
{{- end -}}
{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "ceph-csi-cephfs.nodeplugin.fullname" -}}
{{- if .Values.nodeplugin.fullnameOverride -}}
{{- .Values.nodeplugin.fullnameOverride | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- $name := default .Chart.Name .Values.nameOverride -}}
{{- if contains $name .Release.Name -}}
{{- printf "%s-%s" .Release.Name .Values.nodeplugin.name | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- printf "%s-%s-%s" .Release.Name $name .Values.nodeplugin.name | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- end -}}
{{- end -}}
{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "ceph-csi-cephfs.provisioner.fullname" -}}
{{- if .Values.provisioner.fullnameOverride -}}
{{- .Values.provisioner.fullnameOverride | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- $name := default .Chart.Name .Values.nameOverride -}}
{{- if contains $name .Release.Name -}}
{{- printf "%s-%s" .Release.Name .Values.provisioner.name | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- printf "%s-%s-%s" .Release.Name $name .Values.provisioner.name | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- end -}}
{{- end -}}
{{/*
Create chart name and version as used by the chart label.
*/}}
{{- define "ceph-csi-cephfs.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{/*
Create the name of the service account to use
*/}}
{{- define "ceph-csi-cephfs.serviceAccountName.attacher" -}}
{{- if .Values.serviceAccounts.attacher.create -}}
{{ default (include "ceph-csi-cephfs.attacher.fullname" .) .Values.serviceAccounts.attacher.name }}
{{- else -}}
{{ default "default" .Values.serviceAccounts.attacher.name }}
{{- end -}}
{{- end -}}
{{/*
Create the name of the service account to use
*/}}
{{- define "ceph-csi-cephfs.serviceAccountName.nodeplugin" -}}
{{- if .Values.serviceAccounts.nodeplugin.create -}}
{{ default (include "ceph-csi-cephfs.nodeplugin.fullname" .) .Values.serviceAccounts.nodeplugin.name }}
{{- else -}}
{{ default "default" .Values.serviceAccounts.nodeplugin.name }}
{{- end -}}
{{- end -}}
{{/*
Create the name of the service account to use
*/}}
{{- define "ceph-csi-cephfs.serviceAccountName.provisioner" -}}
{{- if .Values.serviceAccounts.provisioner.create -}}
{{ default (include "ceph-csi-cephfs.provisioner.fullname" .) .Values.serviceAccounts.provisioner.name }}
{{- else -}}
{{ default "default" .Values.serviceAccounts.provisioner.name }}
{{- end -}}
{{- end -}}

28
deploy/cephfs/helm/templates/attacher-clusterrole.yaml Normal file
View File

@ -0,0 +1,28 @@
{{- if .Values.rbac.create -}}
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: {{ include "ceph-csi-cephfs.attacher.fullname" . }}
labels:
app: {{ include "ceph-csi-cephfs.name" . }}
chart: {{ include "ceph-csi-cephfs.chart" . }}
component: {{ .Values.attacher.name }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
rules:
- apiGroups: [""]
resources: ["events"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get", "list", "watch"]
- apiGroups: ["storage.k8s.io"]
resources: ["volumeattachments"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["csi.storage.k8s.io"]
resources: ["csinodeinfos"]
verbs: ["get", "list", "watch"]
{{- end -}}

20
deploy/cephfs/helm/templates/attacher-clusterrolebinding.yaml Normal file
View File

@ -0,0 +1,20 @@
{{- if .Values.rbac.create -}}
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: {{ include "ceph-csi-cephfs.attacher.fullname" . }}
labels:
app: {{ include "ceph-csi-cephfs.name" . }}
chart: {{ include "ceph-csi-cephfs.chart" . }}
component: {{ .Values.attacher.name }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
subjects:
- kind: ServiceAccount
name: {{ include "ceph-csi-cephfs.serviceAccountName.attacher" . }}
namespace: {{ .Release.Namespace }}
roleRef:
kind: ClusterRole
name: {{ include "ceph-csi-cephfs.attacher.fullname" . }}
apiGroup: rbac.authorization.k8s.io
{{- end -}}

18
deploy/cephfs/helm/templates/attacher-service.yaml Normal file
View File

@ -0,0 +1,18 @@
kind: Service
apiVersion: v1
metadata:
name: {{ include "ceph-csi-cephfs.attacher.fullname" . }}
labels:
app: {{ include "ceph-csi-cephfs.name" . }}
chart: {{ include "ceph-csi-cephfs.chart" . }}
component: {{ .Values.attacher.name }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
spec:
selector:
app: {{ include "ceph-csi-cephfs.name" . }}
component: {{ .Values.attacher.name }}
release: {{ .Release.Name }}
ports:
- name: dummy
port: 12345

12
deploy/cephfs/helm/templates/attacher-serviceaccount.yaml Normal file
View File

@ -0,0 +1,12 @@
{{- if .Values.serviceAccounts.attacher.create -}}
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ include "ceph-csi-cephfs.serviceAccountName.attacher" . }}
labels:
app: {{ include "ceph-csi-cephfs.name" . }}
chart: {{ include "ceph-csi-cephfs.chart" . }}
component: {{ .Values.attacher.name }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
{{- end -}}

60
deploy/cephfs/helm/templates/attacher-statefulset.yaml Normal file
View File

@ -0,0 +1,60 @@
kind: StatefulSet
apiVersion: apps/v1beta1
metadata:
name: {{ include "ceph-csi-cephfs.attacher.fullname" . }}
labels:
app: {{ include "ceph-csi-cephfs.name" . }}
chart: {{ include "ceph-csi-cephfs.chart" . }}
component: {{ .Values.attacher.name }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
spec:
serviceName: {{ include "ceph-csi-cephfs.attacher.fullname" . }}
replicas: {{ .Values.attacher.replicas }}
selector:
matchLabels:
app: {{ include "ceph-csi-cephfs.name" . }}
component: {{ .Values.attacher.name }}
release: {{ .Release.Name }}
template:
metadata:
labels:
app: {{ include "ceph-csi-cephfs.name" . }}
chart: {{ include "ceph-csi-cephfs.chart" . }}
component: {{ .Values.attacher.name }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
spec:
serviceAccountName: {{ include "ceph-csi-cephfs.serviceAccountName.attacher" . }}
containers:
- name: csi-cephfsplugin-attacher
image: "{{ .Values.attacher.image.repository }}:{{ .Values.attacher.image.tag }}"
args:
- "--v=5"
- "--csi-address=$(ADDRESS)"
env:
- name: ADDRESS
value: "{{ .Values.socketDir }}/{{ .Values.socketFile }}"
imagePullPolicy: {{ .Values.attacher.image.pullPolicy }}
volumeMounts:
- name: socket-dir
mountPath: {{ .Values.socketDir }}
resources:
{{ toYaml .Values.attacher.resources | indent 12 }}
volumes:
- name: socket-dir
hostPath:
path: {{ .Values.socketDir }}
type: DirectoryOrCreate
{{- if .Values.attacher.affinity -}}
affinity:
{{ toYaml .Values.attacher.affinity . | indent 8 }}
{{- end -}}
{{- if .Values.attacher.nodeSelector -}}
nodeSelector:
{{ toYaml .Values.attacher.nodeSelector | indent 8 }}
{{- end -}}
{{- if .Values.attacher.tolerations -}}
tolerations:
{{ toYaml .Values.attacher.tolerations | indent 8 }}
{{- end -}}

31
deploy/cephfs/helm/templates/nodeplugin-clusterrole.yaml Normal file
View File

@ -0,0 +1,31 @@
{{- if .Values.rbac.create -}}
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }}
labels:
app: {{ include "ceph-csi-cephfs.name" . }}
chart: {{ include "ceph-csi-cephfs.chart" . }}
component: {{ .Values.nodeplugin.name }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
rules:
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get", "list"]
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get", "list", "update"]
- apiGroups: [""]
resources: ["namespaces"]
verbs: ["get", "list"]
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["volumeattachments"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get", "list"]
{{- end -}}

20
deploy/cephfs/helm/templates/nodeplugin-clusterrolebinding.yaml Normal file
View File

@ -0,0 +1,20 @@
{{- if .Values.rbac.create -}}
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }}
labels:
app: {{ include "ceph-csi-cephfs.name" . }}
chart: {{ include "ceph-csi-cephfs.chart" . }}
component: {{ .Values.nodeplugin.name }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
subjects:
- kind: ServiceAccount
name: {{ include "ceph-csi-cephfs.serviceAccountName.nodeplugin" . }}
namespace: {{ .Release.Namespace }}
roleRef:
kind: ClusterRole
name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }}
apiGroup: rbac.authorization.k8s.io
{{- end -}}

150
deploy/cephfs/helm/templates/nodeplugin-daemonset.yaml Normal file
View File

@ -0,0 +1,150 @@
kind: DaemonSet
apiVersion: apps/v1beta2
metadata:
name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }}
labels:
app: {{ include "ceph-csi-cephfs.name" . }}
chart: {{ include "ceph-csi-cephfs.chart" . }}
component: {{ .Values.nodeplugin.name }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
spec:
selector:
matchLabels:
app: {{ include "ceph-csi-cephfs.name" . }}
component: {{ .Values.nodeplugin.name }}
release: {{ .Release.Name }}
template:
metadata:
labels:
app: {{ include "ceph-csi-cephfs.name" . }}
chart: {{ include "ceph-csi-cephfs.chart" . }}
component: {{ .Values.nodeplugin.name }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
spec:
serviceAccountName: {{ include "ceph-csi-cephfs.serviceAccountName.nodeplugin" . }}
hostNetwork: true
hostPID: true
# to use e.g. Rook orchestrated cluster, and mons' FQDN is
# resolved through k8s service, set dns policy to cluster first
dnsPolicy: ClusterFirstWithHostNet
containers:
- name: driver-registrar
image: "{{ .Values.nodeplugin.registrar.image.repository }}:{{ .Values.nodeplugin.registrar.image.tag }}"
args:
- "--v=5"
- "--csi-address=/csi/{{ .Values.socketFile }}"
- "--kubelet-registration-path={{ .Values.socketDir }}/{{ .Values.socketFile }}"
lifecycle:
preStop:
exec:
command: [
"/bin/sh", "-c",
'rm -rf /registration/{{ .Values.driverName }}
/registration/{{ .Values.driverName }}-reg.sock'
]
env:
- name: KUBE_NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
imagePullPolicy: {{ .Values.nodeplugin.registrar.image.imagePullPolicy }}
volumeMounts:
- name: plugin-dir
mountPath: /csi
- name: registration-dir
mountPath: /registration
resources:
{{ toYaml .Values.nodeplugin.registrar.resources | indent 12 }}
- name: csi-cephfsplugin
securityContext:
privileged: true
capabilities:
add: ["SYS_ADMIN"]
allowPrivilegeEscalation: true
image: "{{ .Values.nodeplugin.plugin.image.repository }}:{{ .Values.nodeplugin.plugin.image.tag }}"
args :
- "--nodeid=$(NODE_ID)"
- "--endpoint=$(CSI_ENDPOINT)"
- "--v=5"
- "--drivername=$(DRIVER_NAME)"
- "--metadatastorage=k8s_configmap"
- "--mountcachedir=/mount-cache-dir"
env:
- name: HOST_ROOTFS
value: "/rootfs"
- name: DRIVER_NAME
value: {{ .Values.driverName }}
- name: NODE_ID
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: CSI_ENDPOINT
value: "unix:/{{ .Values.socketDir }}/{{ .Values.socketFile }}"
imagePullPolicy: {{ .Values.nodeplugin.plugin.image.imagePullPolicy }}
volumeMounts:
- name: mount-cache-dir
mountPath: /mount-cache-dir
- name: plugin-dir
mountPath: {{ .Values.socketDir }}
- name: pods-mount-dir
mountPath: /var/lib/kubelet/pods
mountPropagation: "Bidirectional"
- name: plugin-mount-dir
mountPath: {{ .Values.volumeDevicesDir }}
mountPropagation: "Bidirectional"
- mountPath: /dev
name: host-dev
- mountPath: /rootfs
name: host-rootfs
- mountPath: /sys
name: host-sys
- mountPath: /lib/modules
name: lib-modules
readOnly: true
resources:
{{ toYaml .Values.nodeplugin.plugin.resources | indent 12 }}
volumes:
- name: mount-cache-dir
emptyDir: {}
- name: plugin-dir
hostPath:
path: {{ .Values.socketDir }}
type: DirectoryOrCreate
- name: plugin-mount-dir
hostPath:
path: {{ .Values.volumeDevicesDir }}
type: DirectoryOrCreate
- name: registration-dir
hostPath:
path: {{ .Values.registrationDir }}
type: Directory
- name: pods-mount-dir
hostPath:
path: /var/lib/kubelet/pods
type: Directory
- name: host-dev
hostPath:
path: /dev
- name: host-rootfs
hostPath:
path: /
- name: host-sys
hostPath:
path: /sys
- name: lib-modules
hostPath:
path: /lib/modules
{{- if .Values.nodeplugin.affinity -}}
affinity:
{{ toYaml .Values.nodeplugin.affinity . | indent 8 }}
{{- end -}}
{{- if .Values.nodeplugin.nodeSelector -}}
nodeSelector:
{{ toYaml .Values.nodeplugin.nodeSelector | indent 8 }}
{{- end -}}
{{- if .Values.nodeplugin.tolerations -}}
tolerations:
{{ toYaml .Values.nodeplugin.tolerations | indent 8 }}
{{- end -}}

12
deploy/cephfs/helm/templates/nodeplugin-serviceaccount.yaml Normal file
View File

@ -0,0 +1,12 @@
{{- if .Values.serviceAccounts.nodeplugin.create -}}
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ include "ceph-csi-cephfs.serviceAccountName.nodeplugin" . }}
labels:
app: {{ include "ceph-csi-cephfs.name" . }}
chart: {{ include "ceph-csi-cephfs.chart" . }}
component: {{ .Values.nodeplugin.name }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
{{- end -}}

37
deploy/cephfs/helm/templates/provisioner-clusterrole.yaml Normal file
View File

@ -0,0 +1,37 @@
{{- if .Values.rbac.create -}}
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }}
labels:
app: {{ include "ceph-csi-cephfs.name" . }}
chart: {{ include "ceph-csi-cephfs.chart" . }}
component: {{ .Values.provisioner.name }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
rules:
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get", "list"]
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["events"]
verbs: ["list", "watch", "create", "update", "patch"]
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get", "list", "create", "delete"]
- apiGroups: ["csi.storage.k8s.io"]
resources: ["csinodeinfos"]
verbs: ["get", "list", "watch"]
{{- end -}}

20
deploy/cephfs/helm/templates/provisioner-clusterrolebinding.yaml Normal file
View File

@ -0,0 +1,20 @@
{{- if .Values.rbac.create -}}
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }}
labels:
app: {{ include "ceph-csi-cephfs.name" . }}
chart: {{ include "ceph-csi-cephfs.chart" . }}
component: {{ .Values.provisioner.name }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
subjects:
- kind: ServiceAccount
name: {{ include "ceph-csi-cephfs.serviceAccountName.provisioner" . }}
namespace: {{ .Release.Namespace }}
roleRef:
kind: ClusterRole
name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }}
apiGroup: rbac.authorization.k8s.io
{{- end -}}

19
deploy/cephfs/helm/templates/provisioner-role.yaml Normal file
View File

@ -0,0 +1,19 @@
{{- if .Values.rbac.create -}}
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }}
labels:
app: {{ include "ceph-csi-cephfs.name" . }}
chart: {{ include "ceph-csi-cephfs.chart" . }}
component: {{ .Values.provisioner.name }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
rules:
- apiGroups: [""]
resources: ["endpoints"]
verbs: ["get", "watch", "list", "delete", "update", "create"]
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get", "list", "watch", "create", "delete"]
{{- end -}}

21
deploy/cephfs/helm/templates/provisioner-rolebinding.yaml Normal file
View File

@ -0,0 +1,21 @@
{{- if .Values.rbac.create -}}
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }}
labels:
app: {{ include "ceph-csi-cephfs.name" . }}
chart: {{ include "ceph-csi-cephfs.chart" . }}
component: {{ .Values.provisioner.name }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
subjects:
- kind: ServiceAccount
name: {{ include "ceph-csi-cephfs.serviceAccountName.provisioner" . }}
namespace: {{ .Release.Namespace }}
roleRef:
kind: Role
name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }}
apiGroup: rbac.authorization.k8s.io
namespace: {{ .Release.Namespace }}
{{- end -}}

18
deploy/cephfs/helm/templates/provisioner-service.yaml Normal file
View File

@ -0,0 +1,18 @@
kind: Service
apiVersion: v1
metadata:
name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }}
labels:
app: {{ include "ceph-csi-cephfs.name" . }}
chart: {{ include "ceph-csi-cephfs.chart" . }}
component: {{ .Values.provisioner.name }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
spec:
selector:
app: {{ include "ceph-csi-cephfs.name" . }}
component: {{ .Values.provisioner.name }}
release: {{ .Release.Name }}
ports:
- name: dummy
port: 12345

12
deploy/cephfs/helm/templates/provisioner-serviceaccount.yaml Normal file
View File

@ -0,0 +1,12 @@
{{- if .Values.serviceAccounts.provisioner.create -}}
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ include "ceph-csi-cephfs.serviceAccountName.provisioner" . }}
labels:
app: {{ include "ceph-csi-cephfs.name" . }}
chart: {{ include "ceph-csi-cephfs.chart" . }}
component: {{ .Values.provisioner.name }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
{{- end -}}

94
deploy/cephfs/helm/templates/provisioner-statefulset.yaml Normal file
View File

@ -0,0 +1,94 @@
kind: StatefulSet
apiVersion: apps/v1beta1
metadata:
name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }}
labels:
app: {{ include "ceph-csi-cephfs.name" . }}
chart: {{ include "ceph-csi-cephfs.chart" . }}
component: {{ .Values.provisioner.name }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
spec:
serviceName: {{ include "ceph-csi-cephfs.provisioner.fullname" . }}
replicas: {{ .Values.provisioner.replicas }}
selector:
matchLabels:
app: {{ include "ceph-csi-cephfs.name" . }}
component: {{ .Values.provisioner.name }}
release: {{ .Release.Name }}
template:
metadata:
labels:
app: {{ include "ceph-csi-cephfs.name" . }}
chart: {{ include "ceph-csi-cephfs.chart" . }}
component: {{ .Values.provisioner.name }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
spec:
serviceAccountName: {{ include "ceph-csi-cephfs.serviceAccountName.provisioner" . }}
containers:
- name: csi-provisioner
image: "{{ .Values.provisioner.image.repository }}:{{ .Values.provisioner.image.tag }}"
args:
- "--csi-address=$(ADDRESS)"
- "--v=5"
env:
- name: ADDRESS
value: "{{ .Values.socketDir }}/{{ .Values.socketFile }}"
imagePullPolicy: {{ .Values.provisioner.image.pullPolicy }}
volumeMounts:
- name: socket-dir
mountPath: {{ .Values.socketDir }}
resources:
{{ toYaml .Values.provisioner.resources | indent 12 }}
- name: csi-cephfsplugin
securityContext:
privileged: true
capabilities:
add: ["SYS_ADMIN"]
allowPrivilegeEscalation: true
image: "{{ .Values.nodeplugin.plugin.image.repository }}:{{ .Values.nodeplugin.plugin.image.tag }}"
args :
- "--nodeid=$(NODE_ID)"
- "--endpoint=$(CSI_ENDPOINT)"
- "--v=5"
- "--drivername=$(DRIVER_NAME)"
- "--metadatastorage=k8s_configmap"
env:
- name: HOST_ROOTFS
value: "/rootfs"
- name: DRIVER_NAME
value: {{ .Values.driverName }}
- name: NODE_ID
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: CSI_ENDPOINT
value: "unix:/{{ .Values.socketDir }}/{{ .Values.socketFile }}"
imagePullPolicy: {{ .Values.nodeplugin.plugin.image.imagePullPolicy }}
volumeMounts:
- name: socket-dir
mountPath: {{ .Values.socketDir }}
- name: host-rootfs
mountPath: "/rootfs"
resources:
{{ toYaml .Values.nodeplugin.plugin.resources | indent 12 }}
volumes:
- name: socket-dir
emptyDir: {}
#FIXME this seems way too much. Why is it needed at all for this?
- name: host-rootfs
hostPath:
path: /
{{- if .Values.provisioner.affinity -}}
affinity:
{{ toYaml .Values.provisioner.affinity . | indent 8 }}
{{- end -}}
{{- if .Values.provisioner.nodeSelector -}}
nodeSelector:
{{ toYaml .Values.provisioner.nodeSelector | indent 8 }}
{{- end -}}
{{- if .Values.provisioner.tolerations -}}
tolerations:
{{ toYaml .Values.provisioner.tolerations | indent 8 }}
{{- end -}}

80
deploy/cephfs/helm/values.yaml Normal file
View File

@ -0,0 +1,80 @@
---
rbac:
create: true
serviceAccounts:
attacher:
create: true
name:
nodeplugin:
create: true
name:
provisioner:
create: true
name:
socketDir: /var/lib/kubelet/plugins/cephfs.csi.ceph.com
socketFile: csi.sock
registrationDir: /var/lib/kubelet/plugins_registry
volumeDevicesDir: /var/lib/kubelet/plugins/kubernetes.io/csi/volumeDevices
driverName: cephfs.csi.ceph.com
attacher:
name: attacher
replicaCount: 1
image:
repository: quay.io/k8scsi/csi-attacher
tag: v1.0.1
pullPolicy: IfNotPresent
resources: {}
nodeSelector: {}
tolerations: []
affinity: {}
nodeplugin:
name: nodeplugin
registrar:
image:
repository: quay.io/k8scsi/csi-node-driver-registrar
tag: v1.0.2
pullPolicy: IfNotPresent
resources: {}
plugin:
image:
repository: quay.io/cephcsi/cephfsplugin
tag: v1.0.0
pullPolicy: IfNotPresent
resources: {}
nodeSelector: {}
tolerations: []
affinity: {}
provisioner:
name: provisioner
replicaCount: 1
image:
repository: quay.io/k8scsi/csi-provisioner
tag: v1.0.1
pullPolicy: IfNotPresent
resources: {}
nodeSelector: {}
tolerations: []
affinity: {}

17
deploy/cephfs/kubernetes/csi-attacher-rbac.yaml
View File

@ -1,17 +1,15 @@
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: csi-attacher
name: cephfs-csi-attacher
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: external-attacher-runner
name: cephfs-external-attacher-runner
rules:
- apiGroups: [""]
resources: ["events"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "update"]
@ -21,17 +19,20 @@ rules:
- apiGroups: ["storage.k8s.io"]
resources: ["volumeattachments"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["csi.storage.k8s.io"]
resources: ["csinodeinfos"]
verbs: ["get", "list", "watch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: csi-attacher-role
name: cephfs-csi-attacher-role
subjects:
- kind: ServiceAccount
name: csi-attacher
name: cephfs-csi-attacher
namespace: default
roleRef:
kind: ClusterRole
name: external-attacher-runner
name: cephfs-external-attacher-runner
apiGroup: rbac.authorization.k8s.io

5
deploy/cephfs/kubernetes/csi-cephfsplugin-attacher.yaml
View File

@ -1,3 +1,4 @@
---
kind: Service
apiVersion: v1
metadata:
@ -24,10 +25,10 @@ spec:
labels:
app: csi-cephfsplugin-attacher
spec:
serviceAccount: csi-attacher
serviceAccount: cephfs-csi-attacher
containers:
- name: csi-cephfsplugin-attacher
image: quay.io/k8scsi/csi-attacher:v0.3.0
image: quay.io/k8scsi/csi-attacher:v1.0.1
args:
- "--v=5"
- "--csi-address=$(ADDRESS)"

53
deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml
View File

@ -1,3 +1,4 @@
---
kind: Service
apiVersion: v1
metadata:
@ -24,23 +25,65 @@ spec:
labels:
app: csi-cephfsplugin-provisioner
spec:
serviceAccount: csi-provisioner
serviceAccount: cephfs-csi-provisioner
containers:
- name: csi-provisioner
image: quay.io/k8scsi/csi-provisioner:v0.3.0
image: quay.io/k8scsi/csi-provisioner:v1.0.1
args:
- "--provisioner=cephfs.csi.ceph.com"
- "--csi-address=$(ADDRESS)"
- "--v=5"
env:
- name: ADDRESS
value: /var/lib/kubelet/plugins/cephfs.csi.ceph.com/csi.sock
value: unix:///csi/csi-provisioner.sock
imagePullPolicy: "IfNotPresent"
volumeMounts:
- name: socket-dir
mountPath: /var/lib/kubelet/plugins/cephfs.csi.ceph.com
mountPath: /csi
- name: csi-cephfsplugin
securityContext:
privileged: true
capabilities:
add: ["SYS_ADMIN"]
image: quay.io/cephcsi/cephfsplugin:v1.0.0
args:
- "--nodeid=$(NODE_ID)"
- "--endpoint=$(CSI_ENDPOINT)"
- "--v=5"
- "--drivername=cephfs.csi.ceph.com"
- "--metadatastorage=k8s_configmap"
env:
- name: NODE_ID
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: CSI_ENDPOINT
value: unix:///csi/csi-provisioner.sock
imagePullPolicy: "IfNotPresent"
volumeMounts:
- name: socket-dir
mountPath: /csi
- name: host-sys
mountPath: /sys
- name: lib-modules
mountPath: /lib/modules
readOnly: true
- name: host-dev
mountPath: /dev
volumes:
- name: socket-dir
hostPath:
path: /var/lib/kubelet/plugins/cephfs.csi.ceph.com
type: DirectoryOrCreate
- name: host-sys
hostPath:
path: /sys
- name: lib-modules
hostPath:
path: /lib/modules
- name: host-dev
hostPath:
path: /dev

55
deploy/cephfs/kubernetes/csi-cephfsplugin.yaml
View File

@ -1,3 +1,4 @@
---
kind: DaemonSet
apiVersion: apps/v1beta2
metadata:
@ -11,30 +12,34 @@ spec:
labels:
app: csi-cephfsplugin
spec:
serviceAccount: csi-nodeplugin
serviceAccount: cephfs-csi-nodeplugin
hostNetwork: true
# to use e.g. Rook orchestrated cluster, and mons' FQDN is
# resolved through k8s service, set dns policy to cluster first
dnsPolicy: ClusterFirstWithHostNet
containers:
- name: driver-registrar
image: quay.io/k8scsi/driver-registrar:v0.3.0
image: quay.io/k8scsi/csi-node-driver-registrar:v1.0.2
args:
- "--v=5"
- "--csi-address=$(ADDRESS)"
- "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)"
- "--csi-address=/csi/csi.sock"
- "--kubelet-registration-path=/var/lib/kubelet/plugins/cephfs.csi.ceph.com/csi.sock"
lifecycle:
preStop:
exec:
command: [
"/bin/sh", "-c",
"rm -rf /registration/csi-cephfsplugin \
/registration/csi-cephfsplugin-reg.sock"
]
env:
- name: ADDRESS
value: /var/lib/kubelet/plugins/cephfs.csi.ceph.com/csi.sock
- name: DRIVER_REG_SOCK_PATH
value: /var/lib/kubelet/plugins/cephfs.csi.ceph.com/csi.sock
- name: KUBE_NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
volumeMounts:
- name: socket-dir
mountPath: /var/lib/kubelet/plugins/cephfs.csi.ceph.com
- name: plugin-dir
mountPath: /csi
- name: registration-dir
mountPath: /registration
- name: csi-cephfsplugin
@ -43,13 +48,14 @@ spec:
capabilities:
add: ["SYS_ADMIN"]
allowPrivilegeEscalation: true
image: quay.io/cephcsi/cephfsplugin:v0.3.0
image: quay.io/cephcsi/cephfsplugin:v1.0.0
args:
- "--nodeid=$(NODE_ID)"
- "--endpoint=$(CSI_ENDPOINT)"
- "--v=5"
- "--drivername=cephfs.csi.ceph.com"
- "--metadatastorage=k8s_configmap"
- "--mountcachedir=/mount-cache-dir"
env:
- name: NODE_ID
valueFrom:
@ -60,38 +66,45 @@ spec:
fieldRef:
fieldPath: metadata.namespace
- name: CSI_ENDPOINT
value: unix://var/lib/kubelet/plugins/cephfs.csi.ceph.com/csi.sock
value: unix:///csi/csi.sock
imagePullPolicy: "IfNotPresent"
volumeMounts:
- name: mount-cache-dir
mountPath: /mount-cache-dir
- name: plugin-dir
mountPath: /var/lib/kubelet/plugins/cephfs.csi.ceph.com
mountPath: /csi
- name: csi-plugins-dir
mountPath: /var/lib/kubelet/plugins/kubernetes.io/csi
mountPropagation: "Bidirectional"
- name: pods-mount-dir
mountPath: /var/lib/kubelet/pods
mountPropagation: "Bidirectional"
- mountPath: /sys
name: host-sys
- name: host-sys
mountPath: /sys
- name: lib-modules
mountPath: /lib/modules
readOnly: true
- name: host-dev
mountPath: /dev
volumes:
- name: mount-cache-dir
emptyDir: {}
- name: plugin-dir
hostPath:
path: /var/lib/kubelet/plugins/cephfs.csi.ceph.com
path: /var/lib/kubelet/plugins/cephfs.csi.ceph.com/
type: DirectoryOrCreate
- name: csi-plugins-dir
hostPath:
path: /var/lib/kubelet/plugins/kubernetes.io/csi
type: DirectoryOrCreate
- name: registration-dir
hostPath:
path: /var/lib/kubelet/plugins/
path: /var/lib/kubelet/plugins_registry/
type: Directory
- name: pods-mount-dir
hostPath:
path: /var/lib/kubelet/pods
type: Directory
- name: socket-dir
hostPath:
path: /var/lib/kubelet/plugins/cephfs.csi.ceph.com
type: DirectoryOrCreate
- name: host-sys
hostPath:
path: /sys

19
deploy/cephfs/kubernetes/csi-nodeplugin-rbac.yaml
View File

@ -1,14 +1,18 @@
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: csi-nodeplugin
name: cephfs-csi-nodeplugin
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: csi-nodeplugin
name: cephfs-csi-nodeplugin
rules:
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get", "list"]
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get", "list", "update"]
@ -21,20 +25,17 @@ rules:
- apiGroups: ["storage.k8s.io"]
resources: ["volumeattachments"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get", "list", "create", "delete"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: csi-nodeplugin
name: cephfs-csi-nodeplugin
subjects:
- kind: ServiceAccount
name: csi-nodeplugin
name: cephfs-csi-nodeplugin
namespace: default
roleRef:
kind: ClusterRole
name: csi-nodeplugin
apiGroup: rbac.authorization.k8s.io
name: cephfs-csi-nodeplugin
apiGroup: rbac.authorization.k8s.io

57
deploy/cephfs/kubernetes/csi-provisioner-rbac.yaml
View File

@ -1,17 +1,24 @@
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: csi-provisioner
name: cephfs-csi-provisioner
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: external-provisioner-runner
name: cephfs-external-provisioner-runner
rules:
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get", "list"]
- apiGroups: [""]
resources: ["events"]
verbs: ["list", "watch", "create", "update", "patch"]
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "create", "delete"]
@ -21,20 +28,52 @@ rules:
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["events"]
verbs: ["list", "watch", "create", "update", "patch"]
- apiGroups: ["csi.storage.k8s.io"]
resources: ["csinodeinfos"]
verbs: ["get", "list", "watch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: csi-provisioner-role
name: cephfs-csi-provisioner-role
subjects:
- kind: ServiceAccount
name: csi-provisioner
name: cephfs-csi-provisioner
namespace: default
roleRef:
kind: ClusterRole
name: external-provisioner-runner
name: cephfs-external-provisioner-runner
apiGroup: rbac.authorization.k8s.io
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
# replace with non-default namespace name
namespace: default
name: cephfs-external-provisioner-cfg
rules:
- apiGroups: [""]
resources: ["endpoints"]
verbs: ["get", "watch", "list", "delete", "update", "create"]
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get", "list", "create", "delete"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: cephfs-csi-provisioner-role-cfg
# replace with non-default namespace name
namespace: default
subjects:
- kind: ServiceAccount
name: cephfs-csi-provisioner
# replace with non-default namespace name
namespace: default
roleRef:
kind: Role
name: cephfs-external-provisioner-cfg
apiGroup: rbac.authorization.k8s.io

16
deploy/rbd/helm/Chart.yaml
View File

@ -1,12 +1,14 @@
---
apiVersion: v1
appVersion: "0.3.0"
description: Container Storage Interface (CSI) driver, provisioner, and attacher for Ceph RBD
appVersion: "1.0.0"
description: "Container Storage Interface (CSI) driver,
provisioner, snapshotter, and attacher for Ceph RBD"
name: ceph-csi-rbd
version: 0.1.0
version: 0.5.1
keywords:
- ceph
- rbd
- ceph-csi
- ceph
- rbd
- ceph-csi
home: https://github.com/ceph/ceph-csi
sources:
- https://github.com/ceph/ceph-csi/tree/master/deploy/rbd/kubernetes
- https://github.com/ceph/ceph-csi/tree/csi-v1.0/deploy/rbd/helm

29
deploy/rbd/helm/README.md Normal file
View File

@ -0,0 +1,29 @@
# ceph-csi-rbd
The ceph-csi-rbd chart adds rbd volume support to your cluster.
## Install Chart
To install the Chart into your Kubernetes cluster
```bash
helm install --namespace "ceph-csi-rbd" --name "ceph-csi-rbd" ceph-csi/ceph-csi-rbd
```
After installation succeeds, you can get a status of Chart
```bash
helm status "ceph-csi-rbd"
```
If you want to delete your Chart, use this command
```bash
helm delete --purge "ceph-csi-rbd"
```
If you want to delete the namespace, use this command
```bash
kubectl delete namespace ceph-csi-rbd
```

3
deploy/rbd/helm/templates/NOTES.txt
View File

@ -1 +1,2 @@
The Ceph RBD Container Storage Interface has been deployed.
Examples on how to configure a storage class and start using the driver are here:
https://github.com/ceph/ceph-csi/tree/csi-v1.0/examples/rbd

3
deploy/rbd/helm/templates/attacher-clusterrole.yaml
View File

@ -22,4 +22,7 @@ rules:
- apiGroups: ["storage.k8s.io"]
resources: ["volumeattachments"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["csi.storage.k8s.io"]
resources: ["csinodeinfos"]
verbs: ["get", "list", "watch"]
{{- end -}}

5
deploy/rbd/helm/templates/attacher-statefulset.yaml
View File

@ -11,6 +11,11 @@ metadata:
spec:
serviceName: {{ include "ceph-csi-rbd.attacher.fullname" . }}
replicas: {{ .Values.attacher.replicas }}
selector:
matchLabels:
app: {{ include "ceph-csi-rbd.name" . }}
component: {{ .Values.attacher.name }}
release: {{ .Release.Name }}
template:
metadata:
labels:

3
deploy/rbd/helm/templates/nodeplugin-clusterrole.yaml
View File

@ -22,4 +22,7 @@ rules:
- apiGroups: ["storage.k8s.io"]
resources: ["volumeattachments"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get", "list"]
{{- end -}}

40
deploy/rbd/helm/templates/nodeplugin-daemonset.yaml
View File

@ -34,21 +34,25 @@ spec:
image: "{{ .Values.nodeplugin.registrar.image.repository }}:{{ .Values.nodeplugin.registrar.image.tag }}"
args:
- "--v=5"
- "--csi-address=$(ADDRESS)"
- "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)"
- "--csi-address=/csi/{{ .Values.socketFile }}"
- "--kubelet-registration-path={{ .Values.socketDir }}/{{ .Values.socketFile }}"
lifecycle:
preStop:
exec:
command: [
"/bin/sh", "-c",
'rm -rf /registration/{{ .Values.driverName }}
/registration/{{ .Values.driverName }}-reg.sock'
]
env:
- name: ADDRESS
value: "{{ .Values.socketDir }}/{{ .Values.socketFile }}"
- name: DRIVER_REG_SOCK_PATH
value: "{{ .Values.socketDir }}/{{ .Values.socketFile }}"
- name: KUBE_NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
imagePullPolicy: {{ .Values.nodeplugin.registrar.image.imagePullPolicy }}
volumeMounts:
- name: socket-dir
mountPath: {{ .Values.socketDir }}
- name: plugin-dir
mountPath: /csi
- name: registration-dir
mountPath: /registration
resources:
@ -64,11 +68,14 @@ spec:
- "--nodeid=$(NODE_ID)"
- "--endpoint=$(CSI_ENDPOINT)"
- "--v=5"
- "--drivername=rbd.csi.ceph.com"
- "--containerized=true"
- "--drivername=$(DRIVER_NAME)"
- "--containerized=true"
- "--metadatastorage=k8s_configmap"
env:
- name: HOST_ROOTFS
value: "/rootfs"
- name: DRIVER_NAME
value: {{ .Values.driverName }}
- name: NODE_ID
valueFrom:
fieldRef:
@ -82,6 +89,9 @@ spec:
- name: pods-mount-dir
mountPath: /var/lib/kubelet/pods
mountPropagation: "Bidirectional"
- name: plugin-mount-dir
mountPath: {{ .Values.volumeDevicesDir }}
mountPropagation: "Bidirectional"
- mountPath: /dev
name: host-dev
- mountPath: /rootfs
@ -98,18 +108,18 @@ spec:
hostPath:
path: {{ .Values.socketDir }}
type: DirectoryOrCreate
- name: plugin-mount-dir
hostPath:
path: {{ .Values.volumeDevicesDir }}
type: DirectoryOrCreate
- name: registration-dir
hostPath:
path: /var/lib/kubelet/plugins/
path: {{ .Values.registrationDir }}
type: Directory
- name: pods-mount-dir
hostPath:
path: /var/lib/kubelet/pods
type: Directory
- name: socket-dir
hostPath:
path: {{ .Values.socketDir }}
type: DirectoryOrCreate
- name: host-dev
hostPath:
path: /dev

24
deploy/rbd/helm/templates/provisioner-clusterrole.yaml
View File

@ -10,6 +10,9 @@ metadata:
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
rules:
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get", "list"]
@ -25,4 +28,25 @@ rules:
- apiGroups: [""]
resources: ["events"]
verbs: ["list", "watch", "create", "update", "patch"]
- apiGroups: [""]
resources: ["endpoints"]
verbs: ["get", "create", "update"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshots"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get", "list", "create", "delete"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshotcontents"]
verbs: ["create", "get", "list", "watch", "update", "delete"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshotclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: ["apiextensions.k8s.io"]
resources: ["customresourcedefinitions"]
verbs: ["create"]
- apiGroups: ["csi.storage.k8s.io"]
resources: ["csinodeinfos"]
verbs: ["get", "list", "watch"]
{{- end -}}

16
deploy/rbd/helm/templates/provisioner-role.yaml Normal file
View File

@ -0,0 +1,16 @@
{{- if .Values.rbac.create -}}
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: {{ include "ceph-csi-rbd.provisioner.fullname" . }}
labels:
app: {{ include "ceph-csi-rbd.name" . }}
chart: {{ include "ceph-csi-rbd.chart" . }}
component: {{ .Values.provisioner.name }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
rules:
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get", "list", "watch", "create", "delete"]
{{- end -}}

21
deploy/rbd/helm/templates/provisioner-rolebinding.yaml Normal file
View File

@ -0,0 +1,21 @@
{{- if .Values.rbac.create -}}
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: {{ include "ceph-csi-rbd.provisioner.fullname" . }}
labels:
app: {{ include "ceph-csi-rbd.name" . }}
chart: {{ include "ceph-csi-rbd.chart" . }}
component: {{ .Values.provisioner.name }}
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
subjects:
- kind: ServiceAccount
name: {{ include "ceph-csi-rbd.serviceAccountName.provisioner" . }}
namespace: {{ .Release.Namespace }}
roleRef:
kind: Role
name: {{ include "ceph-csi-rbd.provisioner.fullname" . }}
apiGroup: rbac.authorization.k8s.io
namespace: {{ .Release.Namespace }}
{{- end -}}

62
deploy/rbd/helm/templates/provisioner-statefulset.yaml
View File

@ -11,6 +11,11 @@ metadata:
spec:
serviceName: {{ include "ceph-csi-rbd.provisioner.fullname" . }}
replicas: {{ .Values.provisioner.replicas }}
selector:
matchLabels:
app: {{ include "ceph-csi-rbd.name" . }}
component: {{ .Values.provisioner.name }}
release: {{ .Release.Name }}
template:
metadata:
labels:
@ -25,7 +30,6 @@ spec:
- name: csi-provisioner
image: "{{ .Values.provisioner.image.repository }}:{{ .Values.provisioner.image.tag }}"
args:
- "--provisioner=rbd.csi.ceph.com"
- "--csi-address=$(ADDRESS)"
- "--v=5"
env:
@ -37,11 +41,63 @@ spec:
mountPath: {{ .Values.socketDir }}
resources:
{{ toYaml .Values.provisioner.resources | indent 12 }}
- name: csi-snapshotter
image: {{ .Values.snapshotter.image.repository }}:{{ .Values.snapshotter.image.tag }}
imagePullPolicy: {{ .Values.nodeplugin.plugin.image.imagePullPolicy }}
args:
- "--csi-address=$(ADDRESS)"
- "--connection-timeout=15s"
- "--v=5"
env:
- name: ADDRESS
value: "{{ .Values.socketDir }}/{{ .Values.socketFile }}"
securityContext:
privileged: true
volumeMounts:
- name: socket-dir
mountPath: {{ .Values.socketDir }}
resources:
{{ toYaml .Values.snapshotter.resources | indent 12 }}
- name: csi-rbdplugin
securityContext:
privileged: true
capabilities:
add: ["SYS_ADMIN"]
allowPrivilegeEscalation: true
image: "{{ .Values.nodeplugin.plugin.image.repository }}:{{ .Values.nodeplugin.plugin.image.tag }}"
args :
- "--nodeid=$(NODE_ID)"
- "--endpoint=$(CSI_ENDPOINT)"
- "--v=5"
- "--drivername=$(DRIVER_NAME)"
- "--containerized=true"
- "--metadatastorage=k8s_configmap"
env:
- name: HOST_ROOTFS
value: "/rootfs"
- name: DRIVER_NAME
value: {{ .Values.driverName }}
- name: NODE_ID
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: CSI_ENDPOINT
value: "unix:/{{ .Values.socketDir }}/{{ .Values.socketFile }}"
imagePullPolicy: {{ .Values.nodeplugin.plugin.image.imagePullPolicy }}
volumeMounts:
- name: socket-dir
mountPath: {{ .Values.socketDir }}
- name: host-rootfs
mountPath: "/rootfs"
resources:
{{ toYaml .Values.nodeplugin.plugin.resources | indent 12 }}
volumes:
- name: socket-dir
emptyDir: {}
#FIXME this seems way too much. Why is it needed at all for this?
- name: host-rootfs
hostPath:
path: {{ .Values.socketDir }}
type: DirectoryOrCreate
path: /
{{- if .Values.provisioner.affinity -}}
affinity:
{{ toYaml .Values.provisioner.affinity . | indent 8 }}

26
deploy/rbd/helm/values.yaml
View File

@ -1,3 +1,4 @@
---
rbac:
create: true
@ -14,6 +15,9 @@ serviceAccounts:
socketDir: /var/lib/kubelet/plugins/rbd.csi.ceph.com
socketFile: csi.sock
registrationDir: /var/lib/kubelet/plugins_registry
volumeDevicesDir: /var/lib/kubelet/plugins/kubernetes.io/csi/volumeDevices
driverName: rbd.csi.ceph.com
attacher:
name: attacher
@ -22,7 +26,7 @@ attacher:
image:
repository: quay.io/k8scsi/csi-attacher
tag: v0.3.0
tag: v1.0.1
pullPolicy: IfNotPresent
resources: {}
@ -38,18 +42,18 @@ nodeplugin:
registrar:
image:
repository: quay.io/k8scsi/driver-registrar
tag: v0.3.0
repository: quay.io/k8scsi/csi-node-driver-registrar
tag: v1.0.2
pullPolicy: IfNotPresent
resources: {}
plugin:
image:
repository: quay.io/cephcsi/rbdplugin
tag: v0.3.0
tag: v1.0.0
pullPolicy: IfNotPresent
resources: {}
nodeSelector: {}
@ -65,7 +69,7 @@ provisioner:
image:
repository: quay.io/k8scsi/csi-provisioner
tag: v0.3.0
tag: v1.0.1
pullPolicy: IfNotPresent
resources: {}
@ -75,3 +79,11 @@ provisioner:
tolerations: []
affinity: {}
snapshotter:
image:
repository: quay.io/k8scsi/csi-snapshotter
tag: v1.0.1
pullPolicy: IfNotPresent
resources: {}

17
deploy/rbd/kubernetes/csi-attacher-rbac.yaml
View File

@ -1,17 +1,15 @@
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: csi-attacher
name: rbd-csi-attacher
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: external-attacher-runner
name: rbd-external-attacher-runner
rules:
- apiGroups: [""]
resources: ["events"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "update"]
@ -21,17 +19,20 @@ rules:
- apiGroups: ["storage.k8s.io"]
resources: ["volumeattachments"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["csi.storage.k8s.io"]
resources: ["csinodeinfos"]
verbs: ["get", "list", "watch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: csi-attacher-role
name: rbd-csi-attacher-role
subjects:
- kind: ServiceAccount
name: csi-attacher
name: rbd-csi-attacher
namespace: default
roleRef:
kind: ClusterRole
name: external-attacher-runner
name: rbd-external-attacher-runner
apiGroup: rbac.authorization.k8s.io

18
deploy/rbd/kubernetes/csi-nodeplugin-rbac.yaml
View File

@ -1,14 +1,18 @@
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: csi-nodeplugin
name: rbd-csi-nodeplugin
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: csi-nodeplugin
name: rbd-csi-nodeplugin
rules:
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get", "list"]
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get", "list", "update"]
@ -23,18 +27,18 @@ rules:
verbs: ["get", "list", "watch", "update"]
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get", "list", "create", "delete"]
verbs: ["get", "list"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: csi-nodeplugin
name: rbd-csi-nodeplugin
subjects:
- kind: ServiceAccount
name: csi-nodeplugin
name: rbd-csi-nodeplugin
namespace: default
roleRef:
kind: ClusterRole
name: csi-nodeplugin
apiGroup: rbac.authorization.k8s.io
name: rbd-csi-nodeplugin
apiGroup: rbac.authorization.k8s.io

69
deploy/rbd/kubernetes/csi-provisioner-rbac.yaml
View File

@ -1,17 +1,24 @@
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: csi-provisioner
name: rbd-csi-provisioner
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: external-provisioner-runner
name: rbd-external-provisioner-runner
rules:
- apiGroups: [""]
resources: ["nodes"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get", "list"]
- apiGroups: [""]
resources: ["events"]
verbs: ["list", "watch", "create", "update", "patch"]
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "create", "delete"]
@ -21,20 +28,64 @@ rules:
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["events"]
verbs: ["list", "watch", "create", "update", "patch"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshots"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshotcontents"]
verbs: ["create", "get", "list", "watch", "update", "delete"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshotclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: ["apiextensions.k8s.io"]
resources: ["customresourcedefinitions"]
verbs: ["create"]
- apiGroups: ["csi.storage.k8s.io"]
resources: ["csinodeinfos"]
verbs: ["get", "list", "watch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: csi-provisioner-role
name: rbd-csi-provisioner-role
subjects:
- kind: ServiceAccount
name: csi-provisioner
name: rbd-csi-provisioner
namespace: default
roleRef:
kind: ClusterRole
name: external-provisioner-runner
name: rbd-external-provisioner-runner
apiGroup: rbac.authorization.k8s.io
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
# replace with non-default namespace name
namespace: default
name: rbd-external-provisioner-cfg
rules:
- apiGroups: [""]
resources: ["endpoints"]
verbs: ["get", "watch", "list", "delete", "update", "create"]
- apiGroups: [""]
resources: ["configmaps"]
verbs: ["get", "list", "watch", "create", "delete"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: rbd-csi-provisioner-role-cfg
# replace with non-default namespace name
namespace: default
subjects:
- kind: ServiceAccount
name: rbd-csi-provisioner
# replace with non-default namespace name
namespace: default
roleRef:
kind: Role
name: rbd-external-provisioner-cfg
apiGroup: rbac.authorization.k8s.io

9
deploy/rbd/kubernetes/csi-rbdplugin-attacher.yaml
View File

@ -1,3 +1,4 @@
---
kind: Service
apiVersion: v1
metadata:
@ -24,20 +25,20 @@ spec:
labels:
app: csi-rbdplugin-attacher
spec:
serviceAccount: csi-attacher
serviceAccount: rbd-csi-attacher
containers:
- name: csi-rbdplugin-attacher
image: quay.io/k8scsi/csi-attacher:v0.3.0
image: quay.io/k8scsi/csi-attacher:v1.0.1
args:
- "--v=5"
- "--csi-address=$(ADDRESS)"
env:
- name: ADDRESS
value: /var/lib/kubelet/plugins/rbd.csi.ceph.com/csi.sock
value: unix:///csi/csi-attacher.sock
imagePullPolicy: "IfNotPresent"
volumeMounts:
- name: socket-dir
mountPath: /var/lib/kubelet/plugins/rbd.csi.ceph.com
mountPath: /csi
volumes:
- name: socket-dir
hostPath:

77
deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml
View File

@ -1,3 +1,4 @@
---
kind: Service
apiVersion: v1
metadata:
@ -24,22 +25,88 @@ spec:
labels:
app: csi-rbdplugin-provisioner
spec:
serviceAccount: csi-provisioner
serviceAccount: rbd-csi-provisioner
containers:
- name: csi-provisioner
image: quay.io/k8scsi/csi-provisioner:v0.3.0
image: quay.io/k8scsi/csi-provisioner:v1.0.1
args:
- "--provisioner=rbd.csi.ceph.com"
- "--csi-address=$(ADDRESS)"
- "--v=5"
env:
- name: ADDRESS
value: /var/lib/kubelet/plugins/rbd.csi.ceph.com/csi.sock
value: unix:///csi/csi-provisioner.sock
imagePullPolicy: "IfNotPresent"
volumeMounts:
- name: socket-dir
mountPath: /var/lib/kubelet/plugins/rbd.csi.ceph.com
mountPath: /csi
- name: csi-snapshotter
image: quay.io/k8scsi/csi-snapshotter:v1.0.1
args:
- "--csi-address=$(ADDRESS)"
- "--connection-timeout=15s"
- "--v=5"
env:
- name: ADDRESS
value: unix:///csi/csi-provisioner.sock
imagePullPolicy: Always
securityContext:
privileged: true
volumeMounts:
- name: socket-dir
mountPath: /csi
- name: csi-rbdplugin
securityContext:
privileged: true
capabilities:
add: ["SYS_ADMIN"]
image: quay.io/cephcsi/rbdplugin:v1.0.0
args:
- "--nodeid=$(NODE_ID)"
- "--endpoint=$(CSI_ENDPOINT)"
- "--v=5"
- "--drivername=rbd.csi.ceph.com"
- "--containerized=true"
- "--metadatastorage=k8s_configmap"
- "--configroot=k8s_objects"
env:
- name: HOST_ROOTFS
value: "/rootfs"
- name: NODE_ID
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: CSI_ENDPOINT
value: unix:///csi/csi-provisioner.sock
imagePullPolicy: "IfNotPresent"
volumeMounts:
- name: socket-dir
mountPath: /csi
- mountPath: /dev
name: host-dev
- mountPath: /rootfs
name: host-rootfs
- mountPath: /sys
name: host-sys
- mountPath: /lib/modules
name: lib-modules
readOnly: true
volumes:
- name: host-dev
hostPath:
path: /dev
- name: host-rootfs
hostPath:
path: /
- name: host-sys
hostPath:
path: /sys
- name: lib-modules
hostPath:
path: /lib/modules
- name: socket-dir
hostPath:
path: /var/lib/kubelet/plugins/rbd.csi.ceph.com

45
deploy/rbd/kubernetes/csi-rbdplugin.yaml
View File

@ -1,3 +1,4 @@
---
kind: DaemonSet
apiVersion: apps/v1beta2
metadata:
@ -11,7 +12,7 @@ spec:
labels:
app: csi-rbdplugin
spec:
serviceAccount: csi-nodeplugin
serviceAccount: rbd-csi-nodeplugin
hostNetwork: true
hostPID: true
# to use e.g. Rook orchestrated cluster, and mons' FQDN is
@ -19,23 +20,27 @@ spec:
dnsPolicy: ClusterFirstWithHostNet
containers:
- name: driver-registrar
image: quay.io/k8scsi/driver-registrar:v0.3.0
image: quay.io/k8scsi/csi-node-driver-registrar:v1.0.2
args:
- "--v=5"
- "--csi-address=$(ADDRESS)"
- "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)"
- "--csi-address=/csi/csi.sock"
- "--kubelet-registration-path=/var/lib/kubelet/plugins/rbd.csi.ceph.com/csi.sock"
lifecycle:
preStop:
exec:
command: [
"/bin/sh", "-c",
"rm -rf /registration/rbd.csi.ceph.com \
/registration/rbd.csi.ceph.com-reg.sock"
]
env:
- name: ADDRESS
value: /var/lib/kubelet/plugins/rbd.csi.ceph.com/csi.sock
- name: DRIVER_REG_SOCK_PATH
value: /var/lib/kubelet/plugins/rbd.csi.ceph.com/csi.sock
- name: KUBE_NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
volumeMounts:
- name: socket-dir
mountPath: /var/lib/kubelet/plugins/rbd.csi.ceph.com
- name: plugin-dir
mountPath: /csi
- name: registration-dir
mountPath: /registration
- name: csi-rbdplugin
@ -44,7 +49,7 @@ spec:
capabilities:
add: ["SYS_ADMIN"]
allowPrivilegeEscalation: true
image: quay.io/cephcsi/rbdplugin:v0.3.0
image: quay.io/cephcsi/rbdplugin:v1.0.0
args:
- "--nodeid=$(NODE_ID)"
- "--endpoint=$(CSI_ENDPOINT)"
@ -52,6 +57,7 @@ spec:
- "--drivername=rbd.csi.ceph.com"
- "--containerized=true"
- "--metadatastorage=k8s_configmap"
- "--configroot=k8s_objects"
env:
- name: HOST_ROOTFS
value: "/rootfs"
@ -64,14 +70,17 @@ spec:
fieldRef:
fieldPath: metadata.namespace
- name: CSI_ENDPOINT
value: unix://var/lib/kubelet/plugins/rbd.csi.ceph.com/csi.sock
value: unix:///csi/csi.sock
imagePullPolicy: "IfNotPresent"
volumeMounts:
- name: plugin-dir
mountPath: /var/lib/kubelet/plugins/rbd.csi.ceph.com
mountPath: /csi
- name: pods-mount-dir
mountPath: /var/lib/kubelet/pods
mountPropagation: "Bidirectional"
- name: plugin-mount-dir
mountPath: /var/lib/kubelet/plugins/kubernetes.io/csi/volumeDevices/
mountPropagation: "Bidirectional"
- mountPath: /dev
name: host-dev
- mountPath: /rootfs
@ -86,18 +95,18 @@ spec:
hostPath:
path: /var/lib/kubelet/plugins/rbd.csi.ceph.com
type: DirectoryOrCreate
- name: plugin-mount-dir
hostPath:
path: /var/lib/kubelet/plugins/kubernetes.io/csi/volumeDevices/
type: DirectoryOrCreate
- name: registration-dir
hostPath:
path: /var/lib/kubelet/plugins/
path: /var/lib/kubelet/plugins_registry/
type: Directory
- name: pods-mount-dir
hostPath:
path: /var/lib/kubelet/pods
type: Directory
- name: socket-dir
hostPath:
path: /var/lib/kubelet/plugins/rbd.csi.ceph.com
type: DirectoryOrCreate
- name: host-dev
hostPath:
path: /dev

72
docs/deploy-cephfs.md
View File

@ -1,10 +1,15 @@
# CSI CephFS plugin
The CSI CephFS plugin is able to both provision new CephFS volumes and attach and mount existing ones to workloads.
The CSI CephFS plugin is able to both provision new CephFS volumes
and attach and mount existing ones to workloads.
## Building
CSI CephFS plugin can be compiled in the form of a binary file or in the form of a Docker image. When compiled as a binary file, the result is stored in `_output/` directory with the name `cephfsplugin`. When compiled as an image, it's stored in the local Docker image store.
CSI CephFS plugin can be compiled in the form of a binary file or in the form
of a Docker image.
When compiled as a binary file, the result is stored in `_output/`
directory with the name `cephfsplugin`.
When compiled as an image, it's stored in the local Docker image store.
Building binary:
@ -29,44 +34,56 @@ Option | Default value | Description
`--nodeid` | _empty_ | This node's ID
`--volumemounter` | _empty_ | default volume mounter. Available options are `kernel` and `fuse`. This is the mount method used if volume parameters don't specify otherwise. If left unspecified, the driver will first probe for `ceph-fuse` in system's path and will choose Ceph kernel client if probing failed.
`--metadatastorage` | _empty_ | Whether metadata should be kept on node as file or in a k8s configmap (`node` or `k8s_configmap`)
`--mountcachedir` | _empty_ | volume mount cache info save dir. If left unspecified, the dirver will not record mount info, or it will save mount info and when driver restart it will remount volume it cached.
**Available environmental variables:**
`KUBERNETES_CONFIG_PATH`: if you use `k8s_configmap` as metadata store, specify the path of your k8s config file (if not specified, the plugin will assume you're running it inside a k8s cluster and find the config itself).
`KUBERNETES_CONFIG_PATH`: if you use `k8s_configmap` as metadata store, specify
the path of your k8s config file (if not specified, the plugin will assume
you're running it inside a k8s cluster and find the config itself).
`POD_NAMESPACE`: if you use `k8s_configmap` as metadata store, `POD_NAMESPACE` is used to define in which namespace you want the configmaps to be stored
`POD_NAMESPACE`: if you use `k8s_configmap` as metadata store, `POD_NAMESPACE`
is used to define in which namespace you want the configmaps to be stored
**Available volume parameters:**
Parameter | Required | Description
---------------------------------------------------------------|-----------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
`monitors` | yes | Comma separated list of Ceph monitors (e.g. `192.168.100.1:6789,192.168.100.2:6789,192.168.100.3:6789`)
`mounter` | no | Mount method to be used for this volume. Available options are `kernel` for Ceph kernel client and `fuse` for Ceph FUSE driver. Defaults to "default mounter", see command line arguments.
`provisionVolume` | yes | Mode of operation. BOOL value. If `true`, a new CephFS volume will be provisioned. If `false`, an existing volume will be used.
`pool` | for `provisionVolume=true` | Ceph pool into which the volume shall be created
`rootPath` | for `provisionVolume=false` | Root path of an existing CephFS volume
`csiProvisionerSecretName`, `csiNodeStageSecretName` | for Kubernetes | name of the Kubernetes Secret object containing Ceph client credentials. Both parameters should have the same value
`csiProvisionerSecretNamespace`, `csiNodeStageSecretNamespace` | for Kubernetes | namespaces of the above Secret objects
Parameter | Required | Description
----------------------------------------------------------------------------------------------------|--------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
`monitors` | yes | Comma separated list of Ceph monitors (e.g. `192.168.100.1:6789,192.168.100.2:6789,192.168.100.3:6789`)
`monValueFromSecret` | one of `monitors` and `monValueFromSecret` must be set | a string pointing the key in the credential secret, whose value is the mon. This is used for the case when the monitors' IP or hostnames are changed, the secret can be updated to pick up the new monitors. If both `monitors` and `monValueFromSecret` are set and the monitors set in the secret exists, `monValueFromSecret` takes precedence.
`mounter` | no | Mount method to be used for this volume. Available options are `kernel` for Ceph kernel client and `fuse` for Ceph FUSE driver. Defaults to "default mounter", see command line arguments.
`provisionVolume` | yes | Mode of operation. BOOL value. If `true`, a new CephFS volume will be provisioned. If `false`, an existing volume will be used.
`pool` | for `provisionVolume=true` | Ceph pool into which the volume shall be created
`rootPath` | for `provisionVolume=false` | Root path of an existing CephFS volume
`csi.storage.k8s.io/provisioner-secret-name`, `csi.storage.k8s.io/node-stage-secret-name` | for Kubernetes | name of the Kubernetes Secret object containing Ceph client credentials. Both parameters should have the same value
`csi.storage.k8s.io/provisioner-secret-namespace`, `csi.storage.k8s.io/node-stage-secret-namespace` | for Kubernetes | namespaces of the above Secret objects
**Required secrets for `provisionVolume=true`:**
**Required secrets for `provisionVolume=true`:**
Admin credentials are required for provisioning new volumes
* `adminID`: ID of an admin client
* `adminKey`: key of the admin client
**Required secrets for `provisionVolume=false`:**
**Required secrets for `provisionVolume=false`:**
User credentials with access to an existing volume
* `userID`: ID of a user client
* `userKey`: key of a user client
Notes on volume size: when provisioning a new volume, `max_bytes` quota attribute for this volume will be set to the requested volume size (see [Ceph quota documentation](http://docs.ceph.com/docs/mimic/cephfs/quota/)). A request for a zero-sized volume means no quota attribute will be set.
Notes on volume size: when provisioning a new volume, `max_bytes` quota
attribute for this volume will be set to the requested volume size (see [Ceph
quota documentation](http://docs.ceph.com/docs/mimic/cephfs/quota/)). A request
for a zero-sized volume means no quota attribute will be set.
## Deployment with Kubernetes
Requires Kubernetes 1.11
Requires Kubernetes 1.13
Your Kubernetes cluster must allow privileged pods (i.e. `--allow-privileged` flag must be set to true for both the API server and the kubelet). Moreover, as stated in the [mount propagation docs](https://kubernetes.io/docs/concepts/storage/volumes/#mount-propagation), the Docker daemon of the cluster nodes must allow shared mounts.
Your Kubernetes cluster must allow privileged pods (i.e. `--allow-privileged`
flag must be set to true for both the API server and the kubelet). Moreover, as
stated in the [mount propagation
docs](https://kubernetes.io/docs/concepts/storage/volumes/#mount-propagation),
the Docker daemon of the cluster nodes must allow shared mounts.
YAML manifests are located in `deploy/cephfs/kubernetes`.
@ -78,7 +95,9 @@ kubectl create -f csi-provisioner-rbac.yaml
kubectl create -f csi-nodeplugin-rbac.yaml
```
Those manifests deploy service accounts, cluster roles and cluster role bindings. These are shared for both RBD and CephFS CSI plugins, as they require the same permissions.
Those manifests deploy service accounts, cluster roles and cluster role
bindings. These are shared for both RBD and CephFS CSI plugins, as they require
the same permissions.
**Deploy CSI sidecar containers:**
@ -87,7 +106,8 @@ kubectl create -f csi-cephfsplugin-attacher.yaml
kubectl create -f csi-cephfsplugin-provisioner.yaml
```
Deploys stateful sets for external-attacher and external-provisioner sidecar containers for CSI CephFS.
Deploys stateful sets for external-attacher and external-provisioner
sidecar containers for CSI CephFS.
**Deploy CSI CephFS driver:**
@ -95,14 +115,15 @@ Deploys stateful sets for external-attacher and external-provisioner sidecar con
kubectl create -f csi-cephfsplugin.yaml
```
Deploys a daemon set with two containers: CSI driver-registrar and the CSI CephFS driver.
Deploys a daemon set with two containers: CSI driver-registrar and
the CSI CephFS driver.
## Verifying the deployment in Kubernetes
After successfuly completing the steps above, you should see output similar to this:
After successfully completing the steps above, you should see output similar to this:
```bash
kubectl get all
$ kubectl get all
NAME READY STATUS RESTARTS AGE
pod/csi-cephfsplugin-attacher-0 1/1 Running 0 26s
pod/csi-cephfsplugin-provisioner-0 1/1 Running 0 25s
@ -119,4 +140,7 @@ You can try deploying a demo pod from `examples/cephfs` to test the deployment f
### Notes on volume deletion
Volumes that were provisioned dynamically (i.e. `provisionVolume=true`) are allowed to be deleted by the driver as well, if the user chooses to do so. Otherwise, the driver is forbidden to delete such volumes - attempting to delete them is a no-op.
Volumes that were provisioned dynamically (i.e. `provisionVolume=true`) are
allowed to be deleted by the driver as well, if the user chooses to do
so.Otherwise, the driver is forbidden to delete such volumes - attempting to
delete them is a no-op.

85
docs/deploy-rbd.md
View File

@ -1,10 +1,14 @@
# CSI RBD Plugin
The RBD CSI plugin is able to provision new RBD images and attach and mount those to workloads.
The RBD CSI plugin is able to provision new RBD images and
attach and mount those to workloads.
## Building
CSI RBD plugin can be compiled in a form of a binary file or in a form of a Docker image. When compiled as a binary file, the result is stored in `_output/` directory with the name `rbdplugin`. When compiled as an image, it's stored in the local Docker image store.
CSI RBD plugin can be compiled in a form of a binary file or in a form of a
Docker image. When compiled as a binary file, the result is stored in
`_output/` directory with the name `rbdplugin`. When compiled as an image, it's
stored in the local Docker image store.
Building binary:
@ -22,45 +26,67 @@ make image-rbdplugin
**Available command line arguments:**
Option | Default value | Description
--------------------|-----------------------|---------------------------------------------------------------------------------------------------
`--endpoint` | `unix://tmp/csi.sock` | CSI endpoint, must be a UNIX socket
`--drivername` | `rbd.csi.ceph.com` | name of the driver (Kubernetes: `provisioner` field in StorageClass must correspond to this value)
`--nodeid` | _empty_ | This node's ID
`--containerized` | true | Whether running in containerized mode
`--metadatastorage` | _empty_ | Whether should metadata be kept on node as file or in a k8s configmap (`node` or `k8s_configmap`)
Option | Default value | Description
------ | ------------- | -----------
`--endpoint` | `unix://tmp/csi.sock` | CSI endpoint, must be a UNIX socket
`--drivername` | `rbd.csi.ceph.com` | name of the driver (Kubernetes: `provisioner` field in StorageClass must correspond to this value)
`--nodeid` | _empty_ | This node's ID
`--containerized` | true | Whether running in containerized mode
`--metadatastorage` | _empty_ | Whether should metadata be kept on node as file or in a k8s configmap (`node` or `k8s_configmap`)
`--configroot` | `/etc/csi-config` | Directory in which CSI specific Ceph cluster configurations are present, OR the value `k8s_objects` if present as kubernetes secrets"
**Available environmental variables:**
`HOST_ROOTFS`: rbdplugin searches `/proc` directory under the directory set by `HOST_ROOTFS`.
`KUBERNETES_CONFIG_PATH`: if you use `k8s_configmap` as metadata store, specify the path of your k8s config file (if not specified, the plugin will assume you're running it inside a k8s cluster and find the config itself).
`KUBERNETES_CONFIG_PATH`: if you use `k8s_configmap` as metadata store, specify
the path of your k8s config file (if not specified, the plugin will assume
you're running it inside a k8s cluster and find the config itself).
`POD_NAMESPACE`: if you use `k8s_configmap` as metadata store, `POD_NAMESPACE` is used to define in which namespace you want the configmaps to be stored
`POD_NAMESPACE`: if you use `k8s_configmap` as metadata store,
`POD_NAMESPACE` is used to define in which namespace you want
the configmaps to be stored
**Available volume parameters:**
Parameter | Required | Description
-----------------------------------------------------------------|--------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
`monitors` | one of `monitors` and `monValueFromSecret` must be set | Comma separated list of Ceph monitors (e.g. `192.168.100.1:6789,192.168.100.2:6789,192.168.100.3:6789`)
`monValueFromSecret` | one of `monitors` and `monValueFromSecret` must be set | a string pointing the key in the credential secret, whose value is the mon. This is used for the case when the monitors' IP or hostnames are changed, the secret can be updated to pick up the new monitors.
`pool` | yes | Ceph pool into which the RBD image shall be created
`imageFormat` | no | RBD image format. Defaults to `2`. See [man pages](http://docs.ceph.com/docs/mimic/man/8/rbd/#cmdoption-rbd-image-format)
`imageFeatures` | no | RBD image features. Available for `imageFormat=2`. CSI RBD currently supports only `layering` feature. See [man pages](http://docs.ceph.com/docs/mimic/man/8/rbd/#cmdoption-rbd-image-feature)
`csiProvisionerSecretName`, `csiNodePublishSecretName` | for Kubernetes | name of the Kubernetes Secret object containing Ceph client credentials. Both parameters should have the same value
`csiProvisionerSecretNamespace`, `csiNodePublishSecretNamespace` | for Kubernetes | namespaces of the above Secret objects
`mounter` | no | if set to `rbd-nbd`, use `rbd-nbd` on nodes that have `rbd-nbd` and `nbd` kernel modules to map rbd images
Parameter | Required | Description
--------- | -------- | -----------
`monitors` | one of `monitors`, `clusterID` or `monValueFromSecret` must be set | Comma separated list of Ceph monitors (e.g. `192.168.100.1:6789,192.168.100.2:6789,192.168.100.3:6789`)
`monValueFromSecret` | one of `monitors`, `clusterID` or and `monValueFromSecret` must be set | a string pointing the key in the credential secret, whose value is the mon. This is used for the case when the monitors' IP or hostnames are changed, the secret can be updated to pick up the new monitors.
`clusterID` | one of `monitors`, `clusterID` or `monValueFromSecret` must be set | String representing a Ceph cluster, must be unique across all Ceph clusters in use for provisioning, cannot be greater than 36 bytes in length, and should remain immutable for the lifetime of the Ceph cluster in use
`pool` | yes | Ceph pool into which the RBD image shall be created
`imageFormat` | no | RBD image format. Defaults to `2`. See [man pages](http://docs.ceph.com/docs/mimic/man/8/rbd/#cmdoption-rbd-image-format)
`imageFeatures` | no | RBD image features. Available for `imageFormat=2`. CSI RBD currently supports only `layering` feature. See [man pages](http://docs.ceph.com/docs/mimic/man/8/rbd/#cmdoption-rbd-image-feature)
`csi.storage.k8s.io/provisioner-secret-name`, `csi.storage.k8s.io/node-publish-secret-name` | for Kubernetes | name of the Kubernetes Secret object containing Ceph client credentials. Both parameters should have the same value
`csi.storage.k8s.io/provisioner-secret-namespace`, `csi.storage.k8s.io/node-publish-secret-namespace` | for Kubernetes | namespaces of the above Secret objects
`mounter`| no | if set to `rbd-nbd`, use `rbd-nbd` on nodes that have `rbd-nbd` and `nbd` kernel modules to map rbd images
**Required secrets:**
Admin credentials are required for provisioning new RBD images
`ADMIN_NAME`: `ADMIN_PASSWORD` - note that the key of the key-value pair is the name of the client with admin privileges, and the value is its password
NOTE: If `clusterID` parameter is used, then an accompanying Ceph cluster
configuration secret or config files needs to be provided to the running pods.
Refer to [Cluster ID based configuration](../examples/README.md#cluster-id-based-configuration)
for more information. A suggested way to populate the clusterID is to use the
output of `ceph fsid` of the Ceph cluster to be used for provisioning.
**Required secrets:**
Admin credentials are required for provisioning new RBD images `ADMIN_NAME`:
`ADMIN_PASSWORD` - note that the key of the key-value pair is the name of the
client with admin privileges, and the value is its password
If clusterID is specified, then a secret with various keys and values as
specified in `examples/rbd/template-ceph-cluster-ID-secret.yaml` needs to be
created, with the secret name matching the string value provided as the
`clusterID`.
## Deployment with Kubernetes
Requires Kubernetes 1.11
Your Kubernetes cluster must allow privileged pods (i.e. `--allow-privileged` flag must be set to true for both the API server and the kubelet). Moreover, as stated in the [mount propagation docs](https://kubernetes.io/docs/concepts/storage/volumes/#mount-propagation), the Docker daemon of the cluster nodes must allow shared mounts.
Your Kubernetes cluster must allow privileged pods (i.e. `--allow-privileged`
flag must be set to true for both the API server and the kubelet). Moreover, as
stated in the [mount propagation
docs](https://kubernetes.io/docs/concepts/storage/volumes/#mount-propagation),
the Docker daemon of the cluster nodes must allow shared mounts.
YAML manifests are located in `deploy/rbd/kubernetes`.
@ -72,7 +98,9 @@ kubectl create -f csi-provisioner-rbac.yaml
kubectl create -f csi-nodeplugin-rbac.yaml
```
Those manifests deploy service accounts, cluster roles and cluster role bindings. These are shared for both RBD and CephFS CSI plugins, as they require the same permissions.
Those manifests deploy service accounts, cluster roles and cluster role
bindings. These are shared for both RBD and CephFS CSI plugins, as they require
the same permissions.
**Deploy CSI sidecar containers:**
@ -81,7 +109,8 @@ kubectl create -f csi-rbdplugin-attacher.yaml
kubectl create -f csi-rbdplugin-provisioner.yaml
```
Deploys stateful sets for external-attacher and external-provisioner sidecar containers for CSI RBD.
Deploys stateful sets for external-attacher and external-provisioner
sidecar containers for CSI RBD.
**Deploy RBD CSI driver:**
@ -93,7 +122,7 @@ Deploys a daemon set with two containers: CSI driver-registrar and the CSI RBD d
## Verifying the deployment in Kubernetes
After successfuly completing the steps above, you should see output similar to this:
After successfully completing the steps above, you should see output similar to this:
```bash
$ kubectl get all

241
examples/README.md
View File

@ -1,10 +1,24 @@
# How to test RBD and CephFS plugins with Kubernetes 1.11
# How to test RBD and CephFS plugins with Kubernetes 1.13
Both `rbd` and `cephfs` directories contain `plugin-deploy.sh` and `plugin-teardown.sh` helper scripts. You can use those to help you deploy/tear down RBACs, sidecar containers and the plugin in one go. By default, they look for the YAML manifests in `../../deploy/{rbd,cephfs}/kubernetes`. You can override this path by running `$ ./plugin-deploy.sh /path/to/my/manifests`.
Both `rbd` and `cephfs` directories contain `plugin-deploy.sh` and
`plugin-teardown.sh` helper scripts. You can use those to help you
deploy/teardown RBACs, sidecar containers and the plugin in one go.
By default, they look for the YAML manifests in
`../../deploy/{rbd,cephfs}/kubernetes`.
You can override this path by running `$ ./plugin-deploy.sh /path/to/my/manifests`.
Once the plugin is successfully deployed, you'll need to customize `storageclass.yaml` and `secret.yaml` manifests to reflect your Ceph cluster setup. Please consult the documentation for info about available parameters.
Once the plugin is successfully deployed, you'll need to customize
`storageclass.yaml` and `secret.yaml` manifests to reflect your Ceph cluster
setup.
Please consult the documentation for info about available parameters.
After configuring the secrets, monitors, etc. you can deploy a testing Pod mounting a RBD image / CephFS volume:
**NOTE:** See section
[Cluster ID based configuration](#cluster-id-based-configuration) if using
the `clusterID` instead of `monitors` or `monValueFromSecret` option in the
storage class for RBD based provisioning before proceeding.
After configuring the secrets, monitors, etc. you can deploy a
testing Pod mounting a RBD image / CephFS volume:
```bash
kubectl create -f secret.yaml
@ -20,42 +34,221 @@ Other helper scripts:
## How to test RBD Snapshot feature
Before continuing, make sure you enabled the required [feature gate](https://kubernetes-csi.github.io/docs/snapshot-restore-feature.html#snapshot-apis) in your Kubernetes cluster.
Before continuing, make sure you enabled the required
feature gate `VolumeSnapshotDataSource=true` in your Kubernetes cluster.
In the `examples/rbd` directory you will find four files related to snapshots: [csi-snapshotter-rbac.yaml](./rbd/csi-snapshotter-rbac.yaml), [csi-snapshotter.yaml](./rbd/csi-snapshotter.yaml), [snapshotclass.yaml](./rbd/snapshotclass.yaml) and [snapshot.yaml](./rbd/snapshot.yaml).
In the `examples/rbd` directory you will find two files related to snapshots:
[snapshotclass.yaml](./rbd/snapshotclass.yaml) and
[snapshot.yaml](./rbd/snapshot.yaml).
Once you created your RBD volume, you'll need to customize at least `snapshotclass.yaml` and make sure the `monitors` and `pool` parameters match your Ceph cluster setup. If you followed the documentation to create the rbdplugin, you shouldn't have to edit any other file. If you didn't, make sure every parameters in `csi-snapshotter.yaml` reflect your configuration.
Once you created your RBD volume, you'll need to customize at least
`snapshotclass.yaml` and make sure the `monitors` and `pool` parameters match
your Ceph cluster setup.
If you followed the documentation to create the rbdplugin, you shouldn't
have to edit any other file.
After configuring everything you needed, deploy the csi-snapshotter:
After configuring everything you needed, deploy the snapshot class:
```bash
kubectl create -f csi-snapshotter-rbac.yaml
kubectl create -f csi-snapshotter.yaml
kubectl create -f snapshotclass.yaml
```
Verify that the snapshot class was created:
```console
$ kubectl get volumesnapshotclass
NAME AGE
csi-rbdplugin-snapclass 4s
```
Create a snapshot from the existing PVC:
```bash
kubectl create -f snapshot.yaml
```
To verify if your volume snapshot has successfully been created, run the following:
```bash
$ kubectl get volumesnapshotclass
NAME AGE
csi-rbdplugin-snapclass 4s
```console
$ kubectl get volumesnapshot
NAME AGE
rbd-pvc-snapshot 6s
```
To be sure everything is OK you can run `rbd snap ls [your-pvc-name]` inside one of your Ceph pod.
## How to test RBD Block Volume
Before continuing, make sure you enabled the required [feature gate](https://kubernetes-csi.github.io/docs/raw-block.html#usage) in your Kubernetes cluster.
In order to create a PVC in Block mode, you need to specify `volumeMode: Block` in the PVC spec.
In the `examples/rbd` directory, you can test RBD block volume by running:
To check the status of the snapshot, run the following:
```bash
kubectl create -f pvc-block.yaml
kubectl create -f pod-block.yaml
$ kubectl describe volumesnapshot rbd-pvc-snapshot
Name: rbd-pvc-snapshot
Namespace: default
Labels: <none>
Annotations: <none>
API Version: snapshot.storage.k8s.io/v1alpha1
Kind: VolumeSnapshot
Metadata:
Creation Timestamp: 2019-02-06T08:52:34Z
Finalizers:
snapshot.storage.kubernetes.io/volumesnapshot-protection
Generation: 5
Resource Version: 84239
Self Link: /apis/snapshot.storage.k8s.io/v1alpha1/namespaces/default/volumesnapshots/rbd-pvc-snapshot
UID: 8b9b5740-29ec-11e9-8e0f-b8ca3aad030b
Spec:
Snapshot Class Name: csi-rbdplugin-snapclass
Snapshot Content Name: snapcontent-8b9b5740-29ec-11e9-8e0f-b8ca3aad030b
Source:
API Group: <nil>
Kind: PersistentVolumeClaim
Name: rbd-pvc
Status:
Creation Time: 2019-02-06T08:52:34Z
Ready To Use: true
Restore Size: 1Gi
Events: <none>
```
To be sure everything is OK you can run `rbd snap ls [your-pvc-name]` inside
one of your Ceph pod.
To restore the snapshot to a new PVC, deploy
[pvc-restore.yaml](./rbd/pvc-restore.yaml) and a testing pod
[pod-restore.yaml](./rbd/pod-restore.yaml):
```bash
kubectl create -f pvc-restore.yaml
kubectl create -f pod-restore.yaml
```
## How to test RBD MULTI_NODE_MULTI_WRITER BLOCK feature
Requires feature-gates: `BlockVolume=true` `CSIBlockVolume=true`
*NOTE* The MULTI_NODE_MULTI_WRITER capability is only available for
Volumes that are of access_type `block`
*WARNING* This feature is strictly for workloads that know how to deal
with concurrent access to the Volume (eg Active/Passive applications).
Using RWX modes on non clustered file systems with applications trying
to simultaneously access the Volume will likely result in data corruption!
Following are examples for issuing a request for a `Block`
`ReadWriteMany` Claim, and using the resultant Claim for a POD
```yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: block-pvc
spec:
accessModes:
- ReadWriteMany
volumeMode: Block
resources:
requests:
storage: 1Gi
storageClassName: csi-rbd
```
Create a POD that uses this PVC:
```yaml
apiVersion: v1
kind: Pod
metadata:
name: my-pod
spec:
containers:
- name: my-container
image: debian
command: ["/bin/bash", "-c"]
args: [ "tail -f /dev/null" ]
volumeDevices:
- devicePath: /dev/rbdblock
name: my-volume
imagePullPolicy: IfNotPresent
volumes:
- name: my-volume
persistentVolumeClaim:
claimName: block-pvc
```
Now, we can create a second POD (ensure the POD is scheduled on a different
node; multiwriter single node works without this feature) that also uses this
PVC at the same time, again wait for the pod to enter running state, and verify
the block device is available.
```yaml
apiVersion: v1
kind: Pod
metadata:
name: another-pod
spec:
containers:
- name: my-container
image: debian
command: ["/bin/bash", "-c"]
args: [ "tail -f /dev/null" ]
volumeDevices:
- devicePath: /dev/rbdblock
name: my-volume
imagePullPolicy: IfNotPresent
volumes:
- name: my-volume
persistentVolumeClaim:
claimName: block-pvc
```
Wait for the PODs to enter Running state, check that our block device
is available in the container at `/dev/rdbblock` in both containers:
```bash
$ kubectl exec -it my-pod -- fdisk -l /dev/rbdblock
Disk /dev/rbdblock: 1 GiB, 1073741824 bytes, 2097152 sectors
Units: sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 4194304 bytes / 4194304 bytes
```
```bash
$ kubectl exec -it another-pod -- fdisk -l /dev/rbdblock
Disk /dev/rbdblock: 1 GiB, 1073741824 bytes, 2097152 sectors
Units: sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 4194304 bytes / 4194304 bytes
```
## Cluster ID based configuration
Before creating a storage class that uses the option `clusterID` to refer to a
Ceph cluster, the following actions need to be completed.
Get the following information from the Ceph cluster,
* Admin ID and key, that has privileges to perform CRUD operations on the Ceph
cluster and pools of choice
* Key is typically the output of, `ceph auth get-key client.admin` where
`admin` is the Admin ID
* Used to substitute admin/user id and key values in the files below
* Ceph monitor list
* Typically in the output of `ceph mon dump`
* Used to prepare comma separated MON list where required in the files below
* Ceph Cluster fsid
* If choosing to use the Ceph cluster fsid as the unique value of clusterID,
* Output of `ceph fsid`
* Used to substitute `<cluster-id>` references in the files below
Update the template
[template-ceph-cluster-ID-secret.yaml](./rbd/template-ceph-cluster-ID-secret.yaml)
with values from
a Ceph cluster and replace `<cluster-id>` with the chosen clusterID to create
the following secret,
* `kubectl create -f rbd/template-ceph-cluster-ID-secret.yaml`
Storage class and snapshot class, using `<cluster-id>` as the value for the
option `clusterID`, can now be created on the cluster.
Remaining steps to test functionality remains the same as mentioned in the
sections above.

19
examples/cephfs/deployment.yaml
View File

@ -1,3 +1,4 @@
---
apiVersion: apps/v1
kind: Deployment
metadata:
@ -15,13 +16,13 @@ spec:
app: web-server
spec:
containers:
- name: web-server
image: nginx
volumeMounts:
- name: mypvc
mountPath: /var/lib/www/html
- name: web-server
image: nginx
volumeMounts:
- name: mypvc
mountPath: /var/lib/www/html
volumes:
- name: mypvc
persistentVolumeClaim:
claimName: csi-cephfs-pvc
readOnly: false
- name: mypvc
persistentVolumeClaim:
claimName: csi-cephfs-pvc
readOnly: false

4
examples/cephfs/exec-bash.sh
View File

@ -4,7 +4,7 @@ CONTAINER_NAME=csi-cephfsplugin
POD_NAME=$(kubectl get pods -l app=$CONTAINER_NAME -o=name | head -n 1)
function get_pod_status() {
echo -n $(kubectl get $POD_NAME -o jsonpath="{.status.phase}")
echo -n "$(kubectl get "$POD_NAME" -o jsonpath="{.status.phase}")"
}
while [[ "$(get_pod_status)" != "Running" ]]; do
@ -12,4 +12,4 @@ while [[ "$(get_pod_status)" != "Running" ]]; do
echo "Waiting for $POD_NAME (status $(get_pod_status))"
done
kubectl exec -it ${POD_NAME#*/} -c $CONTAINER_NAME bash
kubectl exec -it "${POD_NAME#*/}" -c "$CONTAINER_NAME" bash

4
examples/cephfs/logs.sh
View File

@ -4,7 +4,7 @@ CONTAINER_NAME=csi-cephfsplugin
POD_NAME=$(kubectl get pods -l app=$CONTAINER_NAME -o=name | head -n 1)
function get_pod_status() {
echo -n $(kubectl get $POD_NAME -o jsonpath="{.status.phase}")
echo -n "$(kubectl get "$POD_NAME" -o jsonpath="{.status.phase}")"
}
while [[ "$(get_pod_status)" != "Running" ]]; do
@ -12,4 +12,4 @@ while [[ "$(get_pod_status)" != "Running" ]]; do
echo "Waiting for $POD_NAME (status $(get_pod_status))"
done
kubectl logs -f $POD_NAME -c $CONTAINER_NAME
kubectl logs -f "$POD_NAME" -c "$CONTAINER_NAME"

2
examples/cephfs/plugin-deploy.sh
View File

@ -10,6 +10,6 @@ cd "$deployment_base" || exit 1
objects=(csi-attacher-rbac csi-provisioner-rbac csi-nodeplugin-rbac csi-cephfsplugin-attacher csi-cephfsplugin-provisioner csi-cephfsplugin)
for obj in ${objects[@]}; do
for obj in "${objects[@]}"; do
kubectl create -f "./$obj.yaml"
done

2
examples/cephfs/plugin-teardown.sh
View File

@ -10,6 +10,6 @@ cd "$deployment_base" || exit 1
objects=(csi-cephfsplugin-attacher csi-cephfsplugin-provisioner csi-cephfsplugin csi-attacher-rbac csi-provisioner-rbac csi-nodeplugin-rbac)
for obj in ${objects[@]}; do
for obj in "${objects[@]}"; do
kubectl delete -f "./$obj.yaml"
done

20
examples/cephfs/pod.yaml
View File

@ -1,17 +1,17 @@
---
apiVersion: v1
kind: Pod
metadata:
name: csicephfs-demo-pod
spec:
containers:
- name: web-server
image: nginx
volumeMounts:
- name: mypvc
mountPath: /var/lib/www
- name: web-server
image: nginx
volumeMounts:
- name: mypvc
mountPath: /var/lib/www
volumes:
- name: mypvc
persistentVolumeClaim:
claimName: csi-cephfs-pvc
readOnly: false
- name: mypvc
persistentVolumeClaim:
claimName: csi-cephfs-pvc
readOnly: false

3
examples/cephfs/pvc.yaml
View File

@ -1,10 +1,11 @@
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: csi-cephfs-pvc
spec:
accessModes:
- ReadWriteMany
- ReadWriteMany
resources:
requests:
storage: 5Gi

1
examples/cephfs/secret.yaml
View File

@ -1,3 +1,4 @@
---
apiVersion: v1
kind: Secret
metadata:

25
examples/cephfs/storageclass.yaml
View File

@ -1,11 +1,12 @@
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: csi-cephfs
provisioner: csi-cephfsplugin
provisioner: cephfs.csi.ceph.com
parameters:
# Comma separated list of Ceph monitors
# if using FQDN, make sure csi plugin's dns policy is appropriate.
# if using FQDN, make sure csi plugin's dns policy is appropriate.
monitors: mon1:port,mon2:port,...
# For provisionVolume: "true":
@ -13,7 +14,8 @@ parameters:
# Requires admin credentials (adminID, adminKey).
# For provisionVolume: "false":
# It is assumed the volume already exists and the user is expected
# to provide path to that volume (rootPath) and user credentials (userID, userKey).
# to provide path to that volume (rootPath) and user credentials
# (userID, userKey).
provisionVolume: "true"
# Ceph pool into which the volume shall be created
@ -25,13 +27,16 @@ parameters:
# rootPath: /absolute/path
# The secrets have to contain user and/or Ceph admin credentials.
csiProvisionerSecretName: csi-cephfs-secret
csiProvisionerSecretNamespace: default
csiNodeStageSecretName: csi-cephfs-secret
csiNodeStageSecretNamespace: default
csi.storage.k8s.io/provisioner-secret-name: csi-cephfs-secret
csi.storage.k8s.io/provisioner-secret-namespace: default
csi.storage.k8s.io/node-stage-secret-name: csi-cephfs-secret
csi.storage.k8s.io/node-stage-secret-namespace: default
# (optional) The driver can use either ceph-fuse (fuse) or ceph kernel client (kernel)
# If omitted, default volume mounter will be used - this is determined by probing for ceph-fuse
# or by setting the default mounter explicitly via --volumemounter command-line argument.
# (optional) The driver can use either ceph-fuse (fuse) or
# ceph kernelclient (kernel).
# If omitted, default volume mounter will be used - this is
# determined by probing for ceph-fuse
# or by setting the default mounter explicitly via
# --volumemounter command-line argument.
# mounter: kernel
reclaimPolicy: Delete

45
examples/rbd/csi-snapshotter-rbac.yaml
View File

@ -1,45 +0,0 @@
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: external-snapshotter-runner
rules:
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["events"]
verbs: ["list", "watch", "create", "update", "patch"]
- apiGroups: [""]
resources: ["secrets"]
verbs: ["get", "list"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshotclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshotcontents"]
verbs: ["create", "get", "list", "watch", "update", "delete"]
- apiGroups: ["snapshot.storage.k8s.io"]
resources: ["volumesnapshots"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["apiextensions.k8s.io"]
resources: ["customresourcedefinitions"]
verbs: ["create"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: csi-snapshotter-role
subjects:
- kind: ServiceAccount
name: csi-snapshotter
namespace: default
roleRef:
kind: ClusterRole
name: external-snapshotter-runner
apiGroup: rbac.authorization.k8s.io

56
examples/rbd/csi-snapshotter.yaml
View File

@ -1,56 +0,0 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: csi-snapshotter
---
kind: Service
apiVersion: v1
metadata:
name: csi-snapshotter
labels:
app: csi-snapshotter
spec:
selector:
app: csi-snapshotter
ports:
- name: dummy
port: 12345
---
kind: StatefulSet
apiVersion: apps/v1
metadata:
name: csi-snapshotter
spec:
serviceName: "csi-snapshotter"
replicas: 1
selector:
matchLabels:
app: csi-snapshotter
template:
metadata:
labels:
app: csi-snapshotter
spec:
serviceAccount: csi-snapshotter
containers:
- name: csi-snapshotter
image: quay.io/k8scsi/csi-snapshotter:v0.4.0
args:
- "--csi-address=$(ADDRESS)"
- "--connection-timeout=15s"
- "--v=5"
env:
- name: ADDRESS
value: /csi/csi.sock
imagePullPolicy: Always
securityContext:
privileged: true
volumeMounts:
- name: socket-dir
mountPath: /csi
imagePullPolicy: Always
volumes:
- hostPath:
path: /var/lib/kubelet/plugins/rbd.csi.ceph.com
type: DirectoryOrCreate
name: socket-dir

4
examples/rbd/exec-bash.sh
View File

@ -4,7 +4,7 @@ CONTAINER_NAME=csi-rbdplugin
POD_NAME=$(kubectl get pods -l app=$CONTAINER_NAME -o=name | head -n 1)
function get_pod_status() {
echo -n $(kubectl get $POD_NAME -o jsonpath="{.status.phase}")
echo -n "$(kubectl get "$POD_NAME" -o jsonpath="{.status.phase}")"
}
while [[ "$(get_pod_status)" != "Running" ]]; do
@ -12,4 +12,4 @@ while [[ "$(get_pod_status)" != "Running" ]]; do
echo "Waiting for $POD_NAME (status $(get_pod_status))"
done
kubectl exec -it ${POD_NAME#*/} -c $CONTAINER_NAME bash
kubectl exec -it "${POD_NAME#*/}" -c "$CONTAINER_NAME" bash

4
examples/rbd/logs.sh
View File

@ -4,7 +4,7 @@ CONTAINER_NAME=csi-rbdplugin
POD_NAME=$(kubectl get pods -l app=$CONTAINER_NAME -o=name | head -n 1)
function get_pod_status() {
echo -n $(kubectl get $POD_NAME -o jsonpath="{.status.phase}")
echo -n "$(kubectl get "$POD_NAME" -o jsonpath="{.status.phase}")"
}
while [[ "$(get_pod_status)" != "Running" ]]; do
@ -12,4 +12,4 @@ while [[ "$(get_pod_status)" != "Running" ]]; do
echo "Waiting for $POD_NAME (status $(get_pod_status))"
done
kubectl logs -f $POD_NAME -c $CONTAINER_NAME
kubectl logs -f "$POD_NAME" -c "$CONTAINER_NAME"

2
examples/rbd/plugin-deploy.sh
View File

@ -10,6 +10,6 @@ cd "$deployment_base" || exit 1
objects=(csi-attacher-rbac csi-provisioner-rbac csi-nodeplugin-rbac csi-rbdplugin-attacher csi-rbdplugin-provisioner csi-rbdplugin)
for obj in ${objects[@]}; do
for obj in "${objects[@]}"; do
kubectl create -f "./$obj.yaml"
done

2
examples/rbd/plugin-teardown.sh
View File

@ -10,6 +10,6 @@ cd "$deployment_base" || exit 1
objects=(csi-rbdplugin-attacher csi-rbdplugin-provisioner csi-rbdplugin csi-attacher-rbac csi-provisioner-rbac csi-nodeplugin-rbac)
for obj in ${objects[@]}; do
for obj in "${objects[@]}"; do
kubectl delete -f "./$obj.yaml"
done

16
examples/rbd/pod-block.yaml
View File

@ -1,16 +0,0 @@
apiVersion: v1
kind: Pod
metadata:
name: csirbd-block-pod
spec:
containers:
- name: web-server
image: nginx
volumeDevices:
- name: data
devicePath: /dev/vda
volumes:
- name: data
persistentVolumeClaim:
claimName: rbd-block-pvc
readOnly: false

17
examples/rbd/pod-restore.yaml Normal file
View File

@ -0,0 +1,17 @@
---
apiVersion: v1
kind: Pod
metadata:
name: csirbd-restore-demo-pod
spec:
containers:
- name: web-server
image: nginx
volumeMounts:
- name: mypvc
mountPath: /var/lib/www/html
volumes:
- name: mypvc
persistentVolumeClaim:
claimName: rbd-pvc-restore
readOnly: false

20
examples/rbd/pod.yaml
View File

@ -1,17 +1,17 @@
---
apiVersion: v1
kind: Pod
metadata:
name: csirbd-demo-pod
spec:
containers:
- name: web-server
image: nginx
volumeMounts:
- name: mypvc
mountPath: /var/lib/www/html
- name: web-server
image: nginx
volumeMounts:
- name: mypvc
mountPath: /var/lib/www/html
volumes:
- name: mypvc
persistentVolumeClaim:
claimName: rbd-pvc
readOnly: false
- name: mypvc
persistentVolumeClaim:
claimName: rbd-pvc
readOnly: false

16
examples/rbd/pvc-restore.yaml Normal file
View File

@ -0,0 +1,16 @@
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: rbd-pvc-restore
spec:
storageClassName: csi-rbd
dataSource:
name: rbd-pvc-snapshot
kind: VolumeSnapshot
apiGroup: snapshot.storage.k8s.io
accessModes:
- ReadWriteMany
resources:
requests:
storage: 1Gi

3
examples/rbd/pvc.yaml
View File

@ -1,10 +1,11 @@
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: rbd-pvc
spec:
accessModes:
- ReadWriteOnce
- ReadWriteOnce
resources:
requests:
storage: 1Gi

18
examples/rbd/raw-block-pod.yaml Normal file
View File

@ -0,0 +1,18 @@
---
apiVersion: v1
kind: Pod
metadata:
name: pod-with-raw-block-volume
spec:
containers:
- name: fc-container
image: fedora:26
command: ["/bin/sh", "-c"]
args: ["tail -f /dev/null"]
volumeDevices:
- name: data
devicePath: /dev/xvda
volumes:
- name: data
persistentVolumeClaim:
claimName: raw-block-pvc

7
examples/rbd/pvc-block.yaml → examples/rbd/raw-block-pvc.yaml
View File

@ -1,11 +1,12 @@
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: rbd-block-pvc
name: raw-block-pvc
spec:
volumeMode: Block
accessModes:
- ReadWriteOnce
- ReadWriteOnce
volumeMode: Block
resources:
requests:
storage: 1Gi

5
examples/rbd/secret.yaml
View File

@ -1,8 +1,9 @@
---
apiVersion: v1
kind: Secret
metadata:
name: csi-rbd-secret
namespace: default
namespace: default
data:
# Key value corresponds to a user name defined in ceph cluster
admin: BASE64-ENCODED-PASSWORD
@ -10,4 +11,4 @@ data:
kubernetes: BASE64-ENCODED-PASSWORD
# if monValueFromSecret is set to "monitors", uncomment the
# following and set the mon there
#monitors: BASE64-ENCODED-Comma-Delimited-Mons
# monitors: BASE64-ENCODED-Comma-Delimited-Mons

1
examples/rbd/snapshot.yaml
View File

@ -1,3 +1,4 @@
---
apiVersion: snapshot.storage.k8s.io/v1alpha1
kind: VolumeSnapshot
metadata:

19
examples/rbd/snapshotclass.yaml
View File

@ -1,10 +1,23 @@
---
apiVersion: snapshot.storage.k8s.io/v1alpha1
kind: VolumeSnapshotClass
metadata:
name: csi-rbdplugin-snapclass
snapshotter: csi-rbdplugin
snapshotter: rbd.csi.ceph.com
parameters:
pool: rbd
# Comma separated list of Ceph monitors
# if using FQDN, make sure csi plugin's dns policy is appropriate.
monitors: mon1:port,mon2:port,...
csiSnapshotterSecretName: csi-rbd-secret
csiSnapshotterSecretNamespace: default
# OR,
# String representing a Ceph cluster to provision storage from.
# Should be unique across all Ceph clusters in use for provisioning,
# cannot be greater than 36 bytes in length, and should remain immutable for
# the lifetime of the StorageClass in use.
# If using clusterID, ensure to create a secret, as in
# template-ceph-cluster-ID-secret.yaml, to accompany the string chosen to
# represent the Ceph cluster in clusterID
# clusterID: <cluster-id>
csi.storage.k8s.io/snapshotter-secret-name: csi-rbd-secret
csi.storage.k8s.io/snapshotter-secret-namespace: default

68
examples/rbd/storageclass.yaml
View File

@ -1,37 +1,53 @@
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: csi-rbd
provisioner: csi-rbdplugin
provisioner: rbd.csi.ceph.com
parameters:
# Comma separated list of Ceph monitors
# if using FQDN, make sure csi plugin's dns policy is appropriate.
monitors: mon1:port,mon2:port,...
# Comma separated list of Ceph monitors
# if using FQDN, make sure csi plugin's dns policy is appropriate.
monitors: mon1:port,mon2:port,...
# OR,
# String representing a Ceph cluster to provision storage from.
# Should be unique across all Ceph clusters in use for provisioning,
# cannot be greater than 36 bytes in length, and should remain immutable for
# the lifetime of the StorageClass in use.
# If using clusterID, ensure to create a secret, as in
# template-ceph-cluster-ID-secret.yaml, to accompany the string chosen to
# represent the Ceph cluster in clusterID
# clusterID: <cluster-id>
# OR,
# if "monitors" parameter is not set, driver to get monitors from same
# secret as admin/user credentials. "monValueFromSecret" provides the
# key in the secret whose value is the mons
# monValueFromSecret: "monitors"
# if "monitors" parameter is not set, driver to get monitors from same
# secret as admin/user credentials. "monValueFromSecret" provides the
# key in the secret whose value is the mons
#monValueFromSecret: "monitors"
# Ceph pool into which the RBD image shall be created
pool: rbd
# Ceph pool into which the RBD image shall be created
pool: rbd
# RBD image format. Defaults to "2".
imageFormat: "2"
# RBD image format. Defaults to "2".
imageFormat: "2"
# RBD image features. Available for imageFormat: "2"
# CSI RBD currently supports only `layering` feature.
imageFeatures: layering
# RBD image features. Available for imageFormat: "2". CSI RBD currently supports only `layering` feature.
imageFeatures: layering
# The secrets have to contain Ceph admin credentials.
csiProvisionerSecretName: csi-rbd-secret
csiProvisionerSecretNamespace: default
csiNodePublishSecretName: csi-rbd-secret
csiNodePublishSecretNamespace: default
# The secrets have to contain Ceph admin credentials.
# NOTE: If using "clusterID" instead of "monitors" above, the following
# secrets MAY be added to the ceph-cluster-<cluster-id> secret and skipped
# here
csi.storage.k8s.io/provisioner-secret-name: csi-rbd-secret
csi.storage.k8s.io/provisioner-secret-namespace: default
csi.storage.k8s.io/node-publish-secret-name: csi-rbd-secret
csi.storage.k8s.io/node-publish-secret-namespace: default
# Ceph users for operating RBD
adminid: admin
userid: kubernetes
# uncomment the following to use rbd-nbd as mounter on supported nodes
#mounter: rbd-nbd
# Ceph users for operating RBD
# NOTE: If using "clusterID" instead of "monitors" above, the following
# IDs MAY be added to the ceph-cluster-<cluster-id> secret and skipped
# here
adminid: admin
userid: kubernetes
# uncomment the following to use rbd-nbd as mounter on supported nodes
# mounter: rbd-nbd
reclaimPolicy: Delete

36
examples/rbd/template-ceph-cluster-ID-secret.yaml Normal file
View File

@ -0,0 +1,36 @@
---
# This is a template secret that helps define a Ceph cluster configuration
# as required by the CSI driver. This is used when a StorageClass has the
# "clusterID" defined as one of the parameters, to provide the CSI instance
# Ceph cluster configuration information.
apiVersion: v1
kind: Secret
metadata:
# The <cluster-id> is used by the CSI plugin to uniquely identify and use a
# Ceph cluster, the value MUST match the value provided as `clusterID` in the
# StorageClass
name: ceph-cluster-<cluster-id>
namespace: default
data:
# Base64 encoded and comma separated Ceph cluster monitor list
# - Typically output of: `echo -n "mon1:port,mon2:port,..." | base64`
monitors: <BASE64-ENCODED-MONLIST>
# Base64 encoded and comma separated list of pool names from which volumes
# can be provisioned
pools: <BASE64-ENCODED-POOLIST>
# Base64 encoded admin ID to use for provisioning
# - Typically output of: `echo -n "<admin-id>" | base64`
# Substitute the entire string including angle braces, with the base64 value
adminid: <BASE64-ENCODED-ID>
# Base64 encoded key of the provisioner admin ID
# - Output of: `ceph auth get-key client.<admin-id> | base64`
# Substitute the entire string including angle braces, with the base64 value
adminkey: <BASE64-ENCODED-PASSWORD>
# Base64 encoded user ID to use for publishing
# - Typically output of: `echo -n "<admin-id>" | base64`
# Substitute the entire string including angle braces, with the base64 value
userid: <BASE64-ENCODED-ID>
# Base64 encoded key of the publisher user ID
# - Output of: `ceph auth get-key client.<admin-id> | base64`
# Substitute the entire string including angle braces, with the base64 value
userkey: <BASE64-ENCODED-PASSWORD>

33
examples/rbd/template-csi-rbdplugin-patch.yaml Normal file
View File

@ -0,0 +1,33 @@
---
# This is a patch to the existing daemonset deployment of CSI rbdplugin.
#
# This is to be used when using `clusterID` instead of monitors or
# monValueFromSecret in the StorageClass to specify the Ceph cluster to
# provision storage from, AND when the value of `--configroot` option to the
# CSI pods is NOT "k8s_objects".
#
# This patch file, patches in the specified secret for the 'clusterID' as a
# volume, instead of the Ceph CSI plugin actively fetching and using kubernetes
# secrets.
#
# Post substituting the <cluster-id> in all places execute,
# `kubectl patch daemonset csi-rbdplugin --patch\
# "$(cat template-csi-rbdplugin-patch.yaml)"`
# to patch the daemonset deployment.
#
# `kubectl patch statefulset csi-rbdplugin-provisioner --patch\
# "$(cat template-csi-rbdplugin-patch.yaml)"`
# to patch the statefulset deployment.
spec:
template:
spec:
containers:
- name: csi-rbdplugin
volumeMounts:
- name: ceph-cluster-<cluster-id>
mountPath: "/etc/csi-config/ceph-cluster-<cluster-id>"
readOnly: true
volumes:
- name: ceph-cluster-<cluster-id>
secret:
secretName: ceph-cluster-<cluster-id>

102
pkg/cephfs/cephconf.go
View File

@ -17,114 +17,32 @@ limitations under the License.
package cephfs
import (
"fmt"
"io/ioutil"
"os"
"path"
"text/template"
)
const cephConfig = `[global]
mon_host = {{.Monitors}}
var cephConfig = []byte(`[global]
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx
# Workaround for http://tracker.ceph.com/issues/23446
fuse_set_user_groups = false
`
const cephKeyring = `[client.{{.UserId}}]
key = {{.Key}}
`
const cephSecret = `{{.Key}}`
`)
const (
cephConfigRoot = "/etc/ceph"
cephConfigFileNameFmt = "ceph.share.%s.conf"
cephKeyringFileNameFmt = "ceph.share.%s.client.%s.keyring"
cephSecretFileNameFmt = "ceph.share.%s.client.%s.secret"
cephConfigRoot = "/etc/ceph"
cephConfigPath = "/etc/ceph/ceph.conf"
)
var (
cephConfigTempl *template.Template
cephKeyringTempl *template.Template
cephSecretTempl *template.Template
)
func init() {
fm := map[string]interface{}{
"perms": func(readOnly bool) string {
if readOnly {
return "r"
}
return "rw"
},
}
cephConfigTempl = template.Must(template.New("config").Parse(cephConfig))
cephKeyringTempl = template.Must(template.New("keyring").Funcs(fm).Parse(cephKeyring))
cephSecretTempl = template.Must(template.New("secret").Parse(cephSecret))
func createCephConfigRoot() error {
return os.MkdirAll(cephConfigRoot, 0755) // #nosec
}
type cephConfigWriter interface {
writeToFile() error
}
type cephConfigData struct {
Monitors string
VolumeID volumeID
}
func writeCephTemplate(fileName string, m os.FileMode, t *template.Template, data interface{}) error {
if err := os.MkdirAll(cephConfigRoot, 0755); err != nil {
func writeCephConfig() error {
if err := createCephConfigRoot(); err != nil {
return err
}
f, err := os.OpenFile(path.Join(cephConfigRoot, fileName), os.O_CREATE|os.O_EXCL|os.O_WRONLY, m)
if err != nil {
if os.IsExist(err) {
return nil
}
return err
}
defer f.Close()
return t.Execute(f, data)
}
func (d *cephConfigData) writeToFile() error {
return writeCephTemplate(fmt.Sprintf(cephConfigFileNameFmt, d.VolumeID), 0640, cephConfigTempl, d)
}
type cephKeyringData struct {
UserId, Key string
VolumeID volumeID
}
func (d *cephKeyringData) writeToFile() error {
return writeCephTemplate(fmt.Sprintf(cephKeyringFileNameFmt, d.VolumeID, d.UserId), 0600, cephKeyringTempl, d)
}
type cephSecretData struct {
UserId, Key string
VolumeID volumeID
}
func (d *cephSecretData) writeToFile() error {
return writeCephTemplate(fmt.Sprintf(cephSecretFileNameFmt, d.VolumeID, d.UserId), 0600, cephSecretTempl, d)
}
func getCephSecretPath(volId volumeID, userId string) string {
return path.Join(cephConfigRoot, fmt.Sprintf(cephSecretFileNameFmt, volId, userId))
}
func getCephKeyringPath(volId volumeID, userId string) string {
return path.Join(cephConfigRoot, fmt.Sprintf(cephKeyringFileNameFmt, volId, userId))
}
func getCephConfPath(volId volumeID) string {
return path.Join(cephConfigRoot, fmt.Sprintf(cephConfigFileNameFmt, volId))
return ioutil.WriteFile(cephConfigPath, cephConfig, 0640)
}

108
pkg/cephfs/cephuser.go
View File

@ -17,10 +17,7 @@ limitations under the License.
package cephfs
import (
"bytes"
"encoding/json"
"fmt"
"os"
)
const (
@ -47,76 +44,65 @@ func (ent *cephEntity) toCredentials() *credentials {
}
}
func getCephUserName(volId volumeID) string {
return cephUserPrefix + string(volId)
func getCephUserName(volID volumeID) string {
return cephUserPrefix + string(volID)
}
func getCephUser(adminCr *credentials, volId volumeID) (*cephEntity, error) {
entityName := cephEntityClientPrefix + getCephUserName(volId)
func getSingleCephEntity(args ...string) (*cephEntity, error) {
var ents []cephEntity
args := [...]string{
"auth", "-f", "json", "-c", getCephConfPath(volId), "-n", cephEntityClientPrefix + adminCr.id,
"get", entityName,
}
out, err := execCommand("ceph", args[:]...)
if err != nil {
return nil, fmt.Errorf("cephfs: ceph failed with following error: %s\ncephfs: ceph output: %s", err, out)
}
// Workaround for output from `ceph auth get`
// Contains non-json data: "exported keyring for ENTITY\n\n"
offset := bytes.Index(out, []byte("[{"))
if json.NewDecoder(bytes.NewReader(out[offset:])).Decode(&ents); err != nil {
return nil, fmt.Errorf("failed to decode json: %v", err)
if err := execCommandJSON(&ents, "ceph", args...); err != nil {
return nil, err
}
if len(ents) != 1 {
return nil, fmt.Errorf("got unexpected number of entities for %s: expected 1, got %d", entityName, len(ents))
return nil, fmt.Errorf("got unexpected number of entities: expected 1, got %d", len(ents))
}
return &ents[0], nil
}
func createCephUser(volOptions *volumeOptions, adminCr *credentials, volId volumeID) (*cephEntity, error) {
caps := cephEntityCaps{
Mds: fmt.Sprintf("allow rw path=%s", getVolumeRootPathCeph(volId)),
Mon: "allow r",
Osd: fmt.Sprintf("allow rw pool=%s namespace=%s", volOptions.Pool, getVolumeNamespace(volId)),
}
var ents []cephEntity
args := [...]string{
"auth", "-f", "json", "-c", getCephConfPath(volId), "-n", cephEntityClientPrefix + adminCr.id,
"get-or-create", cephEntityClientPrefix + getCephUserName(volId),
"mds", caps.Mds,
"mon", caps.Mon,
"osd", caps.Osd,
}
if err := execCommandJSON(&ents, "ceph", args[:]...); err != nil {
return nil, fmt.Errorf("error creating ceph user: %v", err)
}
return &ents[0], nil
func genUserIDs(adminCr *credentials, volID volumeID) (adminID, userID string) {
return cephEntityClientPrefix + adminCr.id, cephEntityClientPrefix + getCephUserName(volID)
}
func deleteCephUser(adminCr *credentials, volId volumeID) error {
userId := getCephUserName(volId)
func getCephUser(volOptions *volumeOptions, adminCr *credentials, volID volumeID) (*cephEntity, error) {
adminID, userID := genUserIDs(adminCr, volID)
args := [...]string{
"-c", getCephConfPath(volId), "-n", cephEntityClientPrefix + adminCr.id,
"auth", "rm", cephEntityClientPrefix + userId,
}
if err := execCommandAndValidate("ceph", args[:]...); err != nil {
return err
}
os.Remove(getCephKeyringPath(volId, userId))
os.Remove(getCephSecretPath(volId, userId))
return nil
return getSingleCephEntity(
"-m", volOptions.Monitors,
"-n", adminID,
"--key="+adminCr.key,
"-c", cephConfigPath,
"-f", "json",
"auth", "get", userID,
)
}
func createCephUser(volOptions *volumeOptions, adminCr *credentials, volID volumeID) (*cephEntity, error) {
adminID, userID := genUserIDs(adminCr, volID)
return getSingleCephEntity(
"-m", volOptions.Monitors,
"-n", adminID,
"--key="+adminCr.key,
"-c", cephConfigPath,
"-f", "json",
"auth", "get-or-create", userID,
// User capabilities
"mds", fmt.Sprintf("allow rw path=%s", getVolumeRootPathCeph(volID)),
"mon", "allow r",
"osd", fmt.Sprintf("allow rw pool=%s namespace=%s", volOptions.Pool, getVolumeNamespace(volID)),
)
}
func deleteCephUser(volOptions *volumeOptions, adminCr *credentials, volID volumeID) error {
adminID, userID := genUserIDs(adminCr, volID)
return execCommandErr("ceph",
"-m", volOptions.Monitors,
"-n", adminID,
"--key="+adminCr.key,
"-c", cephConfigPath,
"auth", "rm", userID,
)
}

129
pkg/cephfs/controllerserver.go
View File

@ -17,18 +17,20 @@ limitations under the License.
package cephfs
import (
"github.com/golang/glog"
csicommon "github.com/ceph/ceph-csi/pkg/csi-common"
"github.com/ceph/ceph-csi/pkg/util"
"github.com/container-storage-interface/spec/lib/go/csi"
"golang.org/x/net/context"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"github.com/container-storage-interface/spec/lib/go/csi/v0"
"github.com/kubernetes-csi/drivers/pkg/csi-common"
"github.com/ceph/ceph-csi/pkg/util"
"k8s.io/klog"
"k8s.io/kubernetes/pkg/util/keymutex"
)
type controllerServer struct {
// ControllerServer struct of CEPH CSI driver with supported methods of CSI
// controller server spec.
type ControllerServer struct {
*csicommon.DefaultControllerServer
MetadataStore util.CachePersister
}
@ -38,129 +40,152 @@ type controllerCacheEntry struct {
VolumeID volumeID
}
func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) {
var (
mtxControllerVolumeID = keymutex.NewHashed(0)
)
// CreateVolume creates the volume in backend and store the volume metadata
func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) {
if err := cs.validateCreateVolumeRequest(req); err != nil {
glog.Errorf("CreateVolumeRequest validation failed: %v", err)
klog.Errorf("CreateVolumeRequest validation failed: %v", err)
return nil, err
}
// Configuration
volOptions, err := newVolumeOptions(req.GetParameters())
secret := req.GetSecrets()
volOptions, err := newVolumeOptions(req.GetParameters(), secret)
if err != nil {
glog.Errorf("validation of volume options failed: %v", err)
klog.Errorf("validation of volume options failed: %v", err)
return nil, status.Error(codes.InvalidArgument, err.Error())
}
volId := newVolumeID()
volID := makeVolumeID(req.GetName())
conf := cephConfigData{Monitors: volOptions.Monitors, VolumeID: volId}
if err = conf.writeToFile(); err != nil {
glog.Errorf("failed to write ceph config file to %s: %v", getCephConfPath(volId), err)
return nil, status.Error(codes.Internal, err.Error())
}
mtxControllerVolumeID.LockKey(string(volID))
defer mustUnlock(mtxControllerVolumeID, string(volID))
// Create a volume in case the user didn't provide one
if volOptions.ProvisionVolume {
// Admin credentials are required
cr, err := getAdminCredentials(req.GetControllerCreateSecrets())
cr, err := getAdminCredentials(secret)
if err != nil {
return nil, status.Error(codes.InvalidArgument, err.Error())
}
if err = storeCephCredentials(volId, cr); err != nil {
glog.Errorf("failed to store admin credentials for '%s': %v", cr.id, err)
if err = createVolume(volOptions, cr, volID, req.GetCapacityRange().GetRequiredBytes()); err != nil {
klog.Errorf("failed to create volume %s: %v", req.GetName(), err)
return nil, status.Error(codes.Internal, err.Error())
}
if err = createVolume(volOptions, cr, volId, req.GetCapacityRange().GetRequiredBytes()); err != nil {
glog.Errorf("failed to create volume %s: %v", req.GetName(), err)
if _, err = createCephUser(volOptions, cr, volID); err != nil {
klog.Errorf("failed to create ceph user for volume %s: %v", req.GetName(), err)
return nil, status.Error(codes.Internal, err.Error())
}
if _, err = createCephUser(volOptions, cr, volId); err != nil {
glog.Errorf("failed to create ceph user for volume %s: %v", req.GetName(), err)
return nil, status.Error(codes.Internal, err.Error())
}
glog.Infof("cephfs: successfully created volume %s", volId)
klog.Infof("cephfs: successfully created volume %s", volID)
} else {
glog.Infof("cephfs: volume %s is provisioned statically", volId)
klog.Infof("cephfs: volume %s is provisioned statically", volID)
}
ce := &controllerCacheEntry{VolOptions: *volOptions, VolumeID: volId}
if err := cs.MetadataStore.Create(string(volId), ce); err != nil {
glog.Errorf("failed to store a cache entry for volume %s: %v", volId, err)
ce := &controllerCacheEntry{VolOptions: *volOptions, VolumeID: volID}
if err := cs.MetadataStore.Create(string(volID), ce); err != nil {
klog.Errorf("failed to store a cache entry for volume %s: %v", volID, err)
return nil, status.Error(codes.Internal, err.Error())
}
return &csi.CreateVolumeResponse{
Volume: &csi.Volume{
Id: string(volId),
VolumeId: string(volID),
CapacityBytes: req.GetCapacityRange().GetRequiredBytes(),
Attributes: req.GetParameters(),
VolumeContext: req.GetParameters(),
},
}, nil
}
func (cs *controllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) {
if err := cs.validateDeleteVolumeRequest(req); err != nil {
glog.Errorf("DeleteVolumeRequest validation failed: %v", err)
// DeleteVolume deletes the volume in backend
// and removes the volume metadata from store
// nolint: gocyclo
func (cs *ControllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) {
if err := cs.validateDeleteVolumeRequest(); err != nil {
klog.Errorf("DeleteVolumeRequest validation failed: %v", err)
return nil, err
}
var (
volId = volumeID(req.GetVolumeId())
err error
volID = volumeID(req.GetVolumeId())
secrets = req.GetSecrets()
)
ce := &controllerCacheEntry{}
if err := cs.MetadataStore.Get(string(volId), ce); err != nil {
if err := cs.MetadataStore.Get(string(volID), ce); err != nil {
if err, ok := err.(*util.CacheEntryNotFound); ok {
klog.Infof("cephfs: metadata for volume %s not found, assuming the volume to be already deleted (%v)", volID, err)
return &csi.DeleteVolumeResponse{}, nil
}
return nil, status.Error(codes.Internal, err.Error())
}
if !ce.VolOptions.ProvisionVolume {
// DeleteVolume() is forbidden for statically provisioned volumes!
glog.Warningf("volume %s is provisioned statically, aborting delete", volId)
klog.Warningf("volume %s is provisioned statically, aborting delete", volID)
return &csi.DeleteVolumeResponse{}, nil
}
// mons may have changed since create volume,
// retrieve the latest mons and override old mons
if mon, secretsErr := getMonValFromSecret(secrets); secretsErr == nil && len(mon) > 0 {
klog.Infof("overriding monitors [%q] with [%q] for volume %s", ce.VolOptions.Monitors, mon, volID)
ce.VolOptions.Monitors = mon
}
// Deleting a volume requires admin credentials
cr, err := getAdminCredentials(req.GetControllerDeleteSecrets())
cr, err := getAdminCredentials(secrets)
if err != nil {
glog.Errorf("failed to retrieve admin credentials: %v", err)
klog.Errorf("failed to retrieve admin credentials: %v", err)
return nil, status.Error(codes.InvalidArgument, err.Error())
}
if err = purgeVolume(volId, cr, &ce.VolOptions); err != nil {
glog.Errorf("failed to delete volume %s: %v", volId, err)
mtxControllerVolumeID.LockKey(string(volID))
defer mustUnlock(mtxControllerVolumeID, string(volID))
if err = purgeVolume(volID, cr, &ce.VolOptions); err != nil {
klog.Errorf("failed to delete volume %s: %v", volID, err)
return nil, status.Error(codes.Internal, err.Error())
}
if err = deleteCephUser(cr, volId); err != nil {
glog.Errorf("failed to delete ceph user for volume %s: %v", volId, err)
if err = deleteCephUser(&ce.VolOptions, cr, volID); err != nil {
klog.Errorf("failed to delete ceph user for volume %s: %v", volID, err)
return nil, status.Error(codes.Internal, err.Error())
}
if err := cs.MetadataStore.Delete(string(volId)); err != nil {
if err = cs.MetadataStore.Delete(string(volID)); err != nil {
return nil, status.Error(codes.Internal, err.Error())
}
glog.Infof("cephfs: successfully deleted volume %s", volId)
klog.Infof("cephfs: successfully deleted volume %s", volID)
return &csi.DeleteVolumeResponse{}, nil
}
func (cs *controllerServer) ValidateVolumeCapabilities(
// ValidateVolumeCapabilities checks whether the volume capabilities requested
// are supported.
func (cs *ControllerServer) ValidateVolumeCapabilities(
ctx context.Context,
req *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error) {
// Cephfs doesn't support Block volume
for _, cap := range req.VolumeCapabilities {
if cap.GetBlock() != nil {
return &csi.ValidateVolumeCapabilitiesResponse{Supported: false, Message: ""}, nil
return &csi.ValidateVolumeCapabilitiesResponse{Message: ""}, nil
}
}
return &csi.ValidateVolumeCapabilitiesResponse{Supported: true}, nil
return &csi.ValidateVolumeCapabilitiesResponse{
Confirmed: &csi.ValidateVolumeCapabilitiesResponse_Confirmed{
VolumeCapabilities: req.VolumeCapabilities,
},
}, nil
}

16
pkg/cephfs/credentials.go
View File

@ -19,10 +19,11 @@ package cephfs
import "fmt"
const (
credUserId = "userID"
credUserID = "userID"
credUserKey = "userKey"
credAdminId = "adminID"
credAdminID = "adminID"
credAdminKey = "adminKey"
credMonitors = "monitors"
)
type credentials struct {
@ -48,9 +49,16 @@ func getCredentials(idField, keyField string, secrets map[string]string) (*crede
}
func getUserCredentials(secrets map[string]string) (*credentials, error) {
return getCredentials(credUserId, credUserKey, secrets)
return getCredentials(credUserID, credUserKey, secrets)
}
func getAdminCredentials(secrets map[string]string) (*credentials, error) {
return getCredentials(credAdminId, credAdminKey, secrets)
return getCredentials(credAdminID, credAdminKey, secrets)
}
func getMonValFromSecret(secrets map[string]string) (string, error) {
if mons, ok := secrets[credMonitors]; ok {
return mons, nil
}
return "", fmt.Errorf("missing %q", credMonitors)
}

99
pkg/cephfs/driver.go
View File

@ -17,76 +17,78 @@ limitations under the License.
package cephfs
import (
"github.com/golang/glog"
"github.com/container-storage-interface/spec/lib/go/csi/v0"
csicommon "github.com/kubernetes-csi/drivers/pkg/csi-common"
"k8s.io/klog"
csicommon "github.com/ceph/ceph-csi/pkg/csi-common"
"github.com/ceph/ceph-csi/pkg/util"
"github.com/container-storage-interface/spec/lib/go/csi"
)
const (
// Version of the cephfs csi driver
Version = "0.3.0"
// version of ceph driver
version = "1.0.0"
)
var (
// PluginFolder is the kubelet plugin directory for cephfs plugin
PluginFolder = "/var/lib/kubelet/plugins/"
)
// PluginFolder defines the location of ceph plugin
var PluginFolder = "/var/lib/kubelet/plugins/"
type cephfsDriver struct {
driver *csicommon.CSIDriver
// Driver contains the default identity,node and controller struct
type Driver struct {
cd *csicommon.CSIDriver
is *identityServer
ns *nodeServer
cs *controllerServer
caps []*csi.VolumeCapability_AccessMode
cscaps []*csi.ControllerServiceCapability
is *IdentityServer
ns *NodeServer
cs *ControllerServer
}
var (
driver *cephfsDriver
// DefaultVolumeMounter for mounting volumes
DefaultVolumeMounter string
)
// NewCephFSDriver provides a remote csi cephfs driver object.
func NewCephFSDriver() *cephfsDriver {
return &cephfsDriver{}
// NewDriver returns new ceph driver
func NewDriver() *Driver {
return &Driver{}
}
func NewIdentityServer(d *csicommon.CSIDriver) *identityServer {
return &identityServer{
// NewIdentityServer initialize a identity server for ceph CSI driver
func NewIdentityServer(d *csicommon.CSIDriver) *IdentityServer {
return &IdentityServer{
DefaultIdentityServer: csicommon.NewDefaultIdentityServer(d),
}
}
func NewControllerServer(d *csicommon.CSIDriver, cachePersister util.CachePersister) *controllerServer {
return &controllerServer{
// NewControllerServer initialize a controller server for ceph CSI driver
func NewControllerServer(d *csicommon.CSIDriver, cachePersister util.CachePersister) *ControllerServer {
return &ControllerServer{
DefaultControllerServer: csicommon.NewDefaultControllerServer(d),
MetadataStore: cachePersister,
}
}
func NewNodeServer(d *csicommon.CSIDriver) *nodeServer {
return &nodeServer{
// NewNodeServer initialize a node server for ceph CSI driver.
func NewNodeServer(d *csicommon.CSIDriver) *NodeServer {
return &NodeServer{
DefaultNodeServer: csicommon.NewDefaultNodeServer(d),
}
}
func (fs *cephfsDriver) Run(driverName, nodeId, endpoint, volumeMounter string, cachePersister util.CachePersister) {
glog.Infof("Driver: %v version: %v", driverName, Version)
// Run start a non-blocking grpc controller,node and identityserver for
// ceph CSI driver which can serve multiple parallel requests
func (fs *Driver) Run(driverName, nodeID, endpoint, volumeMounter, mountCacheDir string, cachePersister util.CachePersister) {
klog.Infof("Driver: %v version: %v", driverName, version)
// Configuration
if err := loadAvailableMounters(); err != nil {
glog.Fatalf("cephfs: failed to load ceph mounters: %v", err)
klog.Fatalf("cephfs: failed to load ceph mounters: %v", err)
}
if volumeMounter != "" {
if err := validateMounter(volumeMounter); err != nil {
glog.Fatalln(err)
klog.Fatalln(err)
} else {
DefaultVolumeMounter = volumeMounter
}
@ -97,29 +99,40 @@ func (fs *cephfsDriver) Run(driverName, nodeId, endpoint, volumeMounter string,
DefaultVolumeMounter = availableMounters[0]
}
glog.Infof("cephfs: setting default volume mounter to %s", DefaultVolumeMounter)
klog.Infof("cephfs: setting default volume mounter to %s", DefaultVolumeMounter)
// Initialize default library driver
fs.driver = csicommon.NewCSIDriver(driverName, Version, nodeId)
if fs.driver == nil {
glog.Fatalln("Failed to initialize CSI driver")
if err := writeCephConfig(); err != nil {
klog.Fatalf("failed to write ceph configuration file: %v", err)
}
fs.driver.AddControllerServiceCapabilities([]csi.ControllerServiceCapability_RPC_Type{
initVolumeMountCache(driverName, mountCacheDir, cachePersister)
if mountCacheDir != "" {
if err := remountCachedVolumes(); err != nil {
klog.Warningf("failed to remount cached volumes: %v", err)
//ignore remount fail
}
}
// Initialize default library driver
fs.cd = csicommon.NewCSIDriver(driverName, version, nodeID)
if fs.cd == nil {
klog.Fatalln("failed to initialize CSI driver")
}
fs.cd.AddControllerServiceCapabilities([]csi.ControllerServiceCapability_RPC_Type{
csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME,
})
fs.driver.AddVolumeCapabilityAccessModes([]csi.VolumeCapability_AccessMode_Mode{
fs.cd.AddVolumeCapabilityAccessModes([]csi.VolumeCapability_AccessMode_Mode{
csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER,
})
// Create gRPC servers
fs.is = NewIdentityServer(fs.driver)
fs.ns = NewNodeServer(fs.driver)
fs.is = NewIdentityServer(fs.cd)
fs.ns = NewNodeServer(fs.cd)
fs.cs = NewControllerServer(fs.driver, cachePersister)
fs.cs = NewControllerServer(fs.cd, cachePersister)
server := csicommon.NewNonBlockingGRPCServer()
server.Start(endpoint, fs.is, fs.cs, fs.ns)

12
pkg/cephfs/identityserver.go
View File

@ -19,15 +19,19 @@ package cephfs
import (
"context"
"github.com/container-storage-interface/spec/lib/go/csi/v0"
"github.com/kubernetes-csi/drivers/pkg/csi-common"
csicommon "github.com/ceph/ceph-csi/pkg/csi-common"
"github.com/container-storage-interface/spec/lib/go/csi"
)
type identityServer struct {
// IdentityServer struct of ceph CSI driver with supported methods of CSI
// identity server spec.
type IdentityServer struct {
*csicommon.DefaultIdentityServer
}
func (is *identityServer) GetPluginCapabilities(ctx context.Context, req *csi.GetPluginCapabilitiesRequest) (*csi.GetPluginCapabilitiesResponse, error) {
// GetPluginCapabilities returns available capabilities of the ceph driver
func (is *IdentityServer) GetPluginCapabilities(ctx context.Context, req *csi.GetPluginCapabilitiesRequest) (*csi.GetPluginCapabilitiesResponse, error) {
return &csi.GetPluginCapabilitiesResponse{
Capabilities: []*csi.PluginCapability{
{

311
pkg/cephfs/mountcache.go Normal file
View File

@ -0,0 +1,311 @@
package cephfs
import (
"encoding/base64"
"os"
"sync"
"syscall"
"time"
"github.com/ceph/ceph-csi/pkg/util"
"github.com/pkg/errors"
"k8s.io/klog"
)
type volumeMountCacheEntry struct {
DriverVersion string `json:"driverVersion"`
VolumeID string `json:"volumeID"`
Secrets map[string]string `json:"secrets"`
StagingPath string `json:"stagingPath"`
TargetPaths map[string]bool `json:"targetPaths"`
CreateTime time.Time `json:"createTime"`
}
type volumeMountCacheMap struct {
volumes map[string]volumeMountCacheEntry
nodeCacheStore util.NodeCache
metadataStore util.CachePersister
}
var (
volumeMountCachePrefix = "cephfs-mount-cache-"
volumeMountCache volumeMountCacheMap
volumeMountCacheMtx sync.Mutex
)
func initVolumeMountCache(driverName string, mountCacheDir string, cachePersister util.CachePersister) {
volumeMountCache.volumes = make(map[string]volumeMountCacheEntry)
volumeMountCache.metadataStore = cachePersister
volumeMountCache.nodeCacheStore.BasePath = mountCacheDir
volumeMountCache.nodeCacheStore.CacheDir = driverName
klog.Infof("mount-cache: name: %s, version: %s, mountCacheDir: %s", driverName, version, mountCacheDir)
}
func remountCachedVolumes() error {
if err := os.MkdirAll(volumeMountCache.nodeCacheStore.BasePath, 0755); err != nil {
klog.Errorf("mount-cache: failed to create %s: %v", volumeMountCache.nodeCacheStore.BasePath, err)
return err
}
var remountFailCount, remountSuccCount int64
me := &volumeMountCacheEntry{}
ce := &controllerCacheEntry{}
err := volumeMountCache.nodeCacheStore.ForAll(volumeMountCachePrefix, me, func(identifier string) error {
volID := me.VolumeID
if err := volumeMountCache.metadataStore.Get(volID, ce); err != nil {
if err, ok := err.(*util.CacheEntryNotFound); ok {
klog.Infof("mount-cache: metadata not found, assuming the volume %s to be already deleted (%v)", volID, err)
if err := volumeMountCache.nodeCacheStore.Delete(genVolumeMountCacheFileName(volID)); err == nil {
klog.Infof("mount-cache: metadata not found, delete volume cache entry for volume %s", volID)
}
}
} else {
if err := mountOneCacheEntry(ce, me); err == nil {
remountSuccCount++
volumeMountCache.volumes[me.VolumeID] = *me
klog.Infof("mount-cache: successfully remounted volume %s", volID)
} else {
remountFailCount++
klog.Errorf("mount-cache: failed to remount volume %s", volID)
}
}
return nil
})
if err != nil {
klog.Infof("mount-cache: metastore list cache fail %v", err)
return err
}
if remountFailCount > 0 {
klog.Infof("mount-cache: successfully remounted %d volumes, failed to remount %d volumes", remountSuccCount, remountFailCount)
} else {
klog.Infof("mount-cache: successfully remounted %d volumes", remountSuccCount)
}
return nil
}
func mountOneCacheEntry(ce *controllerCacheEntry, me *volumeMountCacheEntry) error {
volumeMountCacheMtx.Lock()
defer volumeMountCacheMtx.Unlock()
var (
err error
cr *credentials
)
volID := ce.VolumeID
volOptions := ce.VolOptions
if volOptions.ProvisionVolume {
volOptions.RootPath = getVolumeRootPathCeph(volID)
cr, err = getAdminCredentials(decodeCredentials(me.Secrets))
if err != nil {
return err
}
var entity *cephEntity
entity, err = getCephUser(&volOptions, cr, volID)
if err != nil {
return err
}
cr = entity.toCredentials()
} else {
cr, err = getUserCredentials(decodeCredentials(me.Secrets))
if err != nil {
return err
}
}
err = cleanupMountPoint(me.StagingPath)
if err != nil {
klog.Infof("mount-cache: failed to cleanup volume mount point %s, remove it: %s %v", volID, me.StagingPath, err)
return err
}
isMnt, err := isMountPoint(me.StagingPath)
if err != nil {
isMnt = false
klog.Infof("mount-cache: failed to check volume mounted %s: %s %v", volID, me.StagingPath, err)
}
if !isMnt {
m, err := newMounter(&volOptions)
if err != nil {
klog.Errorf("mount-cache: failed to create mounter for volume %s: %v", volID, err)
return err
}
if err := m.mount(me.StagingPath, cr, &volOptions); err != nil {
klog.Errorf("mount-cache: failed to mount volume %s: %v", volID, err)
return err
}
}
for targetPath, readOnly := range me.TargetPaths {
if err := cleanupMountPoint(targetPath); err == nil {
if err := bindMount(me.StagingPath, targetPath, readOnly); err != nil {
klog.Errorf("mount-cache: failed to bind-mount volume %s: %s %s %v %v",
volID, me.StagingPath, targetPath, readOnly, err)
} else {
klog.Infof("mount-cache: successfully bind-mounted volume %s: %s %s %v",
volID, me.StagingPath, targetPath, readOnly)
}
}
}
return nil
}
func cleanupMountPoint(mountPoint string) error {
if _, err := os.Stat(mountPoint); err != nil {
if isCorruptedMnt(err) {
klog.Infof("mount-cache: corrupted mount point %s, need unmount", mountPoint)
err := execCommandErr("umount", mountPoint)
if err != nil {
klog.Infof("mount-cache: failed to umount %s %v", mountPoint, err)
//ignore error return err
}
}
}
if _, err := os.Stat(mountPoint); err != nil {
klog.Errorf("mount-cache: failed to stat mount point %s %v", mountPoint, err)
return err
}
return nil
}
func isCorruptedMnt(err error) bool {
var underlyingError error
switch pe := err.(type) {
case nil:
return false
case *os.PathError:
underlyingError = pe.Err
case *os.LinkError:
underlyingError = pe.Err
case *os.SyscallError:
underlyingError = pe.Err
default:
return false
}
CorruptedErrors := []error{
syscall.ENOTCONN, syscall.ESTALE, syscall.EIO, syscall.EACCES}
for _, v := range CorruptedErrors {
if underlyingError == v {
return true
}
}
return false
}
func genVolumeMountCacheFileName(volID string) string {
cachePath := volumeMountCachePrefix + volID
return cachePath
}
func (mc *volumeMountCacheMap) isEnable() bool {
//if mount cache dir unset, disable state
return mc.nodeCacheStore.BasePath != ""
}
func (mc *volumeMountCacheMap) nodeStageVolume(volID string, stagingTargetPath string, secrets map[string]string) error {
if !mc.isEnable() {
return nil
}
volumeMountCacheMtx.Lock()
defer volumeMountCacheMtx.Unlock()
lastTargetPaths := make(map[string]bool)
me, ok := volumeMountCache.volumes[volID]
if ok {
if me.StagingPath == stagingTargetPath {
klog.Warningf("mount-cache: node unexpected restage volume for volume %s", volID)
return nil
}
lastTargetPaths = me.TargetPaths
klog.Warningf("mount-cache: node stage volume ignore last cache entry for volume %s", volID)
}
me = volumeMountCacheEntry{DriverVersion: version}
me.VolumeID = volID
me.Secrets = encodeCredentials(secrets)
me.StagingPath = stagingTargetPath
me.TargetPaths = lastTargetPaths
me.CreateTime = time.Now()
volumeMountCache.volumes[volID] = me
return mc.nodeCacheStore.Create(genVolumeMountCacheFileName(volID), me)
}
func (mc *volumeMountCacheMap) nodeUnStageVolume(volID string) error {
if !mc.isEnable() {
return nil
}
volumeMountCacheMtx.Lock()
defer volumeMountCacheMtx.Unlock()
delete(volumeMountCache.volumes, volID)
return mc.nodeCacheStore.Delete(genVolumeMountCacheFileName(volID))
}
func (mc *volumeMountCacheMap) nodePublishVolume(volID string, targetPath string, readOnly bool) error {
if !mc.isEnable() {
return nil
}
volumeMountCacheMtx.Lock()
defer volumeMountCacheMtx.Unlock()
_, ok := volumeMountCache.volumes[volID]
if !ok {
return errors.New("mount-cache: node publish volume failed to find cache entry for volume")
}
volumeMountCache.volumes[volID].TargetPaths[targetPath] = readOnly
return mc.updateNodeCache(volID)
}
func (mc *volumeMountCacheMap) nodeUnPublishVolume(volID string, targetPath string) error {
if !mc.isEnable() {
return nil
}
volumeMountCacheMtx.Lock()
defer volumeMountCacheMtx.Unlock()
_, ok := volumeMountCache.volumes[volID]
if !ok {
return errors.New("mount-cache: node unpublish volume failed to find cache entry for volume")
}
delete(volumeMountCache.volumes[volID].TargetPaths, targetPath)
return mc.updateNodeCache(volID)
}
func (mc *volumeMountCacheMap) updateNodeCache(volID string) error {
me := volumeMountCache.volumes[volID]
if err := volumeMountCache.nodeCacheStore.Delete(genVolumeMountCacheFileName(volID)); err == nil {
klog.Infof("mount-cache: metadata not found, delete mount cache failed for volume %s", volID)
}
return mc.nodeCacheStore.Create(genVolumeMountCacheFileName(volID), me)
}
func encodeCredentials(input map[string]string) (output map[string]string) {
output = make(map[string]string)
for key, value := range input {
nKey := base64.StdEncoding.EncodeToString([]byte(key))
nValue := base64.StdEncoding.EncodeToString([]byte(value))
output[nKey] = nValue
}
return output
}
func decodeCredentials(input map[string]string) (output map[string]string) {
output = make(map[string]string)
for key, value := range input {
nKey, err := base64.StdEncoding.DecodeString(key)
if err != nil {
klog.Errorf("mount-cache: decode secret fail")
continue
}
nValue, err := base64.StdEncoding.DecodeString(value)
if err != nil {
klog.Errorf("mount-cache: decode secret fail")
continue
}
output[string(nKey)] = string(nValue)
}
return output
}

38
pkg/cephfs/mountcache_test.go Normal file
View File

@ -0,0 +1,38 @@
package cephfs
import (
"testing"
)
func init() {
}
func TestMountOneCacheEntry(t *testing.T) {
}
func TestRemountHisMountedPath(t *testing.T) {
}
func TestNodeStageVolume(t *testing.T) {
}
func TestNodeUnStageVolume(t *testing.T) {
}
func TestNodePublishVolume(t *testing.T) {
}
func TestNodeUnpublishVolume(t *testing.T) {
}
func TestEncodeDecodeCredentials(t *testing.T) {
secrets := make(map[string]string)
secrets["user_1"] = "value_1"
enSecrets := encodeCredentials(secrets)
deSecrets := decodeCredentials(enSecrets)
for key, value := range secrets {
if deSecrets[key] != value {
t.Errorf("key %s of credentials's value %s change after decode %s ", key, value, deSecrets[key])
}
}
}

179
pkg/cephfs/nodeserver.go
View File

@ -21,63 +21,65 @@ import (
"fmt"
"os"
"github.com/golang/glog"
csicommon "github.com/ceph/ceph-csi/pkg/csi-common"
"github.com/container-storage-interface/spec/lib/go/csi"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"github.com/container-storage-interface/spec/lib/go/csi/v0"
"github.com/kubernetes-csi/drivers/pkg/csi-common"
"k8s.io/klog"
"k8s.io/kubernetes/pkg/util/keymutex"
)
type nodeServer struct {
// NodeServer struct of ceph CSI driver with supported methods of CSI
// node server spec.
type NodeServer struct {
*csicommon.DefaultNodeServer
}
func getCredentialsForVolume(volOptions *volumeOptions, volId volumeID, req *csi.NodeStageVolumeRequest) (*credentials, error) {
var (
mtxNodeVolumeID = keymutex.NewHashed(0)
)
func getCredentialsForVolume(volOptions *volumeOptions, volID volumeID, req *csi.NodeStageVolumeRequest) (*credentials, error) {
var (
userCr *credentials
err error
cr *credentials
secrets = req.GetSecrets()
)
if volOptions.ProvisionVolume {
// The volume is provisioned dynamically, get the credentials directly from Ceph
// First, store admin credentials - those are needed for retrieving the user credentials
// First, get admin credentials - those are needed for retrieving the user credentials
adminCr, err := getAdminCredentials(req.GetNodeStageSecrets())
adminCr, err := getAdminCredentials(secrets)
if err != nil {
return nil, fmt.Errorf("failed to get admin credentials from node stage secrets: %v", err)
}
if err = storeCephCredentials(volId, adminCr); err != nil {
return nil, fmt.Errorf("failed to store ceph admin credentials: %v", err)
}
// Then get the ceph user
entity, err := getCephUser(adminCr, volId)
entity, err := getCephUser(volOptions, adminCr, volID)
if err != nil {
return nil, fmt.Errorf("failed to get ceph user: %v", err)
}
userCr = entity.toCredentials()
cr = entity.toCredentials()
} else {
// The volume is pre-made, credentials are in node stage secrets
userCr, err = getUserCredentials(req.GetNodeStageSecrets())
userCr, err := getUserCredentials(req.GetSecrets())
if err != nil {
return nil, fmt.Errorf("failed to get user credentials from node stage secrets: %v", err)
}
cr = userCr
}
if err = storeCephCredentials(volId, userCr); err != nil {
return nil, fmt.Errorf("failed to store ceph user credentials: %v", err)
}
return userCr, nil
return cr, nil
}
func (ns *nodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) {
// NodeStageVolume mounts the volume to a staging path on the node.
func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) {
if err := validateNodeStageVolumeRequest(req); err != nil {
return nil, status.Error(codes.InvalidArgument, err.Error())
}
@ -85,70 +87,82 @@ func (ns *nodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol
// Configuration
stagingTargetPath := req.GetStagingTargetPath()
volId := volumeID(req.GetVolumeId())
volID := volumeID(req.GetVolumeId())
volOptions, err := newVolumeOptions(req.GetVolumeAttributes())
volOptions, err := newVolumeOptions(req.GetVolumeContext(), req.GetSecrets())
if err != nil {
glog.Errorf("error reading volume options for volume %s: %v", volId, err)
klog.Errorf("error reading volume options for volume %s: %v", volID, err)
return nil, status.Error(codes.InvalidArgument, err.Error())
}
if volOptions.ProvisionVolume {
// Dynamically provisioned volumes don't have their root path set, do it here
volOptions.RootPath = getVolumeRootPathCeph(volId)
volOptions.RootPath = getVolumeRootPathCeph(volID)
}
if err = createMountPoint(stagingTargetPath); err != nil {
glog.Errorf("failed to create staging mount point at %s for volume %s: %v", stagingTargetPath, volId, err)
klog.Errorf("failed to create staging mount point at %s for volume %s: %v", stagingTargetPath, volID, err)
return nil, status.Error(codes.Internal, err.Error())
}
cephConf := cephConfigData{Monitors: volOptions.Monitors, VolumeID: volId}
if err = cephConf.writeToFile(); err != nil {
glog.Errorf("failed to write ceph config file to %s for volume %s: %v", getCephConfPath(volId), volId, err)
return nil, status.Error(codes.Internal, err.Error())
}
mtxNodeVolumeID.LockKey(string(volID))
defer mustUnlock(mtxNodeVolumeID, string(volID))
// Check if the volume is already mounted
isMnt, err := isMountPoint(stagingTargetPath)
if err != nil {
glog.Errorf("stat failed: %v", err)
klog.Errorf("stat failed: %v", err)
return nil, status.Error(codes.Internal, err.Error())
}
if isMnt {
glog.Infof("cephfs: volume %s is already mounted to %s, skipping", volId, stagingTargetPath)
klog.Infof("cephfs: volume %s is already mounted to %s, skipping", volID, stagingTargetPath)
return &csi.NodeStageVolumeResponse{}, nil
}
// It's not, mount now
cr, err := getCredentialsForVolume(volOptions, volId, req)
if err != nil {
glog.Errorf("failed to get ceph credentials for volume %s: %v", volId, err)
return nil, status.Error(codes.Internal, err.Error())
if err = ns.mount(volOptions, req); err != nil {
return nil, err
}
m, err := newMounter(volOptions)
if err != nil {
glog.Errorf("failed to create mounter for volume %s: %v", volId, err)
}
glog.V(4).Infof("cephfs: mounting volume %s with %s", volId, m.name())
if err = m.mount(stagingTargetPath, cr, volOptions, volId); err != nil {
glog.Errorf("failed to mount volume %s: %v", volId, err)
return nil, status.Error(codes.Internal, err.Error())
}
glog.Infof("cephfs: successfully mounted volume %s to %s", volId, stagingTargetPath)
klog.Infof("cephfs: successfully mounted volume %s to %s", volID, stagingTargetPath)
return &csi.NodeStageVolumeResponse{}, nil
}
func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) {
func (*NodeServer) mount(volOptions *volumeOptions, req *csi.NodeStageVolumeRequest) error {
stagingTargetPath := req.GetStagingTargetPath()
volID := volumeID(req.GetVolumeId())
cr, err := getCredentialsForVolume(volOptions, volID, req)
if err != nil {
klog.Errorf("failed to get ceph credentials for volume %s: %v", volID, err)
return status.Error(codes.Internal, err.Error())
}
m, err := newMounter(volOptions)
if err != nil {
klog.Errorf("failed to create mounter for volume %s: %v", volID, err)
return status.Error(codes.Internal, err.Error())
}
klog.V(4).Infof("cephfs: mounting volume %s with %s", volID, m.name())
if err = m.mount(stagingTargetPath, cr, volOptions); err != nil {
klog.Errorf("failed to mount volume %s: %v", volID, err)
return status.Error(codes.Internal, err.Error())
}
if err := volumeMountCache.nodeStageVolume(req.GetVolumeId(), stagingTargetPath, req.GetSecrets()); err != nil {
klog.Warningf("mount-cache: failed to stage volume %s %s: %v", volID, stagingTargetPath, err)
}
return nil
}
// NodePublishVolume mounts the volume mounted to the staging path to the target
// path
func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) {
if err := validateNodePublishVolumeRequest(req); err != nil {
return nil, status.Error(codes.InvalidArgument, err.Error())
}
@ -156,10 +170,10 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis
// Configuration
targetPath := req.GetTargetPath()
volId := req.GetVolumeId()
volID := req.GetVolumeId()
if err := createMountPoint(targetPath); err != nil {
glog.Errorf("failed to create mount point at %s: %v", targetPath, err)
klog.Errorf("failed to create mount point at %s: %v", targetPath, err)
return nil, status.Error(codes.Internal, err.Error())
}
@ -168,66 +182,89 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis
isMnt, err := isMountPoint(targetPath)
if err != nil {
glog.Errorf("stat failed: %v", err)
klog.Errorf("stat failed: %v", err)
return nil, status.Error(codes.Internal, err.Error())
}
if isMnt {
glog.Infof("cephfs: volume %s is already bind-mounted to %s", volId, targetPath)
klog.Infof("cephfs: volume %s is already bind-mounted to %s", volID, targetPath)
return &csi.NodePublishVolumeResponse{}, nil
}
// It's not, mount now
if err = bindMount(req.GetStagingTargetPath(), req.GetTargetPath(), req.GetReadonly()); err != nil {
glog.Errorf("failed to bind-mount volume %s: %v", volId, err)
klog.Errorf("failed to bind-mount volume %s: %v", volID, err)
return nil, status.Error(codes.Internal, err.Error())
}
glog.Infof("cephfs: successfully bind-mounted volume %s to %s", volId, targetPath)
if err := volumeMountCache.nodePublishVolume(volID, targetPath, req.GetReadonly()); err != nil {
klog.Warningf("mount-cache: failed to publish volume %s %s: %v", volID, targetPath, err)
}
klog.Infof("cephfs: successfully bind-mounted volume %s to %s", volID, targetPath)
return &csi.NodePublishVolumeResponse{}, nil
}
func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) {
if err := validateNodeUnpublishVolumeRequest(req); err != nil {
// NodeUnpublishVolume unmounts the volume from the target path
func (ns *NodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) {
var err error
if err = validateNodeUnpublishVolumeRequest(req); err != nil {
return nil, status.Error(codes.InvalidArgument, err.Error())
}
targetPath := req.GetTargetPath()
volID := req.GetVolumeId()
if err = volumeMountCache.nodeUnPublishVolume(volID, targetPath); err != nil {
klog.Warningf("mount-cache: failed to unpublish volume %s %s: %v", volID, targetPath, err)
}
// Unmount the bind-mount
if err := unmountVolume(targetPath); err != nil {
if err = unmountVolume(targetPath); err != nil {
return nil, status.Error(codes.Internal, err.Error())
}
os.Remove(targetPath)
if err = os.Remove(targetPath); err != nil {
return nil, status.Error(codes.Internal, err.Error())
}
glog.Infof("cephfs: successfully unbinded volume %s from %s", req.GetVolumeId(), targetPath)
klog.Infof("cephfs: successfully unbinded volume %s from %s", req.GetVolumeId(), targetPath)
return &csi.NodeUnpublishVolumeResponse{}, nil
}
func (ns *nodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error) {
if err := validateNodeUnstageVolumeRequest(req); err != nil {
// NodeUnstageVolume unstages the volume from the staging path
func (ns *NodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error) {
var err error
if err = validateNodeUnstageVolumeRequest(req); err != nil {
return nil, status.Error(codes.InvalidArgument, err.Error())
}
stagingTargetPath := req.GetStagingTargetPath()
volID := req.GetVolumeId()
if err = volumeMountCache.nodeUnStageVolume(volID); err != nil {
klog.Warningf("mount-cache: failed to unstage volume %s %s: %v", volID, stagingTargetPath, err)
}
// Unmount the volume
if err := unmountVolume(stagingTargetPath); err != nil {
if err = unmountVolume(stagingTargetPath); err != nil {
return nil, status.Error(codes.Internal, err.Error())
}
os.Remove(stagingTargetPath)
if err = os.Remove(stagingTargetPath); err != nil {
return nil, status.Error(codes.Internal, err.Error())
}
glog.Infof("cephfs: successfully umounted volume %s from %s", req.GetVolumeId(), stagingTargetPath)
klog.Infof("cephfs: successfully unmounted volume %s from %s", req.GetVolumeId(), stagingTargetPath)
return &csi.NodeUnstageVolumeResponse{}, nil
}
func (ns *nodeServer) NodeGetCapabilities(ctx context.Context, req *csi.NodeGetCapabilitiesRequest) (*csi.NodeGetCapabilitiesResponse, error) {
// NodeGetCapabilities returns the supported capabilities of the node server
func (ns *NodeServer) NodeGetCapabilities(ctx context.Context, req *csi.NodeGetCapabilitiesRequest) (*csi.NodeGetCapabilitiesResponse, error) {
return &csi.NodeGetCapabilitiesResponse{
Capabilities: []*csi.NodeServiceCapability{
{

118
pkg/cephfs/util.go
View File

@ -19,48 +19,70 @@ package cephfs
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"os"
"os/exec"
"github.com/golang/glog"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"k8s.io/klog"
"github.com/container-storage-interface/spec/lib/go/csi/v0"
"github.com/pborman/uuid"
"github.com/ceph/ceph-csi/pkg/util"
"github.com/container-storage-interface/spec/lib/go/csi"
"k8s.io/kubernetes/pkg/util/keymutex"
"k8s.io/kubernetes/pkg/util/mount"
)
type volumeID string
func newVolumeID() volumeID {
return volumeID("csi-cephfs-" + uuid.NewUUID().String())
func mustUnlock(m keymutex.KeyMutex, key string) {
if err := m.UnlockKey(key); err != nil {
klog.Fatalf("failed to unlock mutex for %s: %v", key, err)
}
}
func execCommand(command string, args ...string) ([]byte, error) {
glog.V(4).Infof("cephfs: EXEC %s %s", command, args)
cmd := exec.Command(command, args...)
return cmd.CombinedOutput()
func makeVolumeID(volName string) volumeID {
return volumeID("csi-cephfs-" + volName)
}
func execCommandAndValidate(program string, args ...string) error {
out, err := execCommand(program, args...)
if err != nil {
return fmt.Errorf("cephfs: %s failed with following error: %s\ncephfs: %s output: %s", program, err, program, out)
func execCommand(program string, args ...string) (stdout, stderr []byte, err error) {
var (
cmd = exec.Command(program, args...) // nolint: gosec
sanitizedArgs = util.StripSecretInArgs(args)
stdoutBuf bytes.Buffer
stderrBuf bytes.Buffer
)
cmd.Stdout = &stdoutBuf
cmd.Stderr = &stderrBuf
klog.V(4).Infof("cephfs: EXEC %s %s", program, sanitizedArgs)
if err := cmd.Run(); err != nil {
return nil, nil, fmt.Errorf("an error occurred while running (%d) %s %v: %v: %s",
cmd.Process.Pid, program, sanitizedArgs, err, stderrBuf.Bytes())
}
return nil
return stdoutBuf.Bytes(), stderrBuf.Bytes(), nil
}
func execCommandErr(program string, args ...string) error {
_, _, err := execCommand(program, args...)
return err
}
func execCommandJSON(v interface{}, program string, args ...string) error {
out, err := execCommand(program, args...)
stdout, _, err := execCommand(program, args...)
if err != nil {
return fmt.Errorf("cephfs: %s failed with following error: %s\ncephfs: %s output: %s", program, err, program, out)
return err
}
return json.NewDecoder(bytes.NewReader(out)).Decode(v)
if err = json.Unmarshal(stdout, v); err != nil {
return fmt.Errorf("failed to unmarshal JSON for %s %v: %s: %v", program, util.StripSecretInArgs(args), stdout, err)
}
return nil
}
// Used in isMountPoint()
@ -75,33 +97,13 @@ func isMountPoint(p string) (bool, error) {
return !notMnt, nil
}
func storeCephCredentials(volId volumeID, cr *credentials) error {
keyringData := cephKeyringData{
UserId: cr.id,
Key: cr.key,
VolumeID: volId,
}
if err := keyringData.writeToFile(); err != nil {
return err
}
secret := cephSecretData{
UserId: cr.id,
Key: cr.key,
VolumeID: volId,
}
err := secret.writeToFile()
return err
func pathExists(p string) bool {
_, err := os.Stat(p)
return err == nil
}
//
// Controller service request validation
//
func (cs *controllerServer) validateCreateVolumeRequest(req *csi.CreateVolumeRequest) error {
func (cs *ControllerServer) validateCreateVolumeRequest(req *csi.CreateVolumeRequest) error {
if err := cs.Driver.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME); err != nil {
return fmt.Errorf("invalid CreateVolumeRequest: %v", err)
}
@ -120,10 +122,11 @@ func (cs *controllerServer) validateCreateVolumeRequest(req *csi.CreateVolumeReq
return status.Error(codes.Unimplemented, "block volume not supported")
}
}
return nil
}
func (cs *controllerServer) validateDeleteVolumeRequest(req *csi.DeleteVolumeRequest) error {
func (cs *ControllerServer) validateDeleteVolumeRequest() error {
if err := cs.Driver.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME); err != nil {
return fmt.Errorf("invalid DeleteVolumeRequest: %v", err)
}
@ -131,25 +134,22 @@ func (cs *controllerServer) validateDeleteVolumeRequest(req *csi.DeleteVolumeReq
return nil
}
//
// Node service request validation
//
func validateNodeStageVolumeRequest(req *csi.NodeStageVolumeRequest) error {
if req.GetVolumeCapability() == nil {
return fmt.Errorf("volume capability missing in request")
return errors.New("volume capability missing in request")
}
if req.GetVolumeId() == "" {
return fmt.Errorf("volume ID missing in request")
return errors.New("volume ID missing in request")
}
if req.GetStagingTargetPath() == "" {
return fmt.Errorf("staging target path missing in request")
return errors.New("staging target path missing in request")
}
if req.GetNodeStageSecrets() == nil || len(req.GetNodeStageSecrets()) == 0 {
return fmt.Errorf("stage secrets cannot be nil or empty")
if req.GetSecrets() == nil || len(req.GetSecrets()) == 0 {
return errors.New("stage secrets cannot be nil or empty")
}
return nil
@ -157,11 +157,11 @@ func validateNodeStageVolumeRequest(req *csi.NodeStageVolumeRequest) error {
func validateNodeUnstageVolumeRequest(req *csi.NodeUnstageVolumeRequest) error {
if req.GetVolumeId() == "" {
return fmt.Errorf("volume ID missing in request")
return errors.New("volume ID missing in request")
}
if req.GetStagingTargetPath() == "" {
return fmt.Errorf("staging target path missing in request")
return errors.New("staging target path missing in request")
}
return nil
@ -169,15 +169,15 @@ func validateNodeUnstageVolumeRequest(req *csi.NodeUnstageVolumeRequest) error {
func validateNodePublishVolumeRequest(req *csi.NodePublishVolumeRequest) error {
if req.GetVolumeCapability() == nil {
return fmt.Errorf("volume capability missing in request")
return errors.New("volume capability missing in request")
}
if req.GetVolumeId() == "" {
return fmt.Errorf("volume ID missing in request")
return errors.New("volume ID missing in request")
}
if req.GetTargetPath() == "" {
return fmt.Errorf("varget path missing in request")
return errors.New("target path missing in request")
}
return nil
@ -185,11 +185,11 @@ func validateNodePublishVolumeRequest(req *csi.NodePublishVolumeRequest) error {
func validateNodeUnpublishVolumeRequest(req *csi.NodeUnpublishVolumeRequest) error {
if req.GetVolumeId() == "" {
return fmt.Errorf("volume ID missing in request")
return errors.New("volume ID missing in request")
}
if req.GetTargetPath() == "" {
return fmt.Errorf("target path missing in request")
return errors.New("target path missing in request")
}
return nil

134
pkg/cephfs/volume.go
View File

@ -20,6 +20,8 @@ import (
"fmt"
"os"
"path"
"k8s.io/klog"
)
const (
@ -28,113 +30,127 @@ const (
namespacePrefix = "ns-"
)
var (
cephRootPrefix = PluginFolder + "/controller/volumes/root-"
)
func getCephRootPathLocal(volId volumeID) string {
return cephRootPrefix + string(volId)
func getCephRootPathLocal(volID volumeID) string {
return fmt.Sprintf("%s/controller/volumes/root-%s", PluginFolder, string(volID))
}
func getCephRootVolumePathLocal(volId volumeID) string {
return path.Join(getCephRootPathLocal(volId), cephVolumesRoot, string(volId))
func getCephRootVolumePathLocal(volID volumeID) string {
return path.Join(getCephRootPathLocal(volID), cephVolumesRoot, string(volID))
}
func getVolumeRootPathCeph(volId volumeID) string {
return path.Join("/", cephVolumesRoot, string(volId))
func getVolumeRootPathCeph(volID volumeID) string {
return path.Join("/", cephVolumesRoot, string(volID))
}
func getVolumeNamespace(volId volumeID) string {
return namespacePrefix + string(volId)
func getVolumeNamespace(volID volumeID) string {
return namespacePrefix + string(volID)
}
func setVolumeAttribute(root, attrName, attrValue string) error {
return execCommandAndValidate("setfattr", "-n", attrName, "-v", attrValue, root)
return execCommandErr("setfattr", "-n", attrName, "-v", attrValue, root)
}
func createVolume(volOptions *volumeOptions, adminCr *credentials, volId volumeID, bytesQuota int64) error {
cephRoot := getCephRootPathLocal(volId)
if err := createMountPoint(cephRoot); err != nil {
func createVolume(volOptions *volumeOptions, adminCr *credentials, volID volumeID, bytesQuota int64) error {
if err := mountCephRoot(volID, volOptions, adminCr); err != nil {
return err
}
defer unmountCephRoot(volID)
// RootPath is not set for a dynamically provisioned volume
// Access to cephfs's / is required
volOptions.RootPath = "/"
var (
volRoot = getCephRootVolumePathLocal(volID)
volRootCreating = volRoot + "-creating"
)
m, err := newMounter(volOptions)
if err != nil {
return fmt.Errorf("failed to create mounter: %v", err)
if pathExists(volRoot) {
klog.V(4).Infof("cephfs: volume %s already exists, skipping creation", volID)
return nil
}
if err = m.mount(cephRoot, adminCr, volOptions, volId); err != nil {
return fmt.Errorf("error mounting ceph root: %v", err)
}
defer func() {
unmountVolume(cephRoot)
os.Remove(cephRoot)
}()
volOptions.RootPath = getVolumeRootPathCeph(volId)
localVolRoot := getCephRootVolumePathLocal(volId)
if err := createMountPoint(localVolRoot); err != nil {
if err := createMountPoint(volRootCreating); err != nil {
return err
}
if bytesQuota > 0 {
if err := setVolumeAttribute(localVolRoot, "ceph.quota.max_bytes", fmt.Sprintf("%d", bytesQuota)); err != nil {
if err := setVolumeAttribute(volRootCreating, "ceph.quota.max_bytes", fmt.Sprintf("%d", bytesQuota)); err != nil {
return err
}
}
if err := setVolumeAttribute(localVolRoot, "ceph.dir.layout.pool", volOptions.Pool); err != nil {
if err := setVolumeAttribute(volRootCreating, "ceph.dir.layout.pool", volOptions.Pool); err != nil {
return fmt.Errorf("%v\ncephfs: Does pool '%s' exist?", err, volOptions.Pool)
}
err = setVolumeAttribute(localVolRoot, "ceph.dir.layout.pool_namespace", getVolumeNamespace(volId))
if err := setVolumeAttribute(volRootCreating, "ceph.dir.layout.pool_namespace", getVolumeNamespace(volID)); err != nil {
return err
}
return err
if err := os.Rename(volRootCreating, volRoot); err != nil {
return fmt.Errorf("couldn't mark volume %s as created: %v", volID, err)
}
return nil
}
func purgeVolume(volId volumeID, adminCr *credentials, volOptions *volumeOptions) error {
func purgeVolume(volID volumeID, adminCr *credentials, volOptions *volumeOptions) error {
if err := mountCephRoot(volID, volOptions, adminCr); err != nil {
return err
}
defer unmountCephRoot(volID)
var (
cephRoot = getCephRootPathLocal(volId)
volRoot = getCephRootVolumePathLocal(volId)
volRoot = getCephRootVolumePathLocal(volID)
volRootDeleting = volRoot + "-deleting"
)
if err := createMountPoint(cephRoot); err != nil {
return err
if pathExists(volRoot) {
if err := os.Rename(volRoot, volRootDeleting); err != nil {
return fmt.Errorf("couldn't mark volume %s for deletion: %v", volID, err)
}
} else {
if !pathExists(volRootDeleting) {
klog.V(4).Infof("cephfs: volume %s not found, assuming it to be already deleted", volID)
return nil
}
}
if err := os.RemoveAll(volRootDeleting); err != nil {
return fmt.Errorf("failed to delete volume %s: %v", volID, err)
}
return nil
}
func mountCephRoot(volID volumeID, volOptions *volumeOptions, adminCr *credentials) error {
cephRoot := getCephRootPathLocal(volID)
// Root path is not set for dynamically provisioned volumes
// Access to cephfs's / is required
volOptions.RootPath = "/"
if err := createMountPoint(cephRoot); err != nil {
return err
}
m, err := newMounter(volOptions)
if err != nil {
return fmt.Errorf("failed to create mounter: %v", err)
}
if err = m.mount(cephRoot, adminCr, volOptions, volId); err != nil {
if err = m.mount(cephRoot, adminCr, volOptions); err != nil {
return fmt.Errorf("error mounting ceph root: %v", err)
}
defer func() {
unmountVolume(volRoot)
os.Remove(volRoot)
}()
if err := os.Rename(volRoot, volRootDeleting); err != nil {
return fmt.Errorf("coudln't mark volume %s for deletion: %v", volId, err)
}
if err := os.RemoveAll(volRootDeleting); err != nil {
return fmt.Errorf("failed to delete volume %s: %v", volId, err)
}
return nil
}
func unmountCephRoot(volID volumeID) {
cephRoot := getCephRootPathLocal(volID)
if err := unmountVolume(cephRoot); err != nil {
klog.Errorf("failed to unmount %s with error %s", cephRoot, err)
} else {
if err := os.Remove(cephRoot); err != nil {
klog.Errorf("failed to remove %s with error %s", cephRoot, err)
}
}
}

94
pkg/cephfs/volumemounter.go
View File

@ -17,10 +17,15 @@ limitations under the License.
package cephfs
import (
"bytes"
"errors"
"fmt"
"os"
"os/exec"
"regexp"
"strconv"
"sync"
"k8s.io/klog"
)
const (
@ -30,12 +35,20 @@ const (
var (
availableMounters []string
// maps a mountpoint to PID of its FUSE daemon
fusePidMap = make(map[string]int)
fusePidMapMtx sync.Mutex
fusePidRx = regexp.MustCompile(`(?m)^ceph-fuse\[(.+)\]: starting fuse$`)
)
// Load available ceph mounters installed on system into availableMounters
// Called from driver.go's Run()
func loadAvailableMounters() error {
// #nosec
fuseMounterProbe := exec.Command("ceph-fuse", "--version")
// #nosec
kernelMounterProbe := exec.Command("mount.ceph")
if fuseMounterProbe.Run() == nil {
@ -47,14 +60,14 @@ func loadAvailableMounters() error {
}
if len(availableMounters) == 0 {
return fmt.Errorf("no ceph mounters found on system")
return errors.New("no ceph mounters found on system")
}
return nil
}
type volumeMounter interface {
mount(mountPoint string, cr *credentials, volOptions *volumeOptions, volId volumeID) error
mount(mountPoint string, cr *credentials, volOptions *volumeOptions) error
name() string
}
@ -98,71 +111,84 @@ func newMounter(volOptions *volumeOptions) (volumeMounter, error) {
type fuseMounter struct{}
func mountFuse(mountPoint string, cr *credentials, volOptions *volumeOptions, volId volumeID) error {
func mountFuse(mountPoint string, cr *credentials, volOptions *volumeOptions) error {
args := [...]string{
mountPoint,
"-c", getCephConfPath(volId),
"-n", cephEntityClientPrefix + cr.id,
"--keyring", getCephKeyringPath(volId, cr.id),
"-m", volOptions.Monitors,
"-c", cephConfigPath,
"-n", cephEntityClientPrefix + cr.id, "--key=" + cr.key,
"-r", volOptions.RootPath,
"-o", "nonempty",
}
out, err := execCommand("ceph-fuse", args[:]...)
_, stderr, err := execCommand("ceph-fuse", args[:]...)
if err != nil {
return fmt.Errorf("cephfs: ceph-fuse failed with following error: %s\ncephfs: ceph-fuse output: %s", err, out)
return err
}
if !bytes.Contains(out, []byte("starting fuse")) {
return fmt.Errorf("cephfs: ceph-fuse failed:\ncephfs: ceph-fuse output: %s", out)
// Parse the output:
// We need "starting fuse" meaning the mount is ok
// and PID of the ceph-fuse daemon for unmount
match := fusePidRx.FindSubmatch(stderr)
if len(match) != 2 {
return fmt.Errorf("ceph-fuse failed: %s", stderr)
}
pid, err := strconv.Atoi(string(match[1]))
if err != nil {
return fmt.Errorf("failed to parse FUSE daemon PID: %v", err)
}
fusePidMapMtx.Lock()
fusePidMap[mountPoint] = pid
fusePidMapMtx.Unlock()
return nil
}
func (m *fuseMounter) mount(mountPoint string, cr *credentials, volOptions *volumeOptions, volId volumeID) error {
func (m *fuseMounter) mount(mountPoint string, cr *credentials, volOptions *volumeOptions) error {
if err := createMountPoint(mountPoint); err != nil {
return err
}
return mountFuse(mountPoint, cr, volOptions, volId)
return mountFuse(mountPoint, cr, volOptions)
}
func (m *fuseMounter) name() string { return "Ceph FUSE driver" }
type kernelMounter struct{}
func mountKernel(mountPoint string, cr *credentials, volOptions *volumeOptions, volId volumeID) error {
if err := execCommandAndValidate("modprobe", "ceph"); err != nil {
func mountKernel(mountPoint string, cr *credentials, volOptions *volumeOptions) error {
if err := execCommandErr("modprobe", "ceph"); err != nil {
return err
}
return execCommandAndValidate("mount",
return execCommandErr("mount",
"-t", "ceph",
fmt.Sprintf("%s:%s", volOptions.Monitors, volOptions.RootPath),
mountPoint,
"-o",
fmt.Sprintf("name=%s,secretfile=%s", cr.id, getCephSecretPath(volId, cr.id)),
"-o", fmt.Sprintf("name=%s,secret=%s", cr.id, cr.key),
)
}
func (m *kernelMounter) mount(mountPoint string, cr *credentials, volOptions *volumeOptions, volId volumeID) error {
func (m *kernelMounter) mount(mountPoint string, cr *credentials, volOptions *volumeOptions) error {
if err := createMountPoint(mountPoint); err != nil {
return err
}
return mountKernel(mountPoint, cr, volOptions, volId)
return mountKernel(mountPoint, cr, volOptions)
}
func (m *kernelMounter) name() string { return "Ceph kernel client" }
func bindMount(from, to string, readOnly bool) error {
if err := execCommandAndValidate("mount", "--bind", from, to); err != nil {
if err := execCommandErr("mount", "--bind", from, to); err != nil {
return fmt.Errorf("failed to bind-mount %s to %s: %v", from, to, err)
}
if readOnly {
if err := execCommandAndValidate("mount", "-o", "remount,ro,bind", to); err != nil {
if err := execCommandErr("mount", "-o", "remount,ro,bind", to); err != nil {
return fmt.Errorf("failed read-only remount of %s: %v", to, err)
}
}
@ -171,7 +197,29 @@ func bindMount(from, to string, readOnly bool) error {
}
func unmountVolume(mountPoint string) error {
return execCommandAndValidate("umount", mountPoint)
if err := execCommandErr("umount", mountPoint); err != nil {
return err
}
fusePidMapMtx.Lock()
pid, ok := fusePidMap[mountPoint]
if ok {
delete(fusePidMap, mountPoint)
}
fusePidMapMtx.Unlock()
if ok {
p, err := os.FindProcess(pid)
if err != nil {
klog.Warningf("failed to find process %d: %v", pid, err)
} else {
if _, err = p.Wait(); err != nil {
klog.Warningf("%d is not a child process: %v", pid, err)
}
}
}
return nil
}
func createMountPoint(root string) error {

Some files were not shown because too many files have changed in this diff Show More