mirror of
https://github.com/ceph/ceph-csi.git
synced 2024-11-22 14:20:19 +00:00
Merge pull request #297 from Madhu-1/master
merge csiv1.0 branch to master
This commit is contained in:
commit
426bf67966
23
.github/ISSUE_TEMPLATE/bug_report.md
vendored
23
.github/ISSUE_TEMPLATE/bug_report.md
vendored
@ -1,23 +0,0 @@
|
||||
---
|
||||
name: Bug report
|
||||
about: Create a report to help us improve
|
||||
title: ''
|
||||
labels: ''
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
**Describe the bug**
|
||||
A clear and concise description of what the bug is.
|
||||
|
||||
**Kubernetes and Ceph CSI Versions**
|
||||
Kubernetes: 1.12 and below or 1.13 and above
|
||||
Ceph CSI: 0.3.0 or 1.0.0
|
||||
|
||||
**Ceph CSI Driver logs**
|
||||
Post logs from rbdplugin/cephfs plugin, provisioner, etc.
|
||||
|
||||
**To Reproduce**
|
||||
|
||||
**Expected behavior**
|
||||
A clear and concise description of what you expected to happen.
|
20
.github/ISSUE_TEMPLATE/feature_request.md
vendored
20
.github/ISSUE_TEMPLATE/feature_request.md
vendored
@ -1,20 +0,0 @@
|
||||
---
|
||||
name: Feature request
|
||||
about: Suggest an idea for this project
|
||||
title: ''
|
||||
labels: ''
|
||||
assignees: ''
|
||||
|
||||
---
|
||||
|
||||
**Is your feature request related to a problem? Please describe.**
|
||||
A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
|
||||
|
||||
**Describe the solution you'd like**
|
||||
A clear and concise description of what you want to happen.
|
||||
|
||||
**Describe alternatives you've considered**
|
||||
A clear and concise description of any alternative solutions or features you've considered.
|
||||
|
||||
**Additional context**
|
||||
Add any other context or screenshots about the feature request here.
|
@ -1,3 +1,4 @@
|
||||
---
|
||||
pull_request_rules:
|
||||
- name: automatic merge
|
||||
conditions:
|
||||
|
53
.travis.yml
53
.travis.yml
@ -1,24 +1,55 @@
|
||||
---
|
||||
# need for docker build
|
||||
sudo: true
|
||||
|
||||
addons:
|
||||
apt:
|
||||
packages:
|
||||
- realpath
|
||||
- ruby
|
||||
|
||||
language: go
|
||||
branches:
|
||||
only:
|
||||
- csi-v0.3
|
||||
- master
|
||||
- csi-v1.0 # remove this once csi-v1.0 becomes master
|
||||
|
||||
go: 1.9.x
|
||||
go: 1.11.x
|
||||
|
||||
before_script:
|
||||
- GO_FILES=$(find . -iname '*.go' -type f | grep -v /vendor/)
|
||||
- go get -u golang.org/x/lint/golint #go get github.com/golang/lint/golint
|
||||
env:
|
||||
global:
|
||||
- GOLANGCI_VERSION="v1.15.0"
|
||||
- TEST_COVERAGE=stdout
|
||||
- GO_METALINTER_THREADS=1
|
||||
- GO_COVER_DIR=_output
|
||||
|
||||
script:
|
||||
- test -z $(gofmt -s -l $GO_FILES)
|
||||
- go vet -v $(go list ./... | grep -v /vendor/)
|
||||
- make rbdplugin
|
||||
- make cephfsplugin
|
||||
jobs:
|
||||
include:
|
||||
- name: Linter
|
||||
install:
|
||||
- gem install mdl
|
||||
- pip install --user --upgrade pip
|
||||
- pip install --user yamllint
|
||||
# install golangci-lint
|
||||
- curl -sf
|
||||
"https://install.goreleaser.com/github.com/golangci/golangci-lint.sh"
|
||||
| bash -s -- -b $GOPATH/bin "${GOLANGCI_VERSION}"
|
||||
script:
|
||||
- scripts/lint-text.sh --require-all
|
||||
- scripts/lint-go.sh
|
||||
- scripts/test-go.sh
|
||||
|
||||
- name: rbdplugin
|
||||
script:
|
||||
- make rbdplugin
|
||||
|
||||
- name: cephfsplugin
|
||||
script:
|
||||
- make cephfsplugin
|
||||
|
||||
deploy:
|
||||
- provider: script
|
||||
script:
|
||||
- ./deploy.sh
|
||||
on: # yamllint disable-line rule:truthy
|
||||
all_branches: true
|
||||
script: ./deploy.sh
|
||||
|
283
Gopkg.lock
generated
283
Gopkg.lock
generated
@ -2,145 +2,217 @@
|
||||
|
||||
|
||||
[[projects]]
|
||||
digest = "1:94ffc0947c337d618b6ff5ed9abaddc1217b090c1b3a1ae4739b35b7b25851d5"
|
||||
name = "github.com/container-storage-interface/spec"
|
||||
packages = ["lib/go/csi/v0"]
|
||||
revision = "2178fdeea87f1150a17a63252eee28d4d8141f72"
|
||||
version = "v0.3.0"
|
||||
|
||||
[[projects]]
|
||||
name = "github.com/ghodss/yaml"
|
||||
packages = ["."]
|
||||
revision = "0ca9ea5df5451ffdf184b4428c902747c2c11cd7"
|
||||
packages = ["lib/go/csi"]
|
||||
pruneopts = "NUT"
|
||||
revision = "ed0bb0e1557548aa028307f48728767cfe8f6345"
|
||||
version = "v1.0.0"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:abea725bcf0210887f5da19d804fffa1dd45a42a56bdf5f02322345e3fee4f0d"
|
||||
name = "github.com/gogo/protobuf"
|
||||
packages = [
|
||||
"proto",
|
||||
"sortkeys"
|
||||
"sortkeys",
|
||||
]
|
||||
pruneopts = "NUT"
|
||||
revision = "4cbf7e384e768b4e01799441fdf2a706a5635ae7"
|
||||
version = "v1.2.0"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
name = "github.com/golang/glog"
|
||||
packages = ["."]
|
||||
revision = "23def4e6c14b4da8ac2ed8007337bc5eb5007998"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:bff0ce7c8e3d6357fa5a8549bbe4bdb620bddc13c11ae569aa7248ea92e2139f"
|
||||
name = "github.com/golang/protobuf"
|
||||
packages = [
|
||||
"descriptor",
|
||||
"proto",
|
||||
"protoc-gen-go/descriptor",
|
||||
"ptypes",
|
||||
"ptypes/any",
|
||||
"ptypes/duration",
|
||||
"ptypes/timestamp",
|
||||
"ptypes/wrappers"
|
||||
"ptypes/wrappers",
|
||||
]
|
||||
revision = "b4deda0973fb4c70b50d226b1af49f3da59f5265"
|
||||
version = "v1.1.0"
|
||||
pruneopts = "NUT"
|
||||
revision = "aa810b61a9c79d51363740d207bb46cf8e620ed5"
|
||||
version = "v1.2.0"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:05f95ffdfcf651bdb0f05b40b69e7f5663047f8da75c72d58728acb59b5cc107"
|
||||
name = "github.com/google/btree"
|
||||
packages = ["."]
|
||||
pruneopts = "NUT"
|
||||
revision = "4030bb1f1f0c35b30ca7009e9ebd06849dd45306"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:52c5834e2bebac9030c97cc0798ac11c3aa8a39f098aeb419f142533da6cd3cc"
|
||||
name = "github.com/google/gofuzz"
|
||||
packages = ["."]
|
||||
pruneopts = "NUT"
|
||||
revision = "24818f796faf91cd76ec7bddd72458fbced7a6c1"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:56a1f3949ebb7fa22fa6b4e4ac0fe0f77cc4faee5b57413e6fa9199a8458faf1"
|
||||
name = "github.com/google/uuid"
|
||||
packages = ["."]
|
||||
pruneopts = "NUT"
|
||||
revision = "9b3b1e0f5f99ae461456d768e7d301a7acdaa2d8"
|
||||
version = "v1.1.0"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:06a7dadb7b760767341ffb6c8d377238d68a1226f2b21b5d497d2e3f6ecf6b4e"
|
||||
name = "github.com/googleapis/gnostic"
|
||||
packages = [
|
||||
"OpenAPIv2",
|
||||
"compiler",
|
||||
"extensions"
|
||||
"extensions",
|
||||
]
|
||||
pruneopts = "NUT"
|
||||
revision = "7c663266750e7d82587642f65e60bc4083f1f84e"
|
||||
version = "v0.2.0"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
name = "github.com/howeyc/gopass"
|
||||
packages = ["."]
|
||||
revision = "bf9dde6d0d2c004a008c27aaee91170c786f6db8"
|
||||
digest = "1:7fdf3223c7372d1ced0b98bf53457c5e89d89aecbad9a77ba9fcc6e01f9e5621"
|
||||
name = "github.com/gregjones/httpcache"
|
||||
packages = [
|
||||
".",
|
||||
"diskcache",
|
||||
]
|
||||
pruneopts = "NUT"
|
||||
revision = "c63ab54fda8f77302f8d414e19933f2b6026a089"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:9a52adf44086cead3b384e5d0dbf7a1c1cce65e67552ee3383a8561c42a18cd3"
|
||||
name = "github.com/imdario/mergo"
|
||||
packages = ["."]
|
||||
pruneopts = "NUT"
|
||||
revision = "9f23e2d6bd2a77f959b2bf6acdbefd708a83a4a4"
|
||||
version = "v0.3.6"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:8e36686e8b139f8fe240c1d5cf3a145bc675c22ff8e707857cdd3ae17b00d728"
|
||||
name = "github.com/json-iterator/go"
|
||||
packages = ["."]
|
||||
pruneopts = "NUT"
|
||||
revision = "1624edc4454b8682399def8740d46db5e4362ba4"
|
||||
version = "v1.1.5"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
name = "github.com/kubernetes-csi/drivers"
|
||||
packages = ["pkg/csi-common"]
|
||||
revision = "d8f283cd941f1e24e20e62653476a3722633f43e"
|
||||
digest = "1:2b060bb1a39127e592baf9ab62ec1e94100dc22107f915183f3cd1f6d1cd579a"
|
||||
name = "github.com/kubernetes-csi/csi-lib-utils"
|
||||
packages = ["protosanitizer"]
|
||||
pruneopts = "NUT"
|
||||
revision = "5853414e1d4771302e0df10d1870c444c2135799"
|
||||
version = "v0.2.0"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:2f42fa12d6911c7b7659738758631bec870b7e9b4c6be5444f963cdcfccc191f"
|
||||
name = "github.com/modern-go/concurrent"
|
||||
packages = ["."]
|
||||
pruneopts = "NUT"
|
||||
revision = "bacd9c7ef1dd9b15be4a9909b8ac7a4e313eec94"
|
||||
version = "1.0.3"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:c6aca19413b13dc59c220ad7430329e2ec454cc310bc6d8de2c7e2b93c18a0f6"
|
||||
name = "github.com/modern-go/reflect2"
|
||||
packages = ["."]
|
||||
pruneopts = "NUT"
|
||||
revision = "4b7aa43c6742a2c18fdef89dd197aaae7dac7ccd"
|
||||
version = "1.0.1"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:93b1d84c5fa6d1ea52f4114c37714cddd84d5b78f151b62bb101128dd51399bf"
|
||||
name = "github.com/pborman/uuid"
|
||||
packages = ["."]
|
||||
revision = "e790cca94e6cc75c7064b1332e63811d4aae1a53"
|
||||
version = "v1.1"
|
||||
pruneopts = "NUT"
|
||||
revision = "adf5a7427709b9deb95d29d3fa8a2bf9cfd388f1"
|
||||
version = "v1.2"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:3bf17a6e6eaa6ad24152148a631d18662f7212e21637c2699bff3369b7f00fa2"
|
||||
name = "github.com/petar/GoLLRB"
|
||||
packages = ["llrb"]
|
||||
pruneopts = "NUT"
|
||||
revision = "53be0d36a84c2a886ca057d34b6aa4468df9ccb4"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:6c6d91dc326ed6778783cff869c49fb2f61303cdd2ebbcf90abe53505793f3b6"
|
||||
name = "github.com/peterbourgon/diskv"
|
||||
packages = ["."]
|
||||
pruneopts = "NUT"
|
||||
revision = "5f041e8faa004a95c88a202771f4cc3e991971e6"
|
||||
version = "v2.0.1"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:14715f705ff5dfe0ffd6571d7d201dd8e921030f8070321a79380d8ca4ec1a24"
|
||||
name = "github.com/pkg/errors"
|
||||
packages = ["."]
|
||||
revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
|
||||
version = "v0.8.0"
|
||||
pruneopts = "NUT"
|
||||
revision = "ba968bfe8b2f7e042a574c888954fccecfa385b4"
|
||||
version = "v0.8.1"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:9d8420bbf131d1618bde6530af37c3799340d3762cc47210c1d9532a4c3a2779"
|
||||
name = "github.com/spf13/pflag"
|
||||
packages = ["."]
|
||||
pruneopts = "NUT"
|
||||
revision = "298182f68c66c05229eb03ac171abe6e309ee79a"
|
||||
version = "v1.0.3"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:38f553aff0273ad6f367cb0a0f8b6eecbaef8dc6cb8b50e57b6a81c1d5b1e332"
|
||||
name = "golang.org/x/crypto"
|
||||
packages = ["ssh/terminal"]
|
||||
revision = "505ab145d0a99da450461ae2c1a9f6cd10d1f447"
|
||||
pruneopts = "NUT"
|
||||
revision = "ff983b9c42bc9fbf91556e191cc8efb585c16908"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:d4e37d487310720926343302a747f3f9e8f020e5fe961190c57ce437159a6e63"
|
||||
name = "golang.org/x/net"
|
||||
packages = [
|
||||
"context",
|
||||
"context/ctxhttp",
|
||||
"http/httpguts",
|
||||
"http2",
|
||||
"http2/hpack",
|
||||
"idna",
|
||||
"internal/timeseries",
|
||||
"trace"
|
||||
"trace",
|
||||
]
|
||||
revision = "3673e40ba22529d22c3fd7c93e97b0ce50fa7bdd"
|
||||
pruneopts = "NUT"
|
||||
revision = "915654e7eabcea33ae277abbecf52f0d8b7a9fdc"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:293b8e3359faf71cc5f85f3c144115ca6540396f4e1ff0fa30cd014c04258c30"
|
||||
name = "golang.org/x/oauth2"
|
||||
packages = [
|
||||
".",
|
||||
"internal",
|
||||
]
|
||||
pruneopts = "NUT"
|
||||
revision = "36a7019397c4c86cf59eeab3bc0d188bac444277"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:0f0298002380ddbc31230516fc1dc354ff466e607823e9122d69cce79310bdc9"
|
||||
name = "golang.org/x/sys"
|
||||
packages = [
|
||||
"unix",
|
||||
"windows"
|
||||
"windows",
|
||||
]
|
||||
revision = "e072cadbbdc8dd3d3ffa82b8b4b9304c261d9311"
|
||||
pruneopts = "NUT"
|
||||
revision = "a457fd036447854c0c02e89ea439481bdcf941a2"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:e7071ed636b5422cc51c0e3a6cebc229d6c9fffc528814b519a980641422d619"
|
||||
name = "golang.org/x/text"
|
||||
packages = [
|
||||
"collate",
|
||||
@ -156,40 +228,69 @@
|
||||
"unicode/bidi",
|
||||
"unicode/cldr",
|
||||
"unicode/norm",
|
||||
"unicode/rangetable"
|
||||
"unicode/rangetable",
|
||||
]
|
||||
pruneopts = "NUT"
|
||||
revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0"
|
||||
version = "v0.3.0"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:9fdc2b55e8e0fafe4b41884091e51e77344f7dc511c5acedcfd98200003bff90"
|
||||
name = "golang.org/x/time"
|
||||
packages = ["rate"]
|
||||
pruneopts = "NUT"
|
||||
revision = "85acf8d2951cb2a3bde7632f9ff273ef0379bcbd"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
name = "google.golang.org/genproto"
|
||||
packages = ["googleapis/rpc/status"]
|
||||
revision = "2a72893556e4d1f6c795a4c039314c9fa751eedb"
|
||||
digest = "1:34c10243da5972105edd1b4b883e2bd918fbb3f73fbe14d6af6929e547173494"
|
||||
name = "google.golang.org/appengine"
|
||||
packages = [
|
||||
"internal",
|
||||
"internal/base",
|
||||
"internal/datastore",
|
||||
"internal/log",
|
||||
"internal/remote_api",
|
||||
"internal/urlfetch",
|
||||
"urlfetch",
|
||||
]
|
||||
pruneopts = "NUT"
|
||||
revision = "e9657d882bb81064595ca3b56cbe2546bbabf7b1"
|
||||
version = "v1.4.0"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:077c1c599507b3b3e9156d17d36e1e61928ee9b53a5b420f10f28ebd4a0b275c"
|
||||
name = "google.golang.org/genproto"
|
||||
packages = ["googleapis/rpc/status"]
|
||||
pruneopts = "NUT"
|
||||
revision = "db91494dd46c1fdcbbde05e5ff5eb56df8f7d79a"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:638e6e596d67d0a0c8aeb76ebdcf73561b701ea43f21963b1db231d96ed7db68"
|
||||
name = "google.golang.org/grpc"
|
||||
packages = [
|
||||
".",
|
||||
"balancer",
|
||||
"balancer/base",
|
||||
"balancer/roundrobin",
|
||||
"binarylog/grpc_binarylog_v1",
|
||||
"codes",
|
||||
"connectivity",
|
||||
"credentials",
|
||||
"credentials/internal",
|
||||
"encoding",
|
||||
"encoding/proto",
|
||||
"grpclog",
|
||||
"internal",
|
||||
"internal/backoff",
|
||||
"internal/binarylog",
|
||||
"internal/channelz",
|
||||
"internal/envconfig",
|
||||
"internal/grpcrand",
|
||||
"internal/grpcsync",
|
||||
"internal/syscall",
|
||||
"internal/transport",
|
||||
"keepalive",
|
||||
"metadata",
|
||||
"naming",
|
||||
@ -200,24 +301,29 @@
|
||||
"stats",
|
||||
"status",
|
||||
"tap",
|
||||
"transport"
|
||||
]
|
||||
revision = "168a6198bcb0ef175f7dacec0b8691fc141dc9b8"
|
||||
version = "v1.13.0"
|
||||
pruneopts = "NUT"
|
||||
revision = "df014850f6dee74ba2fc94874043a9f3f75fbfd8"
|
||||
version = "v1.17.0"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:2d1fbdc6777e5408cabeb02bf336305e724b925ff4546ded0fa8715a7267922a"
|
||||
name = "gopkg.in/inf.v0"
|
||||
packages = ["."]
|
||||
pruneopts = "NUT"
|
||||
revision = "d2d2541c53f18d2a059457998ce2876cc8e67cbf"
|
||||
version = "v0.9.1"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:18108594151654e9e696b27b181b953f9a90b16bf14d253dd1b397b025a1487f"
|
||||
name = "gopkg.in/yaml.v2"
|
||||
packages = ["."]
|
||||
pruneopts = "NUT"
|
||||
revision = "51d6538a90f86fe93ac480b35f37b2be17fef232"
|
||||
version = "v2.2.2"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:c453ddc26bdab1e4267683a588ad9046e48d803a73f124fe2927adbab6ff02a5"
|
||||
name = "k8s.io/api"
|
||||
packages = [
|
||||
"admissionregistration/v1alpha1",
|
||||
@ -225,16 +331,19 @@
|
||||
"apps/v1",
|
||||
"apps/v1beta1",
|
||||
"apps/v1beta2",
|
||||
"auditregistration/v1alpha1",
|
||||
"authentication/v1",
|
||||
"authentication/v1beta1",
|
||||
"authorization/v1",
|
||||
"authorization/v1beta1",
|
||||
"autoscaling/v1",
|
||||
"autoscaling/v2beta1",
|
||||
"autoscaling/v2beta2",
|
||||
"batch/v1",
|
||||
"batch/v1beta1",
|
||||
"batch/v2alpha1",
|
||||
"certificates/v1beta1",
|
||||
"coordination/v1beta1",
|
||||
"core/v1",
|
||||
"events/v1beta1",
|
||||
"extensions/v1beta1",
|
||||
@ -244,15 +353,18 @@
|
||||
"rbac/v1alpha1",
|
||||
"rbac/v1beta1",
|
||||
"scheduling/v1alpha1",
|
||||
"scheduling/v1beta1",
|
||||
"settings/v1alpha1",
|
||||
"storage/v1",
|
||||
"storage/v1alpha1",
|
||||
"storage/v1beta1"
|
||||
"storage/v1beta1",
|
||||
]
|
||||
revision = "7aac3e00a1b32fa476b83078cebaaca606b2fb48"
|
||||
version = "kubernetes-1.10.0-beta.1"
|
||||
pruneopts = "NUT"
|
||||
revision = "74b699b93c15473932b89e3d1818ba8282f3b5ab"
|
||||
version = "kubernetes-1.13.3"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:09dee8b7c6cb2fc9c6bee525de3b95199a82a8647a189e153d072a1dfce17de7"
|
||||
name = "k8s.io/apimachinery"
|
||||
packages = [
|
||||
"pkg/api/errors",
|
||||
@ -280,6 +392,7 @@
|
||||
"pkg/util/framer",
|
||||
"pkg/util/intstr",
|
||||
"pkg/util/json",
|
||||
"pkg/util/naming",
|
||||
"pkg/util/net",
|
||||
"pkg/util/runtime",
|
||||
"pkg/util/sets",
|
||||
@ -289,12 +402,14 @@
|
||||
"pkg/util/yaml",
|
||||
"pkg/version",
|
||||
"pkg/watch",
|
||||
"third_party/forked/golang/reflect"
|
||||
"third_party/forked/golang/reflect",
|
||||
]
|
||||
revision = "302974c03f7e50f16561ba237db776ab93594ef6"
|
||||
version = "kubernetes-1.10.0-beta.1"
|
||||
pruneopts = "NUT"
|
||||
revision = "572dfc7bdfcb4531361a17d27b92851f59acf0dc"
|
||||
version = "kubernetes-1.13.3"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:638623327cb201b425a328d0bddb3379b05eb05ef4cab589380f0be07ac1dc17"
|
||||
name = "k8s.io/client-go"
|
||||
packages = [
|
||||
"discovery",
|
||||
@ -305,16 +420,19 @@
|
||||
"kubernetes/typed/apps/v1",
|
||||
"kubernetes/typed/apps/v1beta1",
|
||||
"kubernetes/typed/apps/v1beta2",
|
||||
"kubernetes/typed/auditregistration/v1alpha1",
|
||||
"kubernetes/typed/authentication/v1",
|
||||
"kubernetes/typed/authentication/v1beta1",
|
||||
"kubernetes/typed/authorization/v1",
|
||||
"kubernetes/typed/authorization/v1beta1",
|
||||
"kubernetes/typed/autoscaling/v1",
|
||||
"kubernetes/typed/autoscaling/v2beta1",
|
||||
"kubernetes/typed/autoscaling/v2beta2",
|
||||
"kubernetes/typed/batch/v1",
|
||||
"kubernetes/typed/batch/v1beta1",
|
||||
"kubernetes/typed/batch/v2alpha1",
|
||||
"kubernetes/typed/certificates/v1beta1",
|
||||
"kubernetes/typed/coordination/v1beta1",
|
||||
"kubernetes/typed/core/v1",
|
||||
"kubernetes/typed/events/v1beta1",
|
||||
"kubernetes/typed/extensions/v1beta1",
|
||||
@ -324,11 +442,16 @@
|
||||
"kubernetes/typed/rbac/v1alpha1",
|
||||
"kubernetes/typed/rbac/v1beta1",
|
||||
"kubernetes/typed/scheduling/v1alpha1",
|
||||
"kubernetes/typed/scheduling/v1beta1",
|
||||
"kubernetes/typed/settings/v1alpha1",
|
||||
"kubernetes/typed/storage/v1",
|
||||
"kubernetes/typed/storage/v1alpha1",
|
||||
"kubernetes/typed/storage/v1beta1",
|
||||
"pkg/apis/clientauthentication",
|
||||
"pkg/apis/clientauthentication/v1alpha1",
|
||||
"pkg/apis/clientauthentication/v1beta1",
|
||||
"pkg/version",
|
||||
"plugin/pkg/client/auth/exec",
|
||||
"rest",
|
||||
"rest/watch",
|
||||
"tools/auth",
|
||||
@ -340,34 +463,80 @@
|
||||
"tools/reference",
|
||||
"transport",
|
||||
"util/cert",
|
||||
"util/connrotation",
|
||||
"util/flowcontrol",
|
||||
"util/homedir",
|
||||
"util/integer"
|
||||
"util/integer",
|
||||
]
|
||||
revision = "82eadfdc39007c2eb47e3ddeb7ed7d96365e409d"
|
||||
version = "kubernetes-1.10.0-beta.1"
|
||||
pruneopts = "NUT"
|
||||
revision = "6e4752048fde21176ab35eb54ec1117359830d8a"
|
||||
version = "kubernetes-1.13.3"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:9cc257b3c9ff6a0158c9c661ab6eebda1fe8a4a4453cd5c4044dc9a2ebfb992b"
|
||||
name = "k8s.io/klog"
|
||||
packages = ["."]
|
||||
pruneopts = "NUT"
|
||||
revision = "a5bc97fbc634d635061f3146511332c7e313a55a"
|
||||
version = "v0.1.0"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:cf54450b967dcae4f270dfa44395b01175e5358bbd79fe8a6073b13f220f1c2e"
|
||||
name = "k8s.io/kubernetes"
|
||||
packages = [
|
||||
"pkg/util/file",
|
||||
"pkg/util/io",
|
||||
"pkg/util/keymutex",
|
||||
"pkg/util/mount",
|
||||
"pkg/util/nsenter"
|
||||
"pkg/util/nsenter",
|
||||
]
|
||||
revision = "b1b29978270dc22fecc592ac55d903350454310a"
|
||||
version = "v1.11.1"
|
||||
pruneopts = "NUT"
|
||||
revision = "721bfa751924da8d1680787490c54b9179b1fed0"
|
||||
version = "v1.13.3"
|
||||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:381323c2fe2e890a3dd3b5d6dc6f2199068408cca89b24f6b7ca1c60f32644a5"
|
||||
name = "k8s.io/utils"
|
||||
packages = ["exec"]
|
||||
revision = "66066c83e385e385ccc3c964b44fd7dcd413d0ed"
|
||||
pruneopts = "NUT"
|
||||
revision = "8a16e7dd8fb6d97d1331b0c79a16722f934b00b1"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:8730e0150dfb2b7e173890c8b9868e7a273082ef8e39f4940e3506a481cf895c"
|
||||
name = "sigs.k8s.io/yaml"
|
||||
packages = ["."]
|
||||
pruneopts = "NUT"
|
||||
revision = "fd68e9863619f6ec2fdd8625fe1f02e7c877e480"
|
||||
version = "v1.1.0"
|
||||
|
||||
[solve-meta]
|
||||
analyzer-name = "dep"
|
||||
analyzer-version = 1
|
||||
inputs-digest = "6e997f1f66bb80e4e145da05f1872070947725cba04d3da2521bcdc0e33d7150"
|
||||
input-imports = [
|
||||
"github.com/container-storage-interface/spec/lib/go/csi",
|
||||
"github.com/golang/protobuf/ptypes",
|
||||
"github.com/golang/protobuf/ptypes/timestamp",
|
||||
"github.com/kubernetes-csi/csi-lib-utils/protosanitizer",
|
||||
"github.com/pborman/uuid",
|
||||
"github.com/pkg/errors",
|
||||
"golang.org/x/net/context",
|
||||
"google.golang.org/grpc",
|
||||
"google.golang.org/grpc/codes",
|
||||
"google.golang.org/grpc/status",
|
||||
"k8s.io/api/core/v1",
|
||||
"k8s.io/apimachinery/pkg/api/errors",
|
||||
"k8s.io/apimachinery/pkg/apis/meta/v1",
|
||||
"k8s.io/apimachinery/pkg/util/sets",
|
||||
"k8s.io/apimachinery/pkg/util/wait",
|
||||
"k8s.io/client-go/kubernetes",
|
||||
"k8s.io/client-go/rest",
|
||||
"k8s.io/client-go/tools/clientcmd",
|
||||
"k8s.io/klog",
|
||||
"k8s.io/kubernetes/pkg/util/keymutex",
|
||||
"k8s.io/kubernetes/pkg/util/mount",
|
||||
"k8s.io/kubernetes/pkg/util/nsenter",
|
||||
"k8s.io/utils/exec",
|
||||
]
|
||||
solver-name = "gps-cdcl"
|
||||
solver-version = 1
|
||||
|
18
Gopkg.toml
18
Gopkg.toml
@ -1,14 +1,6 @@
|
||||
[[constraint]]
|
||||
name = "github.com/container-storage-interface/spec"
|
||||
version = "~0.3.0"
|
||||
|
||||
[[constraint]]
|
||||
branch = "master"
|
||||
name = "github.com/kubernetes-csi/drivers"
|
||||
|
||||
[[constraint]]
|
||||
branch = "master"
|
||||
name = "github.com/golang/glog"
|
||||
version = "~1.0.0"
|
||||
|
||||
[[override]]
|
||||
revision = "5db89f0ca68677abc5eefce8f2a0a772c98ba52d"
|
||||
@ -19,15 +11,15 @@
|
||||
version = "1.10.0"
|
||||
|
||||
[[constraint]]
|
||||
version = "kubernetes-1.10.0-beta.1"
|
||||
version = "kubernetes-1.13.3"
|
||||
name = "k8s.io/apimachinery"
|
||||
|
||||
[[constraint]]
|
||||
name = "k8s.io/kubernetes"
|
||||
version = "v1.10.0-beta.1"
|
||||
version = "v1.13.3"
|
||||
|
||||
[[override]]
|
||||
version = "kubernetes-1.10.0-beta.1"
|
||||
version = "kubernetes-1.13.3"
|
||||
name = "k8s.io/api"
|
||||
|
||||
[[override]]
|
||||
@ -36,7 +28,7 @@
|
||||
|
||||
[[constraint]]
|
||||
name = "k8s.io/client-go"
|
||||
version = "kubernetes-1.10.0-beta.1"
|
||||
version = "kubernetes-1.13.3"
|
||||
|
||||
[prune]
|
||||
go-tests = true
|
||||
|
33
Makefile
33
Makefile
@ -14,46 +14,53 @@
|
||||
|
||||
.PHONY: all rbdplugin cephfsplugin
|
||||
|
||||
CONTAINER_CMD?=docker
|
||||
|
||||
RBD_IMAGE_NAME=$(if $(ENV_RBD_IMAGE_NAME),$(ENV_RBD_IMAGE_NAME),quay.io/cephcsi/rbdplugin)
|
||||
RBD_IMAGE_VERSION=$(if $(ENV_RBD_IMAGE_VERSION),$(ENV_RBD_IMAGE_VERSION),v0.3.0)
|
||||
RBD_IMAGE_VERSION=$(if $(ENV_RBD_IMAGE_VERSION),$(ENV_RBD_IMAGE_VERSION),v1.0.0)
|
||||
|
||||
CEPHFS_IMAGE_NAME=$(if $(ENV_CEPHFS_IMAGE_NAME),$(ENV_CEPHFS_IMAGE_NAME),quay.io/cephcsi/cephfsplugin)
|
||||
CEPHFS_IMAGE_VERSION=$(if $(ENV_CEPHFS_IMAGE_VERSION),$(ENV_CEPHFS_IMAGE_VERSION),v0.3.0)
|
||||
CEPHFS_IMAGE_VERSION=$(if $(ENV_CEPHFS_IMAGE_VERSION),$(ENV_CEPHFS_IMAGE_VERSION),v1.0.0)
|
||||
|
||||
$(info rbd image settings: $(RBD_IMAGE_NAME) version $(RBD_IMAGE_VERSION))
|
||||
$(info cephfs image settings: $(CEPHFS_IMAGE_NAME) version $(CEPHFS_IMAGE_VERSION))
|
||||
|
||||
all: rbdplugin cephfsplugin
|
||||
|
||||
test:
|
||||
go test github.com/ceph/ceph-csi/pkg/... -cover
|
||||
go vet github.com/ceph/ceph-csi/pkg/...
|
||||
test: go-test static-check
|
||||
|
||||
go-test:
|
||||
./scripts/test-go.sh
|
||||
|
||||
static-check:
|
||||
./scripts/lint-go.sh
|
||||
./scripts/lint-text.sh
|
||||
|
||||
rbdplugin:
|
||||
if [ ! -d ./vendor ]; then dep ensure -vendor-only; fi
|
||||
CGO_ENABLED=0 GOOS=linux go build -a -ldflags '-extldflags "-static"' -o _output/rbdplugin ./rbd
|
||||
CGO_ENABLED=0 GOOS=linux go build -a -ldflags '-extldflags "-static"' -o _output/rbdplugin ./cmd/rbd
|
||||
|
||||
image-rbdplugin: rbdplugin
|
||||
cp _output/rbdplugin deploy/rbd/docker
|
||||
docker build -t $(RBD_IMAGE_NAME):$(RBD_IMAGE_VERSION) deploy/rbd/docker
|
||||
$(CONTAINER_CMD) build -t $(RBD_IMAGE_NAME):$(RBD_IMAGE_VERSION) deploy/rbd/docker
|
||||
|
||||
cephfsplugin:
|
||||
if [ ! -d ./vendor ]; then dep ensure -vendor-only; fi
|
||||
CGO_ENABLED=0 GOOS=linux go build -a -ldflags '-extldflags "-static"' -o _output/cephfsplugin ./cephfs
|
||||
CGO_ENABLED=0 GOOS=linux go build -a -ldflags '-extldflags "-static"' -o _output/cephfsplugin ./cmd/cephfs
|
||||
|
||||
image-cephfsplugin: cephfsplugin
|
||||
cp _output/cephfsplugin deploy/cephfs/docker
|
||||
docker build -t $(CEPHFS_IMAGE_NAME):$(CEPHFS_IMAGE_VERSION) deploy/cephfs/docker
|
||||
$(CONTAINER_CMD) build -t $(CEPHFS_IMAGE_NAME):$(CEPHFS_IMAGE_VERSION) deploy/cephfs/docker
|
||||
|
||||
push-image-rbdplugin: image-rbdplugin
|
||||
docker push $(RBD_IMAGE_NAME):$(RBD_IMAGE_VERSION)
|
||||
$(CONTAINER_CMD) push $(RBD_IMAGE_NAME):$(RBD_IMAGE_VERSION)
|
||||
|
||||
push-image-cephfsplugin: image-cephfsplugin
|
||||
docker push $(CEPHFS_IMAGE_NAME):$(CEPHFS_IMAGE_VERSION)
|
||||
$(CONTAINER_CMD) push $(CEPHFS_IMAGE_NAME):$(CEPHFS_IMAGE_VERSION)
|
||||
|
||||
clean:
|
||||
go clean -r -x
|
||||
rm -f deploy/rbd/docker/rbdplugin
|
||||
rm -f deploy/cephfs/docker/cephfsplugin
|
||||
rm -f _output/rbdplugin
|
||||
rm -f _output/cephfsplugin
|
||||
rm -f _output/rbdplugin
|
||||
rm -f _output/cephfsplugin
|
||||
|
24
README.md
24
README.md
@ -1,18 +1,22 @@
|
||||
# Ceph CSI
|
||||
# Ceph CSI 1.0.0
|
||||
|
||||
# Supported platforms
|
||||
|
||||
For Kubernetes 1.12 and below, please use [0.3 images and deployments](https://github.com/ceph/ceph-csi/tree/master/deploy/rbd/kubernetes).
|
||||
|
||||
For Kubernetes 1.13 and above, please use [1.0 images and deployments](https://github.com/ceph/ceph-csi/tree/csi-v1.0/deploy/rbd/kubernetes).
|
||||
|
||||
[Container Storage Interface (CSI)](https://github.com/container-storage-interface/) driver, provisioner, and attacher for Ceph RBD and CephFS.
|
||||
[Container Storage Interface
|
||||
(CSI)](https://github.com/container-storage-interface/) driver, provisioner,
|
||||
and attacher for Ceph RBD and CephFS.
|
||||
|
||||
## Overview
|
||||
|
||||
Ceph CSI plugins implement an interface between CSI enabled Container Orchestrator (CO) and CEPH cluster. It allows dynamically provisioning CEPH volumes and attaching them to workloads. Current implementation of Ceph CSI plugins was tested in Kubernetes environment (requires Kubernetes 1.11+), but the code does not rely on any Kubernetes specific calls (WIP to make it k8s agnostic) and should be able to run with any CSI enabled CO.
|
||||
Ceph CSI plugins implement an interface between CSI enabled Container
|
||||
Orchestrator (CO) and CEPH cluster.
|
||||
It allows dynamically provisioning CEPH volumes and attaching them to
|
||||
workloads.
|
||||
Current implementation of Ceph CSI plugins was tested in Kubernetes
|
||||
environment (requires Kubernetes 1.13+), but the code does not rely on
|
||||
any Kubernetes specific calls (WIP to make it k8s agnostic) and
|
||||
should be able to run with any CSI enabled CO.
|
||||
|
||||
For details about configuration and deployment of RBD and CephFS CSI plugins, see documentation in `docs/`.
|
||||
For details about configuration and deployment of RBD and
|
||||
CephFS CSI plugins, see documentation in `docs/`.
|
||||
|
||||
For example usage of RBD and CephFS CSI plugins, see examples in `examples/`.
|
||||
|
||||
|
@ -19,58 +19,45 @@ package main
|
||||
import (
|
||||
"flag"
|
||||
"os"
|
||||
"path"
|
||||
|
||||
"github.com/ceph/ceph-csi/pkg/cephfs"
|
||||
"github.com/ceph/ceph-csi/pkg/util"
|
||||
"github.com/golang/glog"
|
||||
"k8s.io/klog"
|
||||
)
|
||||
|
||||
func init() {
|
||||
flag.Set("logtostderr", "true")
|
||||
}
|
||||
|
||||
var (
|
||||
endpoint = flag.String("endpoint", "unix://tmp/csi.sock", "CSI endpoint")
|
||||
driverName = flag.String("drivername", "cephfs.csi.ceph.com", "name of the driver")
|
||||
nodeId = flag.String("nodeid", "", "node id")
|
||||
nodeID = flag.String("nodeid", "", "node id")
|
||||
volumeMounter = flag.String("volumemounter", "", "default volume mounter (possible options are 'kernel', 'fuse')")
|
||||
metadataStorage = flag.String("metadatastorage", "node", "metadata persistence method [node|k8s_configmap]")
|
||||
metadataStorage = flag.String("metadatastorage", "", "metadata persistence method [node|k8s_configmap]")
|
||||
mountCacheDir = flag.String("mountcachedir", "", "mount info cache save dir")
|
||||
)
|
||||
|
||||
func main() {
|
||||
func init() {
|
||||
klog.InitFlags(nil)
|
||||
if err := flag.Set("logtostderr", "true"); err != nil {
|
||||
klog.Exitf("failed to set logtostderr flag: %v", err)
|
||||
}
|
||||
flag.Parse()
|
||||
}
|
||||
|
||||
func main() {
|
||||
|
||||
err := util.ValidateDriverName(*driverName)
|
||||
if err != nil {
|
||||
glog.Errorf("failed to validate driver name: %v", err)
|
||||
os.Exit(1)
|
||||
klog.Fatalln(err)
|
||||
}
|
||||
//update plugin name
|
||||
cephfs.PluginFolder = cephfs.PluginFolder + *driverName
|
||||
|
||||
if err := createPersistentStorage(path.Join(cephfs.PluginFolder, "controller")); err != nil {
|
||||
glog.Errorf("failed to create persistent storage for controller: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if err := createPersistentStorage(path.Join(cephfs.PluginFolder, "node")); err != nil {
|
||||
glog.Errorf("failed to create persistent storage for node: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
cp, err := util.NewCachePersister(*metadataStorage, *driverName)
|
||||
cp, err := util.CreatePersistanceStorage(cephfs.PluginFolder, *metadataStorage, *driverName)
|
||||
if err != nil {
|
||||
glog.Errorf("failed to define cache persistence method: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
driver := cephfs.NewCephFSDriver()
|
||||
driver.Run(*driverName, *nodeId, *endpoint, *volumeMounter, cp)
|
||||
driver := cephfs.NewDriver()
|
||||
driver.Run(*driverName, *nodeID, *endpoint, *volumeMounter, *mountCacheDir, cp)
|
||||
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
func createPersistentStorage(persistentStoragePath string) error {
|
||||
return os.MkdirAll(persistentStoragePath, os.FileMode(0755))
|
||||
}
|
@ -19,63 +19,46 @@ package main
|
||||
import (
|
||||
"flag"
|
||||
"os"
|
||||
"path"
|
||||
|
||||
"github.com/ceph/ceph-csi/pkg/rbd"
|
||||
"github.com/ceph/ceph-csi/pkg/util"
|
||||
"github.com/golang/glog"
|
||||
"k8s.io/klog"
|
||||
)
|
||||
|
||||
func init() {
|
||||
flag.Set("logtostderr", "true")
|
||||
}
|
||||
|
||||
var (
|
||||
endpoint = flag.String("endpoint", "unix://tmp/csi.sock", "CSI endpoint")
|
||||
driverName = flag.String("drivername", "rbd.csi.ceph.com", "name of the driver")
|
||||
nodeID = flag.String("nodeid", "", "node id")
|
||||
containerized = flag.Bool("containerized", true, "whether run as containerized")
|
||||
metadataStorage = flag.String("metadatastorage", "node", "metadata persistence method [node|k8s_configmap]")
|
||||
metadataStorage = flag.String("metadatastorage", "", "metadata persistence method [node|k8s_configmap]")
|
||||
configRoot = flag.String("configroot", "/etc/csi-config", "directory in which CSI specific Ceph"+
|
||||
" cluster configurations are present, OR the value \"k8s_objects\" if present as kubernetes secrets")
|
||||
)
|
||||
|
||||
func main() {
|
||||
func init() {
|
||||
klog.InitFlags(nil)
|
||||
if err := flag.Set("logtostderr", "true"); err != nil {
|
||||
klog.Exitf("failed to set logtostderr flag: %v", err)
|
||||
}
|
||||
flag.Parse()
|
||||
}
|
||||
|
||||
func main() {
|
||||
|
||||
err := util.ValidateDriverName(*driverName)
|
||||
if err != nil {
|
||||
glog.Errorf("failed to validate driver name: %v", err)
|
||||
os.Exit(1)
|
||||
klog.Fatalln(err)
|
||||
}
|
||||
//update plugin name
|
||||
rbd.PluginFolder = rbd.PluginFolder + *driverName
|
||||
|
||||
if err := createPersistentStorage(path.Join(rbd.PluginFolder, "controller")); err != nil {
|
||||
glog.Errorf("failed to create persistent storage for controller %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := createPersistentStorage(path.Join(rbd.PluginFolder, "node")); err != nil {
|
||||
glog.Errorf("failed to create persistent storage for node %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
cp, err := util.NewCachePersister(*metadataStorage, *driverName)
|
||||
cp, err := util.CreatePersistanceStorage(rbd.PluginFolder, *metadataStorage, *driverName)
|
||||
if err != nil {
|
||||
glog.Errorf("failed to define cache persistence method: %v", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
driver := rbd.GetRBDDriver()
|
||||
driver.Run(*driverName, *nodeID, *endpoint, *containerized, cp)
|
||||
driver := rbd.NewDriver()
|
||||
driver.Run(*driverName, *nodeID, *endpoint, *configRoot, *containerized, cp)
|
||||
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
func createPersistentStorage(persistentStoragePath string) error {
|
||||
if _, err := os.Stat(persistentStoragePath); os.IsNotExist(err) {
|
||||
if err := os.MkdirAll(persistentStoragePath, os.FileMode(0755)); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
}
|
||||
return nil
|
||||
}
|
58
deploy.sh
58
deploy.sh
@ -1,6 +1,58 @@
|
||||
#!/bin/bash
|
||||
|
||||
if [ "${TRAVIS_BRANCH}" == "master" ] && [ "${TRAVIS_PULL_REQUEST}" == "false" ]; then
|
||||
docker login -u "${QUAY_IO_USERNAME}" -p "${QUAY_IO_PASSWORD}" quay.io
|
||||
make push-image-rbdplugin push-image-cephfsplugin
|
||||
push_helm_chats() {
|
||||
PACKAGE=$1
|
||||
CHANGED=0
|
||||
VERSION=$(grep 'version:' deploy/"$PACKAGE"/helm/Chart.yaml | awk '{print $2}')
|
||||
|
||||
if [ ! -f "tmp/csi-charts/docs/$PACKAGE/ceph-csi-$PACKAGE-$VERSION.tgz" ]; then
|
||||
CHANGED=1
|
||||
ln -s helm deploy/"$PACKAGE"/ceph-csi-"$PACKAGE"
|
||||
mkdir -p tmp/csi-charts/docs/"$PACKAGE"
|
||||
pushd tmp/csi-charts/docs/"$PACKAGE" >/dev/null
|
||||
helm init --client-only
|
||||
helm package ../../../../deploy/"$PACKAGE"/ceph-csi-"$PACKAGE"
|
||||
popd >/dev/null
|
||||
fi
|
||||
|
||||
if [ $CHANGED -eq 1 ]; then
|
||||
pushd tmp/csi-charts/docs >/dev/null
|
||||
helm repo index .
|
||||
git add --all :/ && git commit -m "Update repo"
|
||||
git push https://"$GITHUB_TOKEN"@github.com/ceph/csi-charts
|
||||
popd >/dev/null
|
||||
fi
|
||||
}
|
||||
|
||||
if [ "${TRAVIS_BRANCH}" == 'csi-v0.3' ]; then
|
||||
export RBD_IMAGE_VERSION='v0.3.0'
|
||||
export CEPHFS_IMAGE_VERSION='v0.3.0'
|
||||
elif [ "${TRAVIS_BRANCH}" == 'csi-v1.0' ]; then
|
||||
export RBD_IMAGE_VERSION='v1.0.0'
|
||||
export CEPHFS_IMAGE_VERSION='v1.0.0'
|
||||
else
|
||||
echo "!!! Branch ${TRAVIS_BRANCH} is not a deployable branch; exiting"
|
||||
exit 0 # Exiting 0 so that this isn't marked as failing
|
||||
fi
|
||||
|
||||
if [ "${TRAVIS_PULL_REQUEST}" == "false" ]; then
|
||||
"${CONTAINER_CMD:-docker}" login -u "${QUAY_IO_USERNAME}" -p "${QUAY_IO_PASSWORD}" quay.io
|
||||
make push-image-rbdplugin push-image-cephfsplugin
|
||||
|
||||
set -xe
|
||||
|
||||
mkdir -p tmp
|
||||
pushd tmp >/dev/null
|
||||
|
||||
curl https://raw.githubusercontent.com/helm/helm/master/scripts/get >get_helm.sh
|
||||
chmod 700 get_helm.sh
|
||||
./get_helm.sh
|
||||
|
||||
git clone https://github.com/ceph/csi-charts
|
||||
|
||||
mkdir -p csi-charts/docs
|
||||
popd >/dev/null
|
||||
|
||||
push_helm_chats rbd
|
||||
push_helm_chats cephfs
|
||||
fi
|
||||
|
21
deploy/cephfs/helm/.helmignore
Normal file
21
deploy/cephfs/helm/.helmignore
Normal file
@ -0,0 +1,21 @@
|
||||
# Patterns to ignore when building packages.
|
||||
# This supports shell glob matching, relative path matching, and
|
||||
# negation (prefixed with !). Only one pattern per line.
|
||||
.DS_Store
|
||||
# Common VCS dirs
|
||||
.git/
|
||||
.gitignore
|
||||
.bzr/
|
||||
.bzrignore
|
||||
.hg/
|
||||
.hgignore
|
||||
.svn/
|
||||
# Common backup files
|
||||
*.swp
|
||||
*.bak
|
||||
*.tmp
|
||||
*~
|
||||
# Various IDEs
|
||||
.project
|
||||
.idea/
|
||||
*.tmproj
|
14
deploy/cephfs/helm/Chart.yaml
Normal file
14
deploy/cephfs/helm/Chart.yaml
Normal file
@ -0,0 +1,14 @@
|
||||
---
|
||||
apiVersion: v1
|
||||
appVersion: "1.0.0"
|
||||
description: "Container Storage Interface (CSI) driver,
|
||||
provisioner, and attacher for Ceph cephfs"
|
||||
name: ceph-csi-cephfs
|
||||
version: 0.5.1
|
||||
keywords:
|
||||
- ceph
|
||||
- cephfs
|
||||
- ceph-csi
|
||||
home: https://github.com/ceph/ceph-csi
|
||||
sources:
|
||||
- https://github.com/ceph/ceph-csi/tree/csi-v1.0/deploy/cephfs/helm
|
29
deploy/cephfs/helm/README.md
Normal file
29
deploy/cephfs/helm/README.md
Normal file
@ -0,0 +1,29 @@
|
||||
# ceph-csi-cephfs
|
||||
|
||||
The ceph-csi-cephfs chart adds cephfs volume support to your cluster.
|
||||
|
||||
## Install Chart
|
||||
|
||||
To install the Chart into your Kubernetes cluster
|
||||
|
||||
```bash
|
||||
helm install --namespace "ceph-csi-cephfs" --name "ceph-csi-cephfs" ceph-csi/ceph-csi-cephfs
|
||||
```
|
||||
|
||||
After installation succeeds, you can get a status of Chart
|
||||
|
||||
```bash
|
||||
helm status "ceph-csi-cephfs"
|
||||
```
|
||||
|
||||
If you want to delete your Chart, use this command
|
||||
|
||||
```bash
|
||||
helm delete --purge "ceph-csi-cephfs"
|
||||
```
|
||||
|
||||
If you want to delete the namespace, use this command
|
||||
|
||||
```bash
|
||||
kubectl delete namespace ceph-csi-rbd
|
||||
```
|
2
deploy/cephfs/helm/templates/NOTES.txt
Normal file
2
deploy/cephfs/helm/templates/NOTES.txt
Normal file
@ -0,0 +1,2 @@
|
||||
Examples on how to configure a storage class and start using the driver are here:
|
||||
https://github.com/ceph/ceph-csi/tree/csi-v1.0/examples/cephfs
|
119
deploy/cephfs/helm/templates/_helpers.tpl
Normal file
119
deploy/cephfs/helm/templates/_helpers.tpl
Normal file
@ -0,0 +1,119 @@
|
||||
{{/* vim: set filetype=mustache: */}}
|
||||
{{/*
|
||||
Expand the name of the chart.
|
||||
*/}}
|
||||
{{- define "ceph-csi-cephfs.name" -}}
|
||||
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Create a default fully qualified app name.
|
||||
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
|
||||
If release name contains chart name it will be used as a full name.
|
||||
*/}}
|
||||
{{- define "ceph-csi-cephfs.fullname" -}}
|
||||
{{- if .Values.fullnameOverride -}}
|
||||
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
|
||||
{{- else -}}
|
||||
{{- $name := default .Chart.Name .Values.nameOverride -}}
|
||||
{{- if contains $name .Release.Name -}}
|
||||
{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
|
||||
{{- else -}}
|
||||
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Create a default fully qualified app name.
|
||||
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
|
||||
If release name contains chart name it will be used as a full name.
|
||||
*/}}
|
||||
{{- define "ceph-csi-cephfs.attacher.fullname" -}}
|
||||
{{- if .Values.attacher.fullnameOverride -}}
|
||||
{{- .Values.attacher.fullnameOverride | trunc 63 | trimSuffix "-" -}}
|
||||
{{- else -}}
|
||||
{{- $name := default .Chart.Name .Values.nameOverride -}}
|
||||
{{- if contains $name .Release.Name -}}
|
||||
{{- printf "%s-%s" .Release.Name .Values.attacher.name | trunc 63 | trimSuffix "-" -}}
|
||||
{{- else -}}
|
||||
{{- printf "%s-%s-%s" .Release.Name $name .Values.attacher.name | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Create a default fully qualified app name.
|
||||
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
|
||||
If release name contains chart name it will be used as a full name.
|
||||
*/}}
|
||||
{{- define "ceph-csi-cephfs.nodeplugin.fullname" -}}
|
||||
{{- if .Values.nodeplugin.fullnameOverride -}}
|
||||
{{- .Values.nodeplugin.fullnameOverride | trunc 63 | trimSuffix "-" -}}
|
||||
{{- else -}}
|
||||
{{- $name := default .Chart.Name .Values.nameOverride -}}
|
||||
{{- if contains $name .Release.Name -}}
|
||||
{{- printf "%s-%s" .Release.Name .Values.nodeplugin.name | trunc 63 | trimSuffix "-" -}}
|
||||
{{- else -}}
|
||||
{{- printf "%s-%s-%s" .Release.Name $name .Values.nodeplugin.name | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Create a default fully qualified app name.
|
||||
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
|
||||
If release name contains chart name it will be used as a full name.
|
||||
*/}}
|
||||
{{- define "ceph-csi-cephfs.provisioner.fullname" -}}
|
||||
{{- if .Values.provisioner.fullnameOverride -}}
|
||||
{{- .Values.provisioner.fullnameOverride | trunc 63 | trimSuffix "-" -}}
|
||||
{{- else -}}
|
||||
{{- $name := default .Chart.Name .Values.nameOverride -}}
|
||||
{{- if contains $name .Release.Name -}}
|
||||
{{- printf "%s-%s" .Release.Name .Values.provisioner.name | trunc 63 | trimSuffix "-" -}}
|
||||
{{- else -}}
|
||||
{{- printf "%s-%s-%s" .Release.Name $name .Values.provisioner.name | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Create chart name and version as used by the chart label.
|
||||
*/}}
|
||||
{{- define "ceph-csi-cephfs.chart" -}}
|
||||
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Create the name of the service account to use
|
||||
*/}}
|
||||
{{- define "ceph-csi-cephfs.serviceAccountName.attacher" -}}
|
||||
{{- if .Values.serviceAccounts.attacher.create -}}
|
||||
{{ default (include "ceph-csi-cephfs.attacher.fullname" .) .Values.serviceAccounts.attacher.name }}
|
||||
{{- else -}}
|
||||
{{ default "default" .Values.serviceAccounts.attacher.name }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Create the name of the service account to use
|
||||
*/}}
|
||||
{{- define "ceph-csi-cephfs.serviceAccountName.nodeplugin" -}}
|
||||
{{- if .Values.serviceAccounts.nodeplugin.create -}}
|
||||
{{ default (include "ceph-csi-cephfs.nodeplugin.fullname" .) .Values.serviceAccounts.nodeplugin.name }}
|
||||
{{- else -}}
|
||||
{{ default "default" .Values.serviceAccounts.nodeplugin.name }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Create the name of the service account to use
|
||||
*/}}
|
||||
{{- define "ceph-csi-cephfs.serviceAccountName.provisioner" -}}
|
||||
{{- if .Values.serviceAccounts.provisioner.create -}}
|
||||
{{ default (include "ceph-csi-cephfs.provisioner.fullname" .) .Values.serviceAccounts.provisioner.name }}
|
||||
{{- else -}}
|
||||
{{ default "default" .Values.serviceAccounts.provisioner.name }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
28
deploy/cephfs/helm/templates/attacher-clusterrole.yaml
Normal file
28
deploy/cephfs/helm/templates/attacher-clusterrole.yaml
Normal file
@ -0,0 +1,28 @@
|
||||
{{- if .Values.rbac.create -}}
|
||||
kind: ClusterRole
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: {{ include "ceph-csi-cephfs.attacher.fullname" . }}
|
||||
labels:
|
||||
app: {{ include "ceph-csi-cephfs.name" . }}
|
||||
chart: {{ include "ceph-csi-cephfs.chart" . }}
|
||||
component: {{ .Values.attacher.name }}
|
||||
release: {{ .Release.Name }}
|
||||
heritage: {{ .Release.Service }}
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["events"]
|
||||
verbs: ["get", "list", "watch", "update"]
|
||||
- apiGroups: [""]
|
||||
resources: ["persistentvolumes"]
|
||||
verbs: ["get", "list", "watch", "update"]
|
||||
- apiGroups: [""]
|
||||
resources: ["nodes"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: ["storage.k8s.io"]
|
||||
resources: ["volumeattachments"]
|
||||
verbs: ["get", "list", "watch", "update"]
|
||||
- apiGroups: ["csi.storage.k8s.io"]
|
||||
resources: ["csinodeinfos"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
{{- end -}}
|
@ -0,0 +1,20 @@
|
||||
{{- if .Values.rbac.create -}}
|
||||
kind: ClusterRoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: {{ include "ceph-csi-cephfs.attacher.fullname" . }}
|
||||
labels:
|
||||
app: {{ include "ceph-csi-cephfs.name" . }}
|
||||
chart: {{ include "ceph-csi-cephfs.chart" . }}
|
||||
component: {{ .Values.attacher.name }}
|
||||
release: {{ .Release.Name }}
|
||||
heritage: {{ .Release.Service }}
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "ceph-csi-cephfs.serviceAccountName.attacher" . }}
|
||||
namespace: {{ .Release.Namespace }}
|
||||
roleRef:
|
||||
kind: ClusterRole
|
||||
name: {{ include "ceph-csi-cephfs.attacher.fullname" . }}
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
{{- end -}}
|
18
deploy/cephfs/helm/templates/attacher-service.yaml
Normal file
18
deploy/cephfs/helm/templates/attacher-service.yaml
Normal file
@ -0,0 +1,18 @@
|
||||
kind: Service
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: {{ include "ceph-csi-cephfs.attacher.fullname" . }}
|
||||
labels:
|
||||
app: {{ include "ceph-csi-cephfs.name" . }}
|
||||
chart: {{ include "ceph-csi-cephfs.chart" . }}
|
||||
component: {{ .Values.attacher.name }}
|
||||
release: {{ .Release.Name }}
|
||||
heritage: {{ .Release.Service }}
|
||||
spec:
|
||||
selector:
|
||||
app: {{ include "ceph-csi-cephfs.name" . }}
|
||||
component: {{ .Values.attacher.name }}
|
||||
release: {{ .Release.Name }}
|
||||
ports:
|
||||
- name: dummy
|
||||
port: 12345
|
12
deploy/cephfs/helm/templates/attacher-serviceaccount.yaml
Normal file
12
deploy/cephfs/helm/templates/attacher-serviceaccount.yaml
Normal file
@ -0,0 +1,12 @@
|
||||
{{- if .Values.serviceAccounts.attacher.create -}}
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: {{ include "ceph-csi-cephfs.serviceAccountName.attacher" . }}
|
||||
labels:
|
||||
app: {{ include "ceph-csi-cephfs.name" . }}
|
||||
chart: {{ include "ceph-csi-cephfs.chart" . }}
|
||||
component: {{ .Values.attacher.name }}
|
||||
release: {{ .Release.Name }}
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- end -}}
|
60
deploy/cephfs/helm/templates/attacher-statefulset.yaml
Normal file
60
deploy/cephfs/helm/templates/attacher-statefulset.yaml
Normal file
@ -0,0 +1,60 @@
|
||||
kind: StatefulSet
|
||||
apiVersion: apps/v1beta1
|
||||
metadata:
|
||||
name: {{ include "ceph-csi-cephfs.attacher.fullname" . }}
|
||||
labels:
|
||||
app: {{ include "ceph-csi-cephfs.name" . }}
|
||||
chart: {{ include "ceph-csi-cephfs.chart" . }}
|
||||
component: {{ .Values.attacher.name }}
|
||||
release: {{ .Release.Name }}
|
||||
heritage: {{ .Release.Service }}
|
||||
spec:
|
||||
serviceName: {{ include "ceph-csi-cephfs.attacher.fullname" . }}
|
||||
replicas: {{ .Values.attacher.replicas }}
|
||||
selector:
|
||||
matchLabels:
|
||||
app: {{ include "ceph-csi-cephfs.name" . }}
|
||||
component: {{ .Values.attacher.name }}
|
||||
release: {{ .Release.Name }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: {{ include "ceph-csi-cephfs.name" . }}
|
||||
chart: {{ include "ceph-csi-cephfs.chart" . }}
|
||||
component: {{ .Values.attacher.name }}
|
||||
release: {{ .Release.Name }}
|
||||
heritage: {{ .Release.Service }}
|
||||
spec:
|
||||
serviceAccountName: {{ include "ceph-csi-cephfs.serviceAccountName.attacher" . }}
|
||||
containers:
|
||||
- name: csi-cephfsplugin-attacher
|
||||
image: "{{ .Values.attacher.image.repository }}:{{ .Values.attacher.image.tag }}"
|
||||
args:
|
||||
- "--v=5"
|
||||
- "--csi-address=$(ADDRESS)"
|
||||
env:
|
||||
- name: ADDRESS
|
||||
value: "{{ .Values.socketDir }}/{{ .Values.socketFile }}"
|
||||
imagePullPolicy: {{ .Values.attacher.image.pullPolicy }}
|
||||
volumeMounts:
|
||||
- name: socket-dir
|
||||
mountPath: {{ .Values.socketDir }}
|
||||
resources:
|
||||
{{ toYaml .Values.attacher.resources | indent 12 }}
|
||||
volumes:
|
||||
- name: socket-dir
|
||||
hostPath:
|
||||
path: {{ .Values.socketDir }}
|
||||
type: DirectoryOrCreate
|
||||
{{- if .Values.attacher.affinity -}}
|
||||
affinity:
|
||||
{{ toYaml .Values.attacher.affinity . | indent 8 }}
|
||||
{{- end -}}
|
||||
{{- if .Values.attacher.nodeSelector -}}
|
||||
nodeSelector:
|
||||
{{ toYaml .Values.attacher.nodeSelector | indent 8 }}
|
||||
{{- end -}}
|
||||
{{- if .Values.attacher.tolerations -}}
|
||||
tolerations:
|
||||
{{ toYaml .Values.attacher.tolerations | indent 8 }}
|
||||
{{- end -}}
|
31
deploy/cephfs/helm/templates/nodeplugin-clusterrole.yaml
Normal file
31
deploy/cephfs/helm/templates/nodeplugin-clusterrole.yaml
Normal file
@ -0,0 +1,31 @@
|
||||
{{- if .Values.rbac.create -}}
|
||||
kind: ClusterRole
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }}
|
||||
labels:
|
||||
app: {{ include "ceph-csi-cephfs.name" . }}
|
||||
chart: {{ include "ceph-csi-cephfs.chart" . }}
|
||||
component: {{ .Values.nodeplugin.name }}
|
||||
release: {{ .Release.Name }}
|
||||
heritage: {{ .Release.Service }}
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["configmaps"]
|
||||
verbs: ["get", "list"]
|
||||
- apiGroups: [""]
|
||||
resources: ["nodes"]
|
||||
verbs: ["get", "list", "update"]
|
||||
- apiGroups: [""]
|
||||
resources: ["namespaces"]
|
||||
verbs: ["get", "list"]
|
||||
- apiGroups: [""]
|
||||
resources: ["persistentvolumes"]
|
||||
verbs: ["get", "list", "watch", "update"]
|
||||
- apiGroups: ["storage.k8s.io"]
|
||||
resources: ["volumeattachments"]
|
||||
verbs: ["get", "list", "watch", "update"]
|
||||
- apiGroups: [""]
|
||||
resources: ["configmaps"]
|
||||
verbs: ["get", "list"]
|
||||
{{- end -}}
|
@ -0,0 +1,20 @@
|
||||
{{- if .Values.rbac.create -}}
|
||||
kind: ClusterRoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }}
|
||||
labels:
|
||||
app: {{ include "ceph-csi-cephfs.name" . }}
|
||||
chart: {{ include "ceph-csi-cephfs.chart" . }}
|
||||
component: {{ .Values.nodeplugin.name }}
|
||||
release: {{ .Release.Name }}
|
||||
heritage: {{ .Release.Service }}
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "ceph-csi-cephfs.serviceAccountName.nodeplugin" . }}
|
||||
namespace: {{ .Release.Namespace }}
|
||||
roleRef:
|
||||
kind: ClusterRole
|
||||
name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }}
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
{{- end -}}
|
150
deploy/cephfs/helm/templates/nodeplugin-daemonset.yaml
Normal file
150
deploy/cephfs/helm/templates/nodeplugin-daemonset.yaml
Normal file
@ -0,0 +1,150 @@
|
||||
kind: DaemonSet
|
||||
apiVersion: apps/v1beta2
|
||||
metadata:
|
||||
name: {{ include "ceph-csi-cephfs.nodeplugin.fullname" . }}
|
||||
labels:
|
||||
app: {{ include "ceph-csi-cephfs.name" . }}
|
||||
chart: {{ include "ceph-csi-cephfs.chart" . }}
|
||||
component: {{ .Values.nodeplugin.name }}
|
||||
release: {{ .Release.Name }}
|
||||
heritage: {{ .Release.Service }}
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app: {{ include "ceph-csi-cephfs.name" . }}
|
||||
component: {{ .Values.nodeplugin.name }}
|
||||
release: {{ .Release.Name }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: {{ include "ceph-csi-cephfs.name" . }}
|
||||
chart: {{ include "ceph-csi-cephfs.chart" . }}
|
||||
component: {{ .Values.nodeplugin.name }}
|
||||
release: {{ .Release.Name }}
|
||||
heritage: {{ .Release.Service }}
|
||||
spec:
|
||||
serviceAccountName: {{ include "ceph-csi-cephfs.serviceAccountName.nodeplugin" . }}
|
||||
hostNetwork: true
|
||||
hostPID: true
|
||||
# to use e.g. Rook orchestrated cluster, and mons' FQDN is
|
||||
# resolved through k8s service, set dns policy to cluster first
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
containers:
|
||||
- name: driver-registrar
|
||||
image: "{{ .Values.nodeplugin.registrar.image.repository }}:{{ .Values.nodeplugin.registrar.image.tag }}"
|
||||
args:
|
||||
- "--v=5"
|
||||
- "--csi-address=/csi/{{ .Values.socketFile }}"
|
||||
- "--kubelet-registration-path={{ .Values.socketDir }}/{{ .Values.socketFile }}"
|
||||
lifecycle:
|
||||
preStop:
|
||||
exec:
|
||||
command: [
|
||||
"/bin/sh", "-c",
|
||||
'rm -rf /registration/{{ .Values.driverName }}
|
||||
/registration/{{ .Values.driverName }}-reg.sock'
|
||||
]
|
||||
env:
|
||||
- name: KUBE_NODE_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
imagePullPolicy: {{ .Values.nodeplugin.registrar.image.imagePullPolicy }}
|
||||
volumeMounts:
|
||||
- name: plugin-dir
|
||||
mountPath: /csi
|
||||
- name: registration-dir
|
||||
mountPath: /registration
|
||||
resources:
|
||||
{{ toYaml .Values.nodeplugin.registrar.resources | indent 12 }}
|
||||
- name: csi-cephfsplugin
|
||||
securityContext:
|
||||
privileged: true
|
||||
capabilities:
|
||||
add: ["SYS_ADMIN"]
|
||||
allowPrivilegeEscalation: true
|
||||
image: "{{ .Values.nodeplugin.plugin.image.repository }}:{{ .Values.nodeplugin.plugin.image.tag }}"
|
||||
args :
|
||||
- "--nodeid=$(NODE_ID)"
|
||||
- "--endpoint=$(CSI_ENDPOINT)"
|
||||
- "--v=5"
|
||||
- "--drivername=$(DRIVER_NAME)"
|
||||
- "--metadatastorage=k8s_configmap"
|
||||
- "--mountcachedir=/mount-cache-dir"
|
||||
env:
|
||||
- name: HOST_ROOTFS
|
||||
value: "/rootfs"
|
||||
- name: DRIVER_NAME
|
||||
value: {{ .Values.driverName }}
|
||||
- name: NODE_ID
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
- name: CSI_ENDPOINT
|
||||
value: "unix:/{{ .Values.socketDir }}/{{ .Values.socketFile }}"
|
||||
imagePullPolicy: {{ .Values.nodeplugin.plugin.image.imagePullPolicy }}
|
||||
volumeMounts:
|
||||
- name: mount-cache-dir
|
||||
mountPath: /mount-cache-dir
|
||||
- name: plugin-dir
|
||||
mountPath: {{ .Values.socketDir }}
|
||||
- name: pods-mount-dir
|
||||
mountPath: /var/lib/kubelet/pods
|
||||
mountPropagation: "Bidirectional"
|
||||
- name: plugin-mount-dir
|
||||
mountPath: {{ .Values.volumeDevicesDir }}
|
||||
mountPropagation: "Bidirectional"
|
||||
- mountPath: /dev
|
||||
name: host-dev
|
||||
- mountPath: /rootfs
|
||||
name: host-rootfs
|
||||
- mountPath: /sys
|
||||
name: host-sys
|
||||
- mountPath: /lib/modules
|
||||
name: lib-modules
|
||||
readOnly: true
|
||||
resources:
|
||||
{{ toYaml .Values.nodeplugin.plugin.resources | indent 12 }}
|
||||
volumes:
|
||||
- name: mount-cache-dir
|
||||
emptyDir: {}
|
||||
- name: plugin-dir
|
||||
hostPath:
|
||||
path: {{ .Values.socketDir }}
|
||||
type: DirectoryOrCreate
|
||||
- name: plugin-mount-dir
|
||||
hostPath:
|
||||
path: {{ .Values.volumeDevicesDir }}
|
||||
type: DirectoryOrCreate
|
||||
- name: registration-dir
|
||||
hostPath:
|
||||
path: {{ .Values.registrationDir }}
|
||||
type: Directory
|
||||
- name: pods-mount-dir
|
||||
hostPath:
|
||||
path: /var/lib/kubelet/pods
|
||||
type: Directory
|
||||
- name: host-dev
|
||||
hostPath:
|
||||
path: /dev
|
||||
- name: host-rootfs
|
||||
hostPath:
|
||||
path: /
|
||||
- name: host-sys
|
||||
hostPath:
|
||||
path: /sys
|
||||
- name: lib-modules
|
||||
hostPath:
|
||||
path: /lib/modules
|
||||
{{- if .Values.nodeplugin.affinity -}}
|
||||
affinity:
|
||||
{{ toYaml .Values.nodeplugin.affinity . | indent 8 }}
|
||||
{{- end -}}
|
||||
{{- if .Values.nodeplugin.nodeSelector -}}
|
||||
nodeSelector:
|
||||
{{ toYaml .Values.nodeplugin.nodeSelector | indent 8 }}
|
||||
{{- end -}}
|
||||
{{- if .Values.nodeplugin.tolerations -}}
|
||||
tolerations:
|
||||
{{ toYaml .Values.nodeplugin.tolerations | indent 8 }}
|
||||
{{- end -}}
|
12
deploy/cephfs/helm/templates/nodeplugin-serviceaccount.yaml
Normal file
12
deploy/cephfs/helm/templates/nodeplugin-serviceaccount.yaml
Normal file
@ -0,0 +1,12 @@
|
||||
{{- if .Values.serviceAccounts.nodeplugin.create -}}
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: {{ include "ceph-csi-cephfs.serviceAccountName.nodeplugin" . }}
|
||||
labels:
|
||||
app: {{ include "ceph-csi-cephfs.name" . }}
|
||||
chart: {{ include "ceph-csi-cephfs.chart" . }}
|
||||
component: {{ .Values.nodeplugin.name }}
|
||||
release: {{ .Release.Name }}
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- end -}}
|
37
deploy/cephfs/helm/templates/provisioner-clusterrole.yaml
Normal file
37
deploy/cephfs/helm/templates/provisioner-clusterrole.yaml
Normal file
@ -0,0 +1,37 @@
|
||||
{{- if .Values.rbac.create -}}
|
||||
kind: ClusterRole
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }}
|
||||
labels:
|
||||
app: {{ include "ceph-csi-cephfs.name" . }}
|
||||
chart: {{ include "ceph-csi-cephfs.chart" . }}
|
||||
component: {{ .Values.provisioner.name }}
|
||||
release: {{ .Release.Name }}
|
||||
heritage: {{ .Release.Service }}
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["nodes"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["secrets"]
|
||||
verbs: ["get", "list"]
|
||||
- apiGroups: [""]
|
||||
resources: ["persistentvolumes"]
|
||||
verbs: ["get", "list", "watch", "create", "delete"]
|
||||
- apiGroups: [""]
|
||||
resources: ["persistentvolumeclaims"]
|
||||
verbs: ["get", "list", "watch", "update"]
|
||||
- apiGroups: ["storage.k8s.io"]
|
||||
resources: ["storageclasses"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["events"]
|
||||
verbs: ["list", "watch", "create", "update", "patch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["configmaps"]
|
||||
verbs: ["get", "list", "create", "delete"]
|
||||
- apiGroups: ["csi.storage.k8s.io"]
|
||||
resources: ["csinodeinfos"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
{{- end -}}
|
@ -0,0 +1,20 @@
|
||||
{{- if .Values.rbac.create -}}
|
||||
kind: ClusterRoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }}
|
||||
labels:
|
||||
app: {{ include "ceph-csi-cephfs.name" . }}
|
||||
chart: {{ include "ceph-csi-cephfs.chart" . }}
|
||||
component: {{ .Values.provisioner.name }}
|
||||
release: {{ .Release.Name }}
|
||||
heritage: {{ .Release.Service }}
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "ceph-csi-cephfs.serviceAccountName.provisioner" . }}
|
||||
namespace: {{ .Release.Namespace }}
|
||||
roleRef:
|
||||
kind: ClusterRole
|
||||
name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }}
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
{{- end -}}
|
19
deploy/cephfs/helm/templates/provisioner-role.yaml
Normal file
19
deploy/cephfs/helm/templates/provisioner-role.yaml
Normal file
@ -0,0 +1,19 @@
|
||||
{{- if .Values.rbac.create -}}
|
||||
kind: Role
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }}
|
||||
labels:
|
||||
app: {{ include "ceph-csi-cephfs.name" . }}
|
||||
chart: {{ include "ceph-csi-cephfs.chart" . }}
|
||||
component: {{ .Values.provisioner.name }}
|
||||
release: {{ .Release.Name }}
|
||||
heritage: {{ .Release.Service }}
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["endpoints"]
|
||||
verbs: ["get", "watch", "list", "delete", "update", "create"]
|
||||
- apiGroups: [""]
|
||||
resources: ["configmaps"]
|
||||
verbs: ["get", "list", "watch", "create", "delete"]
|
||||
{{- end -}}
|
21
deploy/cephfs/helm/templates/provisioner-rolebinding.yaml
Normal file
21
deploy/cephfs/helm/templates/provisioner-rolebinding.yaml
Normal file
@ -0,0 +1,21 @@
|
||||
{{- if .Values.rbac.create -}}
|
||||
kind: RoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }}
|
||||
labels:
|
||||
app: {{ include "ceph-csi-cephfs.name" . }}
|
||||
chart: {{ include "ceph-csi-cephfs.chart" . }}
|
||||
component: {{ .Values.provisioner.name }}
|
||||
release: {{ .Release.Name }}
|
||||
heritage: {{ .Release.Service }}
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "ceph-csi-cephfs.serviceAccountName.provisioner" . }}
|
||||
namespace: {{ .Release.Namespace }}
|
||||
roleRef:
|
||||
kind: Role
|
||||
name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }}
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
namespace: {{ .Release.Namespace }}
|
||||
{{- end -}}
|
18
deploy/cephfs/helm/templates/provisioner-service.yaml
Normal file
18
deploy/cephfs/helm/templates/provisioner-service.yaml
Normal file
@ -0,0 +1,18 @@
|
||||
kind: Service
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }}
|
||||
labels:
|
||||
app: {{ include "ceph-csi-cephfs.name" . }}
|
||||
chart: {{ include "ceph-csi-cephfs.chart" . }}
|
||||
component: {{ .Values.provisioner.name }}
|
||||
release: {{ .Release.Name }}
|
||||
heritage: {{ .Release.Service }}
|
||||
spec:
|
||||
selector:
|
||||
app: {{ include "ceph-csi-cephfs.name" . }}
|
||||
component: {{ .Values.provisioner.name }}
|
||||
release: {{ .Release.Name }}
|
||||
ports:
|
||||
- name: dummy
|
||||
port: 12345
|
12
deploy/cephfs/helm/templates/provisioner-serviceaccount.yaml
Normal file
12
deploy/cephfs/helm/templates/provisioner-serviceaccount.yaml
Normal file
@ -0,0 +1,12 @@
|
||||
{{- if .Values.serviceAccounts.provisioner.create -}}
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: {{ include "ceph-csi-cephfs.serviceAccountName.provisioner" . }}
|
||||
labels:
|
||||
app: {{ include "ceph-csi-cephfs.name" . }}
|
||||
chart: {{ include "ceph-csi-cephfs.chart" . }}
|
||||
component: {{ .Values.provisioner.name }}
|
||||
release: {{ .Release.Name }}
|
||||
heritage: {{ .Release.Service }}
|
||||
{{- end -}}
|
94
deploy/cephfs/helm/templates/provisioner-statefulset.yaml
Normal file
94
deploy/cephfs/helm/templates/provisioner-statefulset.yaml
Normal file
@ -0,0 +1,94 @@
|
||||
kind: StatefulSet
|
||||
apiVersion: apps/v1beta1
|
||||
metadata:
|
||||
name: {{ include "ceph-csi-cephfs.provisioner.fullname" . }}
|
||||
labels:
|
||||
app: {{ include "ceph-csi-cephfs.name" . }}
|
||||
chart: {{ include "ceph-csi-cephfs.chart" . }}
|
||||
component: {{ .Values.provisioner.name }}
|
||||
release: {{ .Release.Name }}
|
||||
heritage: {{ .Release.Service }}
|
||||
spec:
|
||||
serviceName: {{ include "ceph-csi-cephfs.provisioner.fullname" . }}
|
||||
replicas: {{ .Values.provisioner.replicas }}
|
||||
selector:
|
||||
matchLabels:
|
||||
app: {{ include "ceph-csi-cephfs.name" . }}
|
||||
component: {{ .Values.provisioner.name }}
|
||||
release: {{ .Release.Name }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: {{ include "ceph-csi-cephfs.name" . }}
|
||||
chart: {{ include "ceph-csi-cephfs.chart" . }}
|
||||
component: {{ .Values.provisioner.name }}
|
||||
release: {{ .Release.Name }}
|
||||
heritage: {{ .Release.Service }}
|
||||
spec:
|
||||
serviceAccountName: {{ include "ceph-csi-cephfs.serviceAccountName.provisioner" . }}
|
||||
containers:
|
||||
- name: csi-provisioner
|
||||
image: "{{ .Values.provisioner.image.repository }}:{{ .Values.provisioner.image.tag }}"
|
||||
args:
|
||||
- "--csi-address=$(ADDRESS)"
|
||||
- "--v=5"
|
||||
env:
|
||||
- name: ADDRESS
|
||||
value: "{{ .Values.socketDir }}/{{ .Values.socketFile }}"
|
||||
imagePullPolicy: {{ .Values.provisioner.image.pullPolicy }}
|
||||
volumeMounts:
|
||||
- name: socket-dir
|
||||
mountPath: {{ .Values.socketDir }}
|
||||
resources:
|
||||
{{ toYaml .Values.provisioner.resources | indent 12 }}
|
||||
- name: csi-cephfsplugin
|
||||
securityContext:
|
||||
privileged: true
|
||||
capabilities:
|
||||
add: ["SYS_ADMIN"]
|
||||
allowPrivilegeEscalation: true
|
||||
image: "{{ .Values.nodeplugin.plugin.image.repository }}:{{ .Values.nodeplugin.plugin.image.tag }}"
|
||||
args :
|
||||
- "--nodeid=$(NODE_ID)"
|
||||
- "--endpoint=$(CSI_ENDPOINT)"
|
||||
- "--v=5"
|
||||
- "--drivername=$(DRIVER_NAME)"
|
||||
- "--metadatastorage=k8s_configmap"
|
||||
env:
|
||||
- name: HOST_ROOTFS
|
||||
value: "/rootfs"
|
||||
- name: DRIVER_NAME
|
||||
value: {{ .Values.driverName }}
|
||||
- name: NODE_ID
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
- name: CSI_ENDPOINT
|
||||
value: "unix:/{{ .Values.socketDir }}/{{ .Values.socketFile }}"
|
||||
imagePullPolicy: {{ .Values.nodeplugin.plugin.image.imagePullPolicy }}
|
||||
volumeMounts:
|
||||
- name: socket-dir
|
||||
mountPath: {{ .Values.socketDir }}
|
||||
- name: host-rootfs
|
||||
mountPath: "/rootfs"
|
||||
resources:
|
||||
{{ toYaml .Values.nodeplugin.plugin.resources | indent 12 }}
|
||||
volumes:
|
||||
- name: socket-dir
|
||||
emptyDir: {}
|
||||
#FIXME this seems way too much. Why is it needed at all for this?
|
||||
- name: host-rootfs
|
||||
hostPath:
|
||||
path: /
|
||||
{{- if .Values.provisioner.affinity -}}
|
||||
affinity:
|
||||
{{ toYaml .Values.provisioner.affinity . | indent 8 }}
|
||||
{{- end -}}
|
||||
{{- if .Values.provisioner.nodeSelector -}}
|
||||
nodeSelector:
|
||||
{{ toYaml .Values.provisioner.nodeSelector | indent 8 }}
|
||||
{{- end -}}
|
||||
{{- if .Values.provisioner.tolerations -}}
|
||||
tolerations:
|
||||
{{ toYaml .Values.provisioner.tolerations | indent 8 }}
|
||||
{{- end -}}
|
80
deploy/cephfs/helm/values.yaml
Normal file
80
deploy/cephfs/helm/values.yaml
Normal file
@ -0,0 +1,80 @@
|
||||
---
|
||||
rbac:
|
||||
create: true
|
||||
|
||||
serviceAccounts:
|
||||
attacher:
|
||||
create: true
|
||||
name:
|
||||
nodeplugin:
|
||||
create: true
|
||||
name:
|
||||
provisioner:
|
||||
create: true
|
||||
name:
|
||||
|
||||
socketDir: /var/lib/kubelet/plugins/cephfs.csi.ceph.com
|
||||
socketFile: csi.sock
|
||||
registrationDir: /var/lib/kubelet/plugins_registry
|
||||
volumeDevicesDir: /var/lib/kubelet/plugins/kubernetes.io/csi/volumeDevices
|
||||
driverName: cephfs.csi.ceph.com
|
||||
attacher:
|
||||
name: attacher
|
||||
|
||||
replicaCount: 1
|
||||
|
||||
image:
|
||||
repository: quay.io/k8scsi/csi-attacher
|
||||
tag: v1.0.1
|
||||
pullPolicy: IfNotPresent
|
||||
|
||||
resources: {}
|
||||
|
||||
nodeSelector: {}
|
||||
|
||||
tolerations: []
|
||||
|
||||
affinity: {}
|
||||
|
||||
nodeplugin:
|
||||
name: nodeplugin
|
||||
|
||||
registrar:
|
||||
image:
|
||||
repository: quay.io/k8scsi/csi-node-driver-registrar
|
||||
tag: v1.0.2
|
||||
pullPolicy: IfNotPresent
|
||||
|
||||
resources: {}
|
||||
|
||||
plugin:
|
||||
image:
|
||||
repository: quay.io/cephcsi/cephfsplugin
|
||||
tag: v1.0.0
|
||||
pullPolicy: IfNotPresent
|
||||
|
||||
resources: {}
|
||||
|
||||
nodeSelector: {}
|
||||
|
||||
tolerations: []
|
||||
|
||||
affinity: {}
|
||||
|
||||
provisioner:
|
||||
name: provisioner
|
||||
|
||||
replicaCount: 1
|
||||
|
||||
image:
|
||||
repository: quay.io/k8scsi/csi-provisioner
|
||||
tag: v1.0.1
|
||||
pullPolicy: IfNotPresent
|
||||
|
||||
resources: {}
|
||||
|
||||
nodeSelector: {}
|
||||
|
||||
tolerations: []
|
||||
|
||||
affinity: {}
|
@ -1,17 +1,15 @@
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: csi-attacher
|
||||
name: cephfs-csi-attacher
|
||||
|
||||
---
|
||||
kind: ClusterRole
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: external-attacher-runner
|
||||
name: cephfs-external-attacher-runner
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["events"]
|
||||
verbs: ["get", "list", "watch", "update"]
|
||||
- apiGroups: [""]
|
||||
resources: ["persistentvolumes"]
|
||||
verbs: ["get", "list", "watch", "update"]
|
||||
@ -21,17 +19,20 @@ rules:
|
||||
- apiGroups: ["storage.k8s.io"]
|
||||
resources: ["volumeattachments"]
|
||||
verbs: ["get", "list", "watch", "update"]
|
||||
- apiGroups: ["csi.storage.k8s.io"]
|
||||
resources: ["csinodeinfos"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
|
||||
---
|
||||
kind: ClusterRoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: csi-attacher-role
|
||||
name: cephfs-csi-attacher-role
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: csi-attacher
|
||||
name: cephfs-csi-attacher
|
||||
namespace: default
|
||||
roleRef:
|
||||
kind: ClusterRole
|
||||
name: external-attacher-runner
|
||||
name: cephfs-external-attacher-runner
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
|
@ -1,3 +1,4 @@
|
||||
---
|
||||
kind: Service
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
@ -24,10 +25,10 @@ spec:
|
||||
labels:
|
||||
app: csi-cephfsplugin-attacher
|
||||
spec:
|
||||
serviceAccount: csi-attacher
|
||||
serviceAccount: cephfs-csi-attacher
|
||||
containers:
|
||||
- name: csi-cephfsplugin-attacher
|
||||
image: quay.io/k8scsi/csi-attacher:v0.3.0
|
||||
image: quay.io/k8scsi/csi-attacher:v1.0.1
|
||||
args:
|
||||
- "--v=5"
|
||||
- "--csi-address=$(ADDRESS)"
|
||||
|
@ -1,3 +1,4 @@
|
||||
---
|
||||
kind: Service
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
@ -24,23 +25,65 @@ spec:
|
||||
labels:
|
||||
app: csi-cephfsplugin-provisioner
|
||||
spec:
|
||||
serviceAccount: csi-provisioner
|
||||
serviceAccount: cephfs-csi-provisioner
|
||||
containers:
|
||||
- name: csi-provisioner
|
||||
image: quay.io/k8scsi/csi-provisioner:v0.3.0
|
||||
image: quay.io/k8scsi/csi-provisioner:v1.0.1
|
||||
args:
|
||||
- "--provisioner=cephfs.csi.ceph.com"
|
||||
- "--csi-address=$(ADDRESS)"
|
||||
- "--v=5"
|
||||
env:
|
||||
- name: ADDRESS
|
||||
value: /var/lib/kubelet/plugins/cephfs.csi.ceph.com/csi.sock
|
||||
value: unix:///csi/csi-provisioner.sock
|
||||
imagePullPolicy: "IfNotPresent"
|
||||
volumeMounts:
|
||||
- name: socket-dir
|
||||
mountPath: /var/lib/kubelet/plugins/cephfs.csi.ceph.com
|
||||
mountPath: /csi
|
||||
- name: csi-cephfsplugin
|
||||
securityContext:
|
||||
privileged: true
|
||||
capabilities:
|
||||
add: ["SYS_ADMIN"]
|
||||
image: quay.io/cephcsi/cephfsplugin:v1.0.0
|
||||
args:
|
||||
- "--nodeid=$(NODE_ID)"
|
||||
- "--endpoint=$(CSI_ENDPOINT)"
|
||||
- "--v=5"
|
||||
- "--drivername=cephfs.csi.ceph.com"
|
||||
- "--metadatastorage=k8s_configmap"
|
||||
env:
|
||||
- name: NODE_ID
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
- name: POD_NAMESPACE
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.namespace
|
||||
- name: CSI_ENDPOINT
|
||||
value: unix:///csi/csi-provisioner.sock
|
||||
imagePullPolicy: "IfNotPresent"
|
||||
volumeMounts:
|
||||
- name: socket-dir
|
||||
mountPath: /csi
|
||||
- name: host-sys
|
||||
mountPath: /sys
|
||||
- name: lib-modules
|
||||
mountPath: /lib/modules
|
||||
readOnly: true
|
||||
- name: host-dev
|
||||
mountPath: /dev
|
||||
volumes:
|
||||
- name: socket-dir
|
||||
hostPath:
|
||||
path: /var/lib/kubelet/plugins/cephfs.csi.ceph.com
|
||||
type: DirectoryOrCreate
|
||||
- name: host-sys
|
||||
hostPath:
|
||||
path: /sys
|
||||
- name: lib-modules
|
||||
hostPath:
|
||||
path: /lib/modules
|
||||
- name: host-dev
|
||||
hostPath:
|
||||
path: /dev
|
||||
|
@ -1,3 +1,4 @@
|
||||
---
|
||||
kind: DaemonSet
|
||||
apiVersion: apps/v1beta2
|
||||
metadata:
|
||||
@ -11,30 +12,34 @@ spec:
|
||||
labels:
|
||||
app: csi-cephfsplugin
|
||||
spec:
|
||||
serviceAccount: csi-nodeplugin
|
||||
serviceAccount: cephfs-csi-nodeplugin
|
||||
hostNetwork: true
|
||||
# to use e.g. Rook orchestrated cluster, and mons' FQDN is
|
||||
# resolved through k8s service, set dns policy to cluster first
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
containers:
|
||||
- name: driver-registrar
|
||||
image: quay.io/k8scsi/driver-registrar:v0.3.0
|
||||
image: quay.io/k8scsi/csi-node-driver-registrar:v1.0.2
|
||||
args:
|
||||
- "--v=5"
|
||||
- "--csi-address=$(ADDRESS)"
|
||||
- "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)"
|
||||
- "--csi-address=/csi/csi.sock"
|
||||
- "--kubelet-registration-path=/var/lib/kubelet/plugins/cephfs.csi.ceph.com/csi.sock"
|
||||
lifecycle:
|
||||
preStop:
|
||||
exec:
|
||||
command: [
|
||||
"/bin/sh", "-c",
|
||||
"rm -rf /registration/csi-cephfsplugin \
|
||||
/registration/csi-cephfsplugin-reg.sock"
|
||||
]
|
||||
env:
|
||||
- name: ADDRESS
|
||||
value: /var/lib/kubelet/plugins/cephfs.csi.ceph.com/csi.sock
|
||||
- name: DRIVER_REG_SOCK_PATH
|
||||
value: /var/lib/kubelet/plugins/cephfs.csi.ceph.com/csi.sock
|
||||
- name: KUBE_NODE_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
volumeMounts:
|
||||
- name: socket-dir
|
||||
mountPath: /var/lib/kubelet/plugins/cephfs.csi.ceph.com
|
||||
- name: plugin-dir
|
||||
mountPath: /csi
|
||||
- name: registration-dir
|
||||
mountPath: /registration
|
||||
- name: csi-cephfsplugin
|
||||
@ -43,13 +48,14 @@ spec:
|
||||
capabilities:
|
||||
add: ["SYS_ADMIN"]
|
||||
allowPrivilegeEscalation: true
|
||||
image: quay.io/cephcsi/cephfsplugin:v0.3.0
|
||||
image: quay.io/cephcsi/cephfsplugin:v1.0.0
|
||||
args:
|
||||
- "--nodeid=$(NODE_ID)"
|
||||
- "--endpoint=$(CSI_ENDPOINT)"
|
||||
- "--v=5"
|
||||
- "--drivername=cephfs.csi.ceph.com"
|
||||
- "--metadatastorage=k8s_configmap"
|
||||
- "--mountcachedir=/mount-cache-dir"
|
||||
env:
|
||||
- name: NODE_ID
|
||||
valueFrom:
|
||||
@ -60,38 +66,45 @@ spec:
|
||||
fieldRef:
|
||||
fieldPath: metadata.namespace
|
||||
- name: CSI_ENDPOINT
|
||||
value: unix://var/lib/kubelet/plugins/cephfs.csi.ceph.com/csi.sock
|
||||
value: unix:///csi/csi.sock
|
||||
imagePullPolicy: "IfNotPresent"
|
||||
volumeMounts:
|
||||
- name: mount-cache-dir
|
||||
mountPath: /mount-cache-dir
|
||||
- name: plugin-dir
|
||||
mountPath: /var/lib/kubelet/plugins/cephfs.csi.ceph.com
|
||||
mountPath: /csi
|
||||
- name: csi-plugins-dir
|
||||
mountPath: /var/lib/kubelet/plugins/kubernetes.io/csi
|
||||
mountPropagation: "Bidirectional"
|
||||
- name: pods-mount-dir
|
||||
mountPath: /var/lib/kubelet/pods
|
||||
mountPropagation: "Bidirectional"
|
||||
- mountPath: /sys
|
||||
name: host-sys
|
||||
- name: host-sys
|
||||
mountPath: /sys
|
||||
- name: lib-modules
|
||||
mountPath: /lib/modules
|
||||
readOnly: true
|
||||
- name: host-dev
|
||||
mountPath: /dev
|
||||
volumes:
|
||||
- name: mount-cache-dir
|
||||
emptyDir: {}
|
||||
- name: plugin-dir
|
||||
hostPath:
|
||||
path: /var/lib/kubelet/plugins/cephfs.csi.ceph.com
|
||||
path: /var/lib/kubelet/plugins/cephfs.csi.ceph.com/
|
||||
type: DirectoryOrCreate
|
||||
- name: csi-plugins-dir
|
||||
hostPath:
|
||||
path: /var/lib/kubelet/plugins/kubernetes.io/csi
|
||||
type: DirectoryOrCreate
|
||||
- name: registration-dir
|
||||
hostPath:
|
||||
path: /var/lib/kubelet/plugins/
|
||||
path: /var/lib/kubelet/plugins_registry/
|
||||
type: Directory
|
||||
- name: pods-mount-dir
|
||||
hostPath:
|
||||
path: /var/lib/kubelet/pods
|
||||
type: Directory
|
||||
- name: socket-dir
|
||||
hostPath:
|
||||
path: /var/lib/kubelet/plugins/cephfs.csi.ceph.com
|
||||
type: DirectoryOrCreate
|
||||
- name: host-sys
|
||||
hostPath:
|
||||
path: /sys
|
||||
|
@ -1,14 +1,18 @@
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: csi-nodeplugin
|
||||
name: cephfs-csi-nodeplugin
|
||||
|
||||
---
|
||||
kind: ClusterRole
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: csi-nodeplugin
|
||||
name: cephfs-csi-nodeplugin
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["configmaps"]
|
||||
verbs: ["get", "list"]
|
||||
- apiGroups: [""]
|
||||
resources: ["nodes"]
|
||||
verbs: ["get", "list", "update"]
|
||||
@ -21,20 +25,17 @@ rules:
|
||||
- apiGroups: ["storage.k8s.io"]
|
||||
resources: ["volumeattachments"]
|
||||
verbs: ["get", "list", "watch", "update"]
|
||||
- apiGroups: [""]
|
||||
resources: ["configmaps"]
|
||||
verbs: ["get", "list", "create", "delete"]
|
||||
|
||||
---
|
||||
kind: ClusterRoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: csi-nodeplugin
|
||||
name: cephfs-csi-nodeplugin
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: csi-nodeplugin
|
||||
name: cephfs-csi-nodeplugin
|
||||
namespace: default
|
||||
roleRef:
|
||||
kind: ClusterRole
|
||||
name: csi-nodeplugin
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
name: cephfs-csi-nodeplugin
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
|
@ -1,17 +1,24 @@
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: csi-provisioner
|
||||
name: cephfs-csi-provisioner
|
||||
|
||||
---
|
||||
kind: ClusterRole
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: external-provisioner-runner
|
||||
name: cephfs-external-provisioner-runner
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["nodes"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["secrets"]
|
||||
verbs: ["get", "list"]
|
||||
- apiGroups: [""]
|
||||
resources: ["events"]
|
||||
verbs: ["list", "watch", "create", "update", "patch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["persistentvolumes"]
|
||||
verbs: ["get", "list", "watch", "create", "delete"]
|
||||
@ -21,20 +28,52 @@ rules:
|
||||
- apiGroups: ["storage.k8s.io"]
|
||||
resources: ["storageclasses"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["events"]
|
||||
verbs: ["list", "watch", "create", "update", "patch"]
|
||||
|
||||
- apiGroups: ["csi.storage.k8s.io"]
|
||||
resources: ["csinodeinfos"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
|
||||
---
|
||||
kind: ClusterRoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: csi-provisioner-role
|
||||
name: cephfs-csi-provisioner-role
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: csi-provisioner
|
||||
name: cephfs-csi-provisioner
|
||||
namespace: default
|
||||
roleRef:
|
||||
kind: ClusterRole
|
||||
name: external-provisioner-runner
|
||||
name: cephfs-external-provisioner-runner
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
|
||||
---
|
||||
kind: Role
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
# replace with non-default namespace name
|
||||
namespace: default
|
||||
name: cephfs-external-provisioner-cfg
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["endpoints"]
|
||||
verbs: ["get", "watch", "list", "delete", "update", "create"]
|
||||
- apiGroups: [""]
|
||||
resources: ["configmaps"]
|
||||
verbs: ["get", "list", "create", "delete"]
|
||||
|
||||
---
|
||||
kind: RoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: cephfs-csi-provisioner-role-cfg
|
||||
# replace with non-default namespace name
|
||||
namespace: default
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: cephfs-csi-provisioner
|
||||
# replace with non-default namespace name
|
||||
namespace: default
|
||||
roleRef:
|
||||
kind: Role
|
||||
name: cephfs-external-provisioner-cfg
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
|
@ -1,12 +1,14 @@
|
||||
---
|
||||
apiVersion: v1
|
||||
appVersion: "0.3.0"
|
||||
description: Container Storage Interface (CSI) driver, provisioner, and attacher for Ceph RBD
|
||||
appVersion: "1.0.0"
|
||||
description: "Container Storage Interface (CSI) driver,
|
||||
provisioner, snapshotter, and attacher for Ceph RBD"
|
||||
name: ceph-csi-rbd
|
||||
version: 0.1.0
|
||||
version: 0.5.1
|
||||
keywords:
|
||||
- ceph
|
||||
- rbd
|
||||
- ceph-csi
|
||||
- ceph
|
||||
- rbd
|
||||
- ceph-csi
|
||||
home: https://github.com/ceph/ceph-csi
|
||||
sources:
|
||||
- https://github.com/ceph/ceph-csi/tree/master/deploy/rbd/kubernetes
|
||||
- https://github.com/ceph/ceph-csi/tree/csi-v1.0/deploy/rbd/helm
|
||||
|
29
deploy/rbd/helm/README.md
Normal file
29
deploy/rbd/helm/README.md
Normal file
@ -0,0 +1,29 @@
|
||||
# ceph-csi-rbd
|
||||
|
||||
The ceph-csi-rbd chart adds rbd volume support to your cluster.
|
||||
|
||||
## Install Chart
|
||||
|
||||
To install the Chart into your Kubernetes cluster
|
||||
|
||||
```bash
|
||||
helm install --namespace "ceph-csi-rbd" --name "ceph-csi-rbd" ceph-csi/ceph-csi-rbd
|
||||
```
|
||||
|
||||
After installation succeeds, you can get a status of Chart
|
||||
|
||||
```bash
|
||||
helm status "ceph-csi-rbd"
|
||||
```
|
||||
|
||||
If you want to delete your Chart, use this command
|
||||
|
||||
```bash
|
||||
helm delete --purge "ceph-csi-rbd"
|
||||
```
|
||||
|
||||
If you want to delete the namespace, use this command
|
||||
|
||||
```bash
|
||||
kubectl delete namespace ceph-csi-rbd
|
||||
```
|
@ -1 +1,2 @@
|
||||
The Ceph RBD Container Storage Interface has been deployed.
|
||||
Examples on how to configure a storage class and start using the driver are here:
|
||||
https://github.com/ceph/ceph-csi/tree/csi-v1.0/examples/rbd
|
||||
|
@ -22,4 +22,7 @@ rules:
|
||||
- apiGroups: ["storage.k8s.io"]
|
||||
resources: ["volumeattachments"]
|
||||
verbs: ["get", "list", "watch", "update"]
|
||||
- apiGroups: ["csi.storage.k8s.io"]
|
||||
resources: ["csinodeinfos"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
{{- end -}}
|
||||
|
@ -11,6 +11,11 @@ metadata:
|
||||
spec:
|
||||
serviceName: {{ include "ceph-csi-rbd.attacher.fullname" . }}
|
||||
replicas: {{ .Values.attacher.replicas }}
|
||||
selector:
|
||||
matchLabels:
|
||||
app: {{ include "ceph-csi-rbd.name" . }}
|
||||
component: {{ .Values.attacher.name }}
|
||||
release: {{ .Release.Name }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
|
@ -22,4 +22,7 @@ rules:
|
||||
- apiGroups: ["storage.k8s.io"]
|
||||
resources: ["volumeattachments"]
|
||||
verbs: ["get", "list", "watch", "update"]
|
||||
- apiGroups: [""]
|
||||
resources: ["configmaps"]
|
||||
verbs: ["get", "list"]
|
||||
{{- end -}}
|
||||
|
@ -34,21 +34,25 @@ spec:
|
||||
image: "{{ .Values.nodeplugin.registrar.image.repository }}:{{ .Values.nodeplugin.registrar.image.tag }}"
|
||||
args:
|
||||
- "--v=5"
|
||||
- "--csi-address=$(ADDRESS)"
|
||||
- "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)"
|
||||
- "--csi-address=/csi/{{ .Values.socketFile }}"
|
||||
- "--kubelet-registration-path={{ .Values.socketDir }}/{{ .Values.socketFile }}"
|
||||
lifecycle:
|
||||
preStop:
|
||||
exec:
|
||||
command: [
|
||||
"/bin/sh", "-c",
|
||||
'rm -rf /registration/{{ .Values.driverName }}
|
||||
/registration/{{ .Values.driverName }}-reg.sock'
|
||||
]
|
||||
env:
|
||||
- name: ADDRESS
|
||||
value: "{{ .Values.socketDir }}/{{ .Values.socketFile }}"
|
||||
- name: DRIVER_REG_SOCK_PATH
|
||||
value: "{{ .Values.socketDir }}/{{ .Values.socketFile }}"
|
||||
- name: KUBE_NODE_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
imagePullPolicy: {{ .Values.nodeplugin.registrar.image.imagePullPolicy }}
|
||||
volumeMounts:
|
||||
- name: socket-dir
|
||||
mountPath: {{ .Values.socketDir }}
|
||||
- name: plugin-dir
|
||||
mountPath: /csi
|
||||
- name: registration-dir
|
||||
mountPath: /registration
|
||||
resources:
|
||||
@ -64,11 +68,14 @@ spec:
|
||||
- "--nodeid=$(NODE_ID)"
|
||||
- "--endpoint=$(CSI_ENDPOINT)"
|
||||
- "--v=5"
|
||||
- "--drivername=rbd.csi.ceph.com"
|
||||
- "--containerized=true"
|
||||
- "--drivername=$(DRIVER_NAME)"
|
||||
- "--containerized=true"
|
||||
- "--metadatastorage=k8s_configmap"
|
||||
env:
|
||||
- name: HOST_ROOTFS
|
||||
value: "/rootfs"
|
||||
- name: DRIVER_NAME
|
||||
value: {{ .Values.driverName }}
|
||||
- name: NODE_ID
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
@ -82,6 +89,9 @@ spec:
|
||||
- name: pods-mount-dir
|
||||
mountPath: /var/lib/kubelet/pods
|
||||
mountPropagation: "Bidirectional"
|
||||
- name: plugin-mount-dir
|
||||
mountPath: {{ .Values.volumeDevicesDir }}
|
||||
mountPropagation: "Bidirectional"
|
||||
- mountPath: /dev
|
||||
name: host-dev
|
||||
- mountPath: /rootfs
|
||||
@ -98,18 +108,18 @@ spec:
|
||||
hostPath:
|
||||
path: {{ .Values.socketDir }}
|
||||
type: DirectoryOrCreate
|
||||
- name: plugin-mount-dir
|
||||
hostPath:
|
||||
path: {{ .Values.volumeDevicesDir }}
|
||||
type: DirectoryOrCreate
|
||||
- name: registration-dir
|
||||
hostPath:
|
||||
path: /var/lib/kubelet/plugins/
|
||||
path: {{ .Values.registrationDir }}
|
||||
type: Directory
|
||||
- name: pods-mount-dir
|
||||
hostPath:
|
||||
path: /var/lib/kubelet/pods
|
||||
type: Directory
|
||||
- name: socket-dir
|
||||
hostPath:
|
||||
path: {{ .Values.socketDir }}
|
||||
type: DirectoryOrCreate
|
||||
- name: host-dev
|
||||
hostPath:
|
||||
path: /dev
|
||||
|
@ -10,6 +10,9 @@ metadata:
|
||||
release: {{ .Release.Name }}
|
||||
heritage: {{ .Release.Service }}
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["nodes"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["secrets"]
|
||||
verbs: ["get", "list"]
|
||||
@ -25,4 +28,25 @@ rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["events"]
|
||||
verbs: ["list", "watch", "create", "update", "patch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["endpoints"]
|
||||
verbs: ["get", "create", "update"]
|
||||
- apiGroups: ["snapshot.storage.k8s.io"]
|
||||
resources: ["volumesnapshots"]
|
||||
verbs: ["get", "list", "watch", "update"]
|
||||
- apiGroups: [""]
|
||||
resources: ["configmaps"]
|
||||
verbs: ["get", "list", "create", "delete"]
|
||||
- apiGroups: ["snapshot.storage.k8s.io"]
|
||||
resources: ["volumesnapshotcontents"]
|
||||
verbs: ["create", "get", "list", "watch", "update", "delete"]
|
||||
- apiGroups: ["snapshot.storage.k8s.io"]
|
||||
resources: ["volumesnapshotclasses"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: ["apiextensions.k8s.io"]
|
||||
resources: ["customresourcedefinitions"]
|
||||
verbs: ["create"]
|
||||
- apiGroups: ["csi.storage.k8s.io"]
|
||||
resources: ["csinodeinfos"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
{{- end -}}
|
||||
|
16
deploy/rbd/helm/templates/provisioner-role.yaml
Normal file
16
deploy/rbd/helm/templates/provisioner-role.yaml
Normal file
@ -0,0 +1,16 @@
|
||||
{{- if .Values.rbac.create -}}
|
||||
kind: Role
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: {{ include "ceph-csi-rbd.provisioner.fullname" . }}
|
||||
labels:
|
||||
app: {{ include "ceph-csi-rbd.name" . }}
|
||||
chart: {{ include "ceph-csi-rbd.chart" . }}
|
||||
component: {{ .Values.provisioner.name }}
|
||||
release: {{ .Release.Name }}
|
||||
heritage: {{ .Release.Service }}
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["configmaps"]
|
||||
verbs: ["get", "list", "watch", "create", "delete"]
|
||||
{{- end -}}
|
21
deploy/rbd/helm/templates/provisioner-rolebinding.yaml
Normal file
21
deploy/rbd/helm/templates/provisioner-rolebinding.yaml
Normal file
@ -0,0 +1,21 @@
|
||||
{{- if .Values.rbac.create -}}
|
||||
kind: RoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: {{ include "ceph-csi-rbd.provisioner.fullname" . }}
|
||||
labels:
|
||||
app: {{ include "ceph-csi-rbd.name" . }}
|
||||
chart: {{ include "ceph-csi-rbd.chart" . }}
|
||||
component: {{ .Values.provisioner.name }}
|
||||
release: {{ .Release.Name }}
|
||||
heritage: {{ .Release.Service }}
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: {{ include "ceph-csi-rbd.serviceAccountName.provisioner" . }}
|
||||
namespace: {{ .Release.Namespace }}
|
||||
roleRef:
|
||||
kind: Role
|
||||
name: {{ include "ceph-csi-rbd.provisioner.fullname" . }}
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
namespace: {{ .Release.Namespace }}
|
||||
{{- end -}}
|
@ -11,6 +11,11 @@ metadata:
|
||||
spec:
|
||||
serviceName: {{ include "ceph-csi-rbd.provisioner.fullname" . }}
|
||||
replicas: {{ .Values.provisioner.replicas }}
|
||||
selector:
|
||||
matchLabels:
|
||||
app: {{ include "ceph-csi-rbd.name" . }}
|
||||
component: {{ .Values.provisioner.name }}
|
||||
release: {{ .Release.Name }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
@ -25,7 +30,6 @@ spec:
|
||||
- name: csi-provisioner
|
||||
image: "{{ .Values.provisioner.image.repository }}:{{ .Values.provisioner.image.tag }}"
|
||||
args:
|
||||
- "--provisioner=rbd.csi.ceph.com"
|
||||
- "--csi-address=$(ADDRESS)"
|
||||
- "--v=5"
|
||||
env:
|
||||
@ -37,11 +41,63 @@ spec:
|
||||
mountPath: {{ .Values.socketDir }}
|
||||
resources:
|
||||
{{ toYaml .Values.provisioner.resources | indent 12 }}
|
||||
- name: csi-snapshotter
|
||||
image: {{ .Values.snapshotter.image.repository }}:{{ .Values.snapshotter.image.tag }}
|
||||
imagePullPolicy: {{ .Values.nodeplugin.plugin.image.imagePullPolicy }}
|
||||
args:
|
||||
- "--csi-address=$(ADDRESS)"
|
||||
- "--connection-timeout=15s"
|
||||
- "--v=5"
|
||||
env:
|
||||
- name: ADDRESS
|
||||
value: "{{ .Values.socketDir }}/{{ .Values.socketFile }}"
|
||||
securityContext:
|
||||
privileged: true
|
||||
volumeMounts:
|
||||
- name: socket-dir
|
||||
mountPath: {{ .Values.socketDir }}
|
||||
resources:
|
||||
{{ toYaml .Values.snapshotter.resources | indent 12 }}
|
||||
- name: csi-rbdplugin
|
||||
securityContext:
|
||||
privileged: true
|
||||
capabilities:
|
||||
add: ["SYS_ADMIN"]
|
||||
allowPrivilegeEscalation: true
|
||||
image: "{{ .Values.nodeplugin.plugin.image.repository }}:{{ .Values.nodeplugin.plugin.image.tag }}"
|
||||
args :
|
||||
- "--nodeid=$(NODE_ID)"
|
||||
- "--endpoint=$(CSI_ENDPOINT)"
|
||||
- "--v=5"
|
||||
- "--drivername=$(DRIVER_NAME)"
|
||||
- "--containerized=true"
|
||||
- "--metadatastorage=k8s_configmap"
|
||||
env:
|
||||
- name: HOST_ROOTFS
|
||||
value: "/rootfs"
|
||||
- name: DRIVER_NAME
|
||||
value: {{ .Values.driverName }}
|
||||
- name: NODE_ID
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
- name: CSI_ENDPOINT
|
||||
value: "unix:/{{ .Values.socketDir }}/{{ .Values.socketFile }}"
|
||||
imagePullPolicy: {{ .Values.nodeplugin.plugin.image.imagePullPolicy }}
|
||||
volumeMounts:
|
||||
- name: socket-dir
|
||||
mountPath: {{ .Values.socketDir }}
|
||||
- name: host-rootfs
|
||||
mountPath: "/rootfs"
|
||||
resources:
|
||||
{{ toYaml .Values.nodeplugin.plugin.resources | indent 12 }}
|
||||
volumes:
|
||||
- name: socket-dir
|
||||
emptyDir: {}
|
||||
#FIXME this seems way too much. Why is it needed at all for this?
|
||||
- name: host-rootfs
|
||||
hostPath:
|
||||
path: {{ .Values.socketDir }}
|
||||
type: DirectoryOrCreate
|
||||
path: /
|
||||
{{- if .Values.provisioner.affinity -}}
|
||||
affinity:
|
||||
{{ toYaml .Values.provisioner.affinity . | indent 8 }}
|
||||
|
@ -1,3 +1,4 @@
|
||||
---
|
||||
rbac:
|
||||
create: true
|
||||
|
||||
@ -14,6 +15,9 @@ serviceAccounts:
|
||||
|
||||
socketDir: /var/lib/kubelet/plugins/rbd.csi.ceph.com
|
||||
socketFile: csi.sock
|
||||
registrationDir: /var/lib/kubelet/plugins_registry
|
||||
volumeDevicesDir: /var/lib/kubelet/plugins/kubernetes.io/csi/volumeDevices
|
||||
driverName: rbd.csi.ceph.com
|
||||
|
||||
attacher:
|
||||
name: attacher
|
||||
@ -22,7 +26,7 @@ attacher:
|
||||
|
||||
image:
|
||||
repository: quay.io/k8scsi/csi-attacher
|
||||
tag: v0.3.0
|
||||
tag: v1.0.1
|
||||
pullPolicy: IfNotPresent
|
||||
|
||||
resources: {}
|
||||
@ -38,18 +42,18 @@ nodeplugin:
|
||||
|
||||
registrar:
|
||||
image:
|
||||
repository: quay.io/k8scsi/driver-registrar
|
||||
tag: v0.3.0
|
||||
repository: quay.io/k8scsi/csi-node-driver-registrar
|
||||
tag: v1.0.2
|
||||
pullPolicy: IfNotPresent
|
||||
|
||||
|
||||
resources: {}
|
||||
|
||||
plugin:
|
||||
image:
|
||||
repository: quay.io/cephcsi/rbdplugin
|
||||
tag: v0.3.0
|
||||
tag: v1.0.0
|
||||
pullPolicy: IfNotPresent
|
||||
|
||||
|
||||
resources: {}
|
||||
|
||||
nodeSelector: {}
|
||||
@ -65,7 +69,7 @@ provisioner:
|
||||
|
||||
image:
|
||||
repository: quay.io/k8scsi/csi-provisioner
|
||||
tag: v0.3.0
|
||||
tag: v1.0.1
|
||||
pullPolicy: IfNotPresent
|
||||
|
||||
resources: {}
|
||||
@ -75,3 +79,11 @@ provisioner:
|
||||
tolerations: []
|
||||
|
||||
affinity: {}
|
||||
|
||||
snapshotter:
|
||||
image:
|
||||
repository: quay.io/k8scsi/csi-snapshotter
|
||||
tag: v1.0.1
|
||||
pullPolicy: IfNotPresent
|
||||
|
||||
resources: {}
|
||||
|
@ -1,17 +1,15 @@
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: csi-attacher
|
||||
name: rbd-csi-attacher
|
||||
|
||||
---
|
||||
kind: ClusterRole
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: external-attacher-runner
|
||||
name: rbd-external-attacher-runner
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["events"]
|
||||
verbs: ["get", "list", "watch", "update"]
|
||||
- apiGroups: [""]
|
||||
resources: ["persistentvolumes"]
|
||||
verbs: ["get", "list", "watch", "update"]
|
||||
@ -21,17 +19,20 @@ rules:
|
||||
- apiGroups: ["storage.k8s.io"]
|
||||
resources: ["volumeattachments"]
|
||||
verbs: ["get", "list", "watch", "update"]
|
||||
- apiGroups: ["csi.storage.k8s.io"]
|
||||
resources: ["csinodeinfos"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
|
||||
---
|
||||
kind: ClusterRoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: csi-attacher-role
|
||||
name: rbd-csi-attacher-role
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: csi-attacher
|
||||
name: rbd-csi-attacher
|
||||
namespace: default
|
||||
roleRef:
|
||||
kind: ClusterRole
|
||||
name: external-attacher-runner
|
||||
name: rbd-external-attacher-runner
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
|
@ -1,14 +1,18 @@
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: csi-nodeplugin
|
||||
name: rbd-csi-nodeplugin
|
||||
|
||||
---
|
||||
kind: ClusterRole
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: csi-nodeplugin
|
||||
name: rbd-csi-nodeplugin
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["secrets"]
|
||||
verbs: ["get", "list"]
|
||||
- apiGroups: [""]
|
||||
resources: ["nodes"]
|
||||
verbs: ["get", "list", "update"]
|
||||
@ -23,18 +27,18 @@ rules:
|
||||
verbs: ["get", "list", "watch", "update"]
|
||||
- apiGroups: [""]
|
||||
resources: ["configmaps"]
|
||||
verbs: ["get", "list", "create", "delete"]
|
||||
verbs: ["get", "list"]
|
||||
|
||||
---
|
||||
kind: ClusterRoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: csi-nodeplugin
|
||||
name: rbd-csi-nodeplugin
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: csi-nodeplugin
|
||||
name: rbd-csi-nodeplugin
|
||||
namespace: default
|
||||
roleRef:
|
||||
kind: ClusterRole
|
||||
name: csi-nodeplugin
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
name: rbd-csi-nodeplugin
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
|
@ -1,17 +1,24 @@
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: csi-provisioner
|
||||
name: rbd-csi-provisioner
|
||||
|
||||
---
|
||||
kind: ClusterRole
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: external-provisioner-runner
|
||||
name: rbd-external-provisioner-runner
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["nodes"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["secrets"]
|
||||
verbs: ["get", "list"]
|
||||
- apiGroups: [""]
|
||||
resources: ["events"]
|
||||
verbs: ["list", "watch", "create", "update", "patch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["persistentvolumes"]
|
||||
verbs: ["get", "list", "watch", "create", "delete"]
|
||||
@ -21,20 +28,64 @@ rules:
|
||||
- apiGroups: ["storage.k8s.io"]
|
||||
resources: ["storageclasses"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["events"]
|
||||
verbs: ["list", "watch", "create", "update", "patch"]
|
||||
|
||||
- apiGroups: ["snapshot.storage.k8s.io"]
|
||||
resources: ["volumesnapshots"]
|
||||
verbs: ["get", "list", "watch", "update"]
|
||||
- apiGroups: ["snapshot.storage.k8s.io"]
|
||||
resources: ["volumesnapshotcontents"]
|
||||
verbs: ["create", "get", "list", "watch", "update", "delete"]
|
||||
- apiGroups: ["snapshot.storage.k8s.io"]
|
||||
resources: ["volumesnapshotclasses"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: ["apiextensions.k8s.io"]
|
||||
resources: ["customresourcedefinitions"]
|
||||
verbs: ["create"]
|
||||
- apiGroups: ["csi.storage.k8s.io"]
|
||||
resources: ["csinodeinfos"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
|
||||
---
|
||||
kind: ClusterRoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: csi-provisioner-role
|
||||
name: rbd-csi-provisioner-role
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: csi-provisioner
|
||||
name: rbd-csi-provisioner
|
||||
namespace: default
|
||||
roleRef:
|
||||
kind: ClusterRole
|
||||
name: external-provisioner-runner
|
||||
name: rbd-external-provisioner-runner
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
|
||||
---
|
||||
kind: Role
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
# replace with non-default namespace name
|
||||
namespace: default
|
||||
name: rbd-external-provisioner-cfg
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["endpoints"]
|
||||
verbs: ["get", "watch", "list", "delete", "update", "create"]
|
||||
- apiGroups: [""]
|
||||
resources: ["configmaps"]
|
||||
verbs: ["get", "list", "watch", "create", "delete"]
|
||||
|
||||
---
|
||||
kind: RoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: rbd-csi-provisioner-role-cfg
|
||||
# replace with non-default namespace name
|
||||
namespace: default
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: rbd-csi-provisioner
|
||||
# replace with non-default namespace name
|
||||
namespace: default
|
||||
roleRef:
|
||||
kind: Role
|
||||
name: rbd-external-provisioner-cfg
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
|
@ -1,3 +1,4 @@
|
||||
---
|
||||
kind: Service
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
@ -24,20 +25,20 @@ spec:
|
||||
labels:
|
||||
app: csi-rbdplugin-attacher
|
||||
spec:
|
||||
serviceAccount: csi-attacher
|
||||
serviceAccount: rbd-csi-attacher
|
||||
containers:
|
||||
- name: csi-rbdplugin-attacher
|
||||
image: quay.io/k8scsi/csi-attacher:v0.3.0
|
||||
image: quay.io/k8scsi/csi-attacher:v1.0.1
|
||||
args:
|
||||
- "--v=5"
|
||||
- "--csi-address=$(ADDRESS)"
|
||||
env:
|
||||
- name: ADDRESS
|
||||
value: /var/lib/kubelet/plugins/rbd.csi.ceph.com/csi.sock
|
||||
value: unix:///csi/csi-attacher.sock
|
||||
imagePullPolicy: "IfNotPresent"
|
||||
volumeMounts:
|
||||
- name: socket-dir
|
||||
mountPath: /var/lib/kubelet/plugins/rbd.csi.ceph.com
|
||||
mountPath: /csi
|
||||
volumes:
|
||||
- name: socket-dir
|
||||
hostPath:
|
||||
|
@ -1,3 +1,4 @@
|
||||
---
|
||||
kind: Service
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
@ -24,22 +25,88 @@ spec:
|
||||
labels:
|
||||
app: csi-rbdplugin-provisioner
|
||||
spec:
|
||||
serviceAccount: csi-provisioner
|
||||
serviceAccount: rbd-csi-provisioner
|
||||
containers:
|
||||
- name: csi-provisioner
|
||||
image: quay.io/k8scsi/csi-provisioner:v0.3.0
|
||||
image: quay.io/k8scsi/csi-provisioner:v1.0.1
|
||||
args:
|
||||
- "--provisioner=rbd.csi.ceph.com"
|
||||
- "--csi-address=$(ADDRESS)"
|
||||
- "--v=5"
|
||||
env:
|
||||
- name: ADDRESS
|
||||
value: /var/lib/kubelet/plugins/rbd.csi.ceph.com/csi.sock
|
||||
value: unix:///csi/csi-provisioner.sock
|
||||
imagePullPolicy: "IfNotPresent"
|
||||
volumeMounts:
|
||||
- name: socket-dir
|
||||
mountPath: /var/lib/kubelet/plugins/rbd.csi.ceph.com
|
||||
mountPath: /csi
|
||||
- name: csi-snapshotter
|
||||
image: quay.io/k8scsi/csi-snapshotter:v1.0.1
|
||||
args:
|
||||
- "--csi-address=$(ADDRESS)"
|
||||
- "--connection-timeout=15s"
|
||||
- "--v=5"
|
||||
env:
|
||||
- name: ADDRESS
|
||||
value: unix:///csi/csi-provisioner.sock
|
||||
imagePullPolicy: Always
|
||||
securityContext:
|
||||
privileged: true
|
||||
volumeMounts:
|
||||
- name: socket-dir
|
||||
mountPath: /csi
|
||||
- name: csi-rbdplugin
|
||||
securityContext:
|
||||
privileged: true
|
||||
capabilities:
|
||||
add: ["SYS_ADMIN"]
|
||||
image: quay.io/cephcsi/rbdplugin:v1.0.0
|
||||
args:
|
||||
- "--nodeid=$(NODE_ID)"
|
||||
- "--endpoint=$(CSI_ENDPOINT)"
|
||||
- "--v=5"
|
||||
- "--drivername=rbd.csi.ceph.com"
|
||||
- "--containerized=true"
|
||||
- "--metadatastorage=k8s_configmap"
|
||||
- "--configroot=k8s_objects"
|
||||
env:
|
||||
- name: HOST_ROOTFS
|
||||
value: "/rootfs"
|
||||
- name: NODE_ID
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
- name: POD_NAMESPACE
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.namespace
|
||||
- name: CSI_ENDPOINT
|
||||
value: unix:///csi/csi-provisioner.sock
|
||||
imagePullPolicy: "IfNotPresent"
|
||||
volumeMounts:
|
||||
- name: socket-dir
|
||||
mountPath: /csi
|
||||
- mountPath: /dev
|
||||
name: host-dev
|
||||
- mountPath: /rootfs
|
||||
name: host-rootfs
|
||||
- mountPath: /sys
|
||||
name: host-sys
|
||||
- mountPath: /lib/modules
|
||||
name: lib-modules
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: host-dev
|
||||
hostPath:
|
||||
path: /dev
|
||||
- name: host-rootfs
|
||||
hostPath:
|
||||
path: /
|
||||
- name: host-sys
|
||||
hostPath:
|
||||
path: /sys
|
||||
- name: lib-modules
|
||||
hostPath:
|
||||
path: /lib/modules
|
||||
- name: socket-dir
|
||||
hostPath:
|
||||
path: /var/lib/kubelet/plugins/rbd.csi.ceph.com
|
||||
|
@ -1,3 +1,4 @@
|
||||
---
|
||||
kind: DaemonSet
|
||||
apiVersion: apps/v1beta2
|
||||
metadata:
|
||||
@ -11,7 +12,7 @@ spec:
|
||||
labels:
|
||||
app: csi-rbdplugin
|
||||
spec:
|
||||
serviceAccount: csi-nodeplugin
|
||||
serviceAccount: rbd-csi-nodeplugin
|
||||
hostNetwork: true
|
||||
hostPID: true
|
||||
# to use e.g. Rook orchestrated cluster, and mons' FQDN is
|
||||
@ -19,23 +20,27 @@ spec:
|
||||
dnsPolicy: ClusterFirstWithHostNet
|
||||
containers:
|
||||
- name: driver-registrar
|
||||
image: quay.io/k8scsi/driver-registrar:v0.3.0
|
||||
image: quay.io/k8scsi/csi-node-driver-registrar:v1.0.2
|
||||
args:
|
||||
- "--v=5"
|
||||
- "--csi-address=$(ADDRESS)"
|
||||
- "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)"
|
||||
- "--csi-address=/csi/csi.sock"
|
||||
- "--kubelet-registration-path=/var/lib/kubelet/plugins/rbd.csi.ceph.com/csi.sock"
|
||||
lifecycle:
|
||||
preStop:
|
||||
exec:
|
||||
command: [
|
||||
"/bin/sh", "-c",
|
||||
"rm -rf /registration/rbd.csi.ceph.com \
|
||||
/registration/rbd.csi.ceph.com-reg.sock"
|
||||
]
|
||||
env:
|
||||
- name: ADDRESS
|
||||
value: /var/lib/kubelet/plugins/rbd.csi.ceph.com/csi.sock
|
||||
- name: DRIVER_REG_SOCK_PATH
|
||||
value: /var/lib/kubelet/plugins/rbd.csi.ceph.com/csi.sock
|
||||
- name: KUBE_NODE_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: spec.nodeName
|
||||
volumeMounts:
|
||||
- name: socket-dir
|
||||
mountPath: /var/lib/kubelet/plugins/rbd.csi.ceph.com
|
||||
- name: plugin-dir
|
||||
mountPath: /csi
|
||||
- name: registration-dir
|
||||
mountPath: /registration
|
||||
- name: csi-rbdplugin
|
||||
@ -44,7 +49,7 @@ spec:
|
||||
capabilities:
|
||||
add: ["SYS_ADMIN"]
|
||||
allowPrivilegeEscalation: true
|
||||
image: quay.io/cephcsi/rbdplugin:v0.3.0
|
||||
image: quay.io/cephcsi/rbdplugin:v1.0.0
|
||||
args:
|
||||
- "--nodeid=$(NODE_ID)"
|
||||
- "--endpoint=$(CSI_ENDPOINT)"
|
||||
@ -52,6 +57,7 @@ spec:
|
||||
- "--drivername=rbd.csi.ceph.com"
|
||||
- "--containerized=true"
|
||||
- "--metadatastorage=k8s_configmap"
|
||||
- "--configroot=k8s_objects"
|
||||
env:
|
||||
- name: HOST_ROOTFS
|
||||
value: "/rootfs"
|
||||
@ -64,14 +70,17 @@ spec:
|
||||
fieldRef:
|
||||
fieldPath: metadata.namespace
|
||||
- name: CSI_ENDPOINT
|
||||
value: unix://var/lib/kubelet/plugins/rbd.csi.ceph.com/csi.sock
|
||||
value: unix:///csi/csi.sock
|
||||
imagePullPolicy: "IfNotPresent"
|
||||
volumeMounts:
|
||||
- name: plugin-dir
|
||||
mountPath: /var/lib/kubelet/plugins/rbd.csi.ceph.com
|
||||
mountPath: /csi
|
||||
- name: pods-mount-dir
|
||||
mountPath: /var/lib/kubelet/pods
|
||||
mountPropagation: "Bidirectional"
|
||||
- name: plugin-mount-dir
|
||||
mountPath: /var/lib/kubelet/plugins/kubernetes.io/csi/volumeDevices/
|
||||
mountPropagation: "Bidirectional"
|
||||
- mountPath: /dev
|
||||
name: host-dev
|
||||
- mountPath: /rootfs
|
||||
@ -86,18 +95,18 @@ spec:
|
||||
hostPath:
|
||||
path: /var/lib/kubelet/plugins/rbd.csi.ceph.com
|
||||
type: DirectoryOrCreate
|
||||
- name: plugin-mount-dir
|
||||
hostPath:
|
||||
path: /var/lib/kubelet/plugins/kubernetes.io/csi/volumeDevices/
|
||||
type: DirectoryOrCreate
|
||||
- name: registration-dir
|
||||
hostPath:
|
||||
path: /var/lib/kubelet/plugins/
|
||||
path: /var/lib/kubelet/plugins_registry/
|
||||
type: Directory
|
||||
- name: pods-mount-dir
|
||||
hostPath:
|
||||
path: /var/lib/kubelet/pods
|
||||
type: Directory
|
||||
- name: socket-dir
|
||||
hostPath:
|
||||
path: /var/lib/kubelet/plugins/rbd.csi.ceph.com
|
||||
type: DirectoryOrCreate
|
||||
- name: host-dev
|
||||
hostPath:
|
||||
path: /dev
|
||||
|
@ -1,10 +1,15 @@
|
||||
# CSI CephFS plugin
|
||||
|
||||
The CSI CephFS plugin is able to both provision new CephFS volumes and attach and mount existing ones to workloads.
|
||||
The CSI CephFS plugin is able to both provision new CephFS volumes
|
||||
and attach and mount existing ones to workloads.
|
||||
|
||||
## Building
|
||||
|
||||
CSI CephFS plugin can be compiled in the form of a binary file or in the form of a Docker image. When compiled as a binary file, the result is stored in `_output/` directory with the name `cephfsplugin`. When compiled as an image, it's stored in the local Docker image store.
|
||||
CSI CephFS plugin can be compiled in the form of a binary file or in the form
|
||||
of a Docker image.
|
||||
When compiled as a binary file, the result is stored in `_output/`
|
||||
directory with the name `cephfsplugin`.
|
||||
When compiled as an image, it's stored in the local Docker image store.
|
||||
|
||||
Building binary:
|
||||
|
||||
@ -29,44 +34,56 @@ Option | Default value | Description
|
||||
`--nodeid` | _empty_ | This node's ID
|
||||
`--volumemounter` | _empty_ | default volume mounter. Available options are `kernel` and `fuse`. This is the mount method used if volume parameters don't specify otherwise. If left unspecified, the driver will first probe for `ceph-fuse` in system's path and will choose Ceph kernel client if probing failed.
|
||||
`--metadatastorage` | _empty_ | Whether metadata should be kept on node as file or in a k8s configmap (`node` or `k8s_configmap`)
|
||||
`--mountcachedir` | _empty_ | volume mount cache info save dir. If left unspecified, the dirver will not record mount info, or it will save mount info and when driver restart it will remount volume it cached.
|
||||
|
||||
**Available environmental variables:**
|
||||
|
||||
`KUBERNETES_CONFIG_PATH`: if you use `k8s_configmap` as metadata store, specify the path of your k8s config file (if not specified, the plugin will assume you're running it inside a k8s cluster and find the config itself).
|
||||
`KUBERNETES_CONFIG_PATH`: if you use `k8s_configmap` as metadata store, specify
|
||||
the path of your k8s config file (if not specified, the plugin will assume
|
||||
you're running it inside a k8s cluster and find the config itself).
|
||||
|
||||
`POD_NAMESPACE`: if you use `k8s_configmap` as metadata store, `POD_NAMESPACE` is used to define in which namespace you want the configmaps to be stored
|
||||
`POD_NAMESPACE`: if you use `k8s_configmap` as metadata store, `POD_NAMESPACE`
|
||||
is used to define in which namespace you want the configmaps to be stored
|
||||
|
||||
**Available volume parameters:**
|
||||
|
||||
Parameter | Required | Description
|
||||
---------------------------------------------------------------|-----------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
|
||||
`monitors` | yes | Comma separated list of Ceph monitors (e.g. `192.168.100.1:6789,192.168.100.2:6789,192.168.100.3:6789`)
|
||||
`mounter` | no | Mount method to be used for this volume. Available options are `kernel` for Ceph kernel client and `fuse` for Ceph FUSE driver. Defaults to "default mounter", see command line arguments.
|
||||
`provisionVolume` | yes | Mode of operation. BOOL value. If `true`, a new CephFS volume will be provisioned. If `false`, an existing volume will be used.
|
||||
`pool` | for `provisionVolume=true` | Ceph pool into which the volume shall be created
|
||||
`rootPath` | for `provisionVolume=false` | Root path of an existing CephFS volume
|
||||
`csiProvisionerSecretName`, `csiNodeStageSecretName` | for Kubernetes | name of the Kubernetes Secret object containing Ceph client credentials. Both parameters should have the same value
|
||||
`csiProvisionerSecretNamespace`, `csiNodeStageSecretNamespace` | for Kubernetes | namespaces of the above Secret objects
|
||||
Parameter | Required | Description
|
||||
----------------------------------------------------------------------------------------------------|--------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
|
||||
`monitors` | yes | Comma separated list of Ceph monitors (e.g. `192.168.100.1:6789,192.168.100.2:6789,192.168.100.3:6789`)
|
||||
`monValueFromSecret` | one of `monitors` and `monValueFromSecret` must be set | a string pointing the key in the credential secret, whose value is the mon. This is used for the case when the monitors' IP or hostnames are changed, the secret can be updated to pick up the new monitors. If both `monitors` and `monValueFromSecret` are set and the monitors set in the secret exists, `monValueFromSecret` takes precedence.
|
||||
`mounter` | no | Mount method to be used for this volume. Available options are `kernel` for Ceph kernel client and `fuse` for Ceph FUSE driver. Defaults to "default mounter", see command line arguments.
|
||||
`provisionVolume` | yes | Mode of operation. BOOL value. If `true`, a new CephFS volume will be provisioned. If `false`, an existing volume will be used.
|
||||
`pool` | for `provisionVolume=true` | Ceph pool into which the volume shall be created
|
||||
`rootPath` | for `provisionVolume=false` | Root path of an existing CephFS volume
|
||||
`csi.storage.k8s.io/provisioner-secret-name`, `csi.storage.k8s.io/node-stage-secret-name` | for Kubernetes | name of the Kubernetes Secret object containing Ceph client credentials. Both parameters should have the same value
|
||||
`csi.storage.k8s.io/provisioner-secret-namespace`, `csi.storage.k8s.io/node-stage-secret-namespace` | for Kubernetes | namespaces of the above Secret objects
|
||||
|
||||
**Required secrets for `provisionVolume=true`:**
|
||||
**Required secrets for `provisionVolume=true`:**
|
||||
Admin credentials are required for provisioning new volumes
|
||||
|
||||
* `adminID`: ID of an admin client
|
||||
* `adminKey`: key of the admin client
|
||||
|
||||
**Required secrets for `provisionVolume=false`:**
|
||||
**Required secrets for `provisionVolume=false`:**
|
||||
User credentials with access to an existing volume
|
||||
|
||||
* `userID`: ID of a user client
|
||||
* `userKey`: key of a user client
|
||||
|
||||
Notes on volume size: when provisioning a new volume, `max_bytes` quota attribute for this volume will be set to the requested volume size (see [Ceph quota documentation](http://docs.ceph.com/docs/mimic/cephfs/quota/)). A request for a zero-sized volume means no quota attribute will be set.
|
||||
Notes on volume size: when provisioning a new volume, `max_bytes` quota
|
||||
attribute for this volume will be set to the requested volume size (see [Ceph
|
||||
quota documentation](http://docs.ceph.com/docs/mimic/cephfs/quota/)). A request
|
||||
for a zero-sized volume means no quota attribute will be set.
|
||||
|
||||
## Deployment with Kubernetes
|
||||
|
||||
Requires Kubernetes 1.11
|
||||
Requires Kubernetes 1.13
|
||||
|
||||
Your Kubernetes cluster must allow privileged pods (i.e. `--allow-privileged` flag must be set to true for both the API server and the kubelet). Moreover, as stated in the [mount propagation docs](https://kubernetes.io/docs/concepts/storage/volumes/#mount-propagation), the Docker daemon of the cluster nodes must allow shared mounts.
|
||||
Your Kubernetes cluster must allow privileged pods (i.e. `--allow-privileged`
|
||||
flag must be set to true for both the API server and the kubelet). Moreover, as
|
||||
stated in the [mount propagation
|
||||
docs](https://kubernetes.io/docs/concepts/storage/volumes/#mount-propagation),
|
||||
the Docker daemon of the cluster nodes must allow shared mounts.
|
||||
|
||||
YAML manifests are located in `deploy/cephfs/kubernetes`.
|
||||
|
||||
@ -78,7 +95,9 @@ kubectl create -f csi-provisioner-rbac.yaml
|
||||
kubectl create -f csi-nodeplugin-rbac.yaml
|
||||
```
|
||||
|
||||
Those manifests deploy service accounts, cluster roles and cluster role bindings. These are shared for both RBD and CephFS CSI plugins, as they require the same permissions.
|
||||
Those manifests deploy service accounts, cluster roles and cluster role
|
||||
bindings. These are shared for both RBD and CephFS CSI plugins, as they require
|
||||
the same permissions.
|
||||
|
||||
**Deploy CSI sidecar containers:**
|
||||
|
||||
@ -87,7 +106,8 @@ kubectl create -f csi-cephfsplugin-attacher.yaml
|
||||
kubectl create -f csi-cephfsplugin-provisioner.yaml
|
||||
```
|
||||
|
||||
Deploys stateful sets for external-attacher and external-provisioner sidecar containers for CSI CephFS.
|
||||
Deploys stateful sets for external-attacher and external-provisioner
|
||||
sidecar containers for CSI CephFS.
|
||||
|
||||
**Deploy CSI CephFS driver:**
|
||||
|
||||
@ -95,14 +115,15 @@ Deploys stateful sets for external-attacher and external-provisioner sidecar con
|
||||
kubectl create -f csi-cephfsplugin.yaml
|
||||
```
|
||||
|
||||
Deploys a daemon set with two containers: CSI driver-registrar and the CSI CephFS driver.
|
||||
Deploys a daemon set with two containers: CSI driver-registrar and
|
||||
the CSI CephFS driver.
|
||||
|
||||
## Verifying the deployment in Kubernetes
|
||||
|
||||
After successfuly completing the steps above, you should see output similar to this:
|
||||
After successfully completing the steps above, you should see output similar to this:
|
||||
|
||||
```bash
|
||||
kubectl get all
|
||||
$ kubectl get all
|
||||
NAME READY STATUS RESTARTS AGE
|
||||
pod/csi-cephfsplugin-attacher-0 1/1 Running 0 26s
|
||||
pod/csi-cephfsplugin-provisioner-0 1/1 Running 0 25s
|
||||
@ -119,4 +140,7 @@ You can try deploying a demo pod from `examples/cephfs` to test the deployment f
|
||||
|
||||
### Notes on volume deletion
|
||||
|
||||
Volumes that were provisioned dynamically (i.e. `provisionVolume=true`) are allowed to be deleted by the driver as well, if the user chooses to do so. Otherwise, the driver is forbidden to delete such volumes - attempting to delete them is a no-op.
|
||||
Volumes that were provisioned dynamically (i.e. `provisionVolume=true`) are
|
||||
allowed to be deleted by the driver as well, if the user chooses to do
|
||||
so.Otherwise, the driver is forbidden to delete such volumes - attempting to
|
||||
delete them is a no-op.
|
||||
|
@ -1,10 +1,14 @@
|
||||
# CSI RBD Plugin
|
||||
|
||||
The RBD CSI plugin is able to provision new RBD images and attach and mount those to workloads.
|
||||
The RBD CSI plugin is able to provision new RBD images and
|
||||
attach and mount those to workloads.
|
||||
|
||||
## Building
|
||||
|
||||
CSI RBD plugin can be compiled in a form of a binary file or in a form of a Docker image. When compiled as a binary file, the result is stored in `_output/` directory with the name `rbdplugin`. When compiled as an image, it's stored in the local Docker image store.
|
||||
CSI RBD plugin can be compiled in a form of a binary file or in a form of a
|
||||
Docker image. When compiled as a binary file, the result is stored in
|
||||
`_output/` directory with the name `rbdplugin`. When compiled as an image, it's
|
||||
stored in the local Docker image store.
|
||||
|
||||
Building binary:
|
||||
|
||||
@ -22,45 +26,67 @@ make image-rbdplugin
|
||||
|
||||
**Available command line arguments:**
|
||||
|
||||
Option | Default value | Description
|
||||
--------------------|-----------------------|---------------------------------------------------------------------------------------------------
|
||||
`--endpoint` | `unix://tmp/csi.sock` | CSI endpoint, must be a UNIX socket
|
||||
`--drivername` | `rbd.csi.ceph.com` | name of the driver (Kubernetes: `provisioner` field in StorageClass must correspond to this value)
|
||||
`--nodeid` | _empty_ | This node's ID
|
||||
`--containerized` | true | Whether running in containerized mode
|
||||
`--metadatastorage` | _empty_ | Whether should metadata be kept on node as file or in a k8s configmap (`node` or `k8s_configmap`)
|
||||
Option | Default value | Description
|
||||
------ | ------------- | -----------
|
||||
`--endpoint` | `unix://tmp/csi.sock` | CSI endpoint, must be a UNIX socket
|
||||
`--drivername` | `rbd.csi.ceph.com` | name of the driver (Kubernetes: `provisioner` field in StorageClass must correspond to this value)
|
||||
`--nodeid` | _empty_ | This node's ID
|
||||
`--containerized` | true | Whether running in containerized mode
|
||||
`--metadatastorage` | _empty_ | Whether should metadata be kept on node as file or in a k8s configmap (`node` or `k8s_configmap`)
|
||||
`--configroot` | `/etc/csi-config` | Directory in which CSI specific Ceph cluster configurations are present, OR the value `k8s_objects` if present as kubernetes secrets"
|
||||
|
||||
**Available environmental variables:**
|
||||
|
||||
`HOST_ROOTFS`: rbdplugin searches `/proc` directory under the directory set by `HOST_ROOTFS`.
|
||||
|
||||
`KUBERNETES_CONFIG_PATH`: if you use `k8s_configmap` as metadata store, specify the path of your k8s config file (if not specified, the plugin will assume you're running it inside a k8s cluster and find the config itself).
|
||||
`KUBERNETES_CONFIG_PATH`: if you use `k8s_configmap` as metadata store, specify
|
||||
the path of your k8s config file (if not specified, the plugin will assume
|
||||
you're running it inside a k8s cluster and find the config itself).
|
||||
|
||||
`POD_NAMESPACE`: if you use `k8s_configmap` as metadata store, `POD_NAMESPACE` is used to define in which namespace you want the configmaps to be stored
|
||||
`POD_NAMESPACE`: if you use `k8s_configmap` as metadata store,
|
||||
`POD_NAMESPACE` is used to define in which namespace you want
|
||||
the configmaps to be stored
|
||||
|
||||
**Available volume parameters:**
|
||||
|
||||
Parameter | Required | Description
|
||||
-----------------------------------------------------------------|--------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
|
||||
`monitors` | one of `monitors` and `monValueFromSecret` must be set | Comma separated list of Ceph monitors (e.g. `192.168.100.1:6789,192.168.100.2:6789,192.168.100.3:6789`)
|
||||
`monValueFromSecret` | one of `monitors` and `monValueFromSecret` must be set | a string pointing the key in the credential secret, whose value is the mon. This is used for the case when the monitors' IP or hostnames are changed, the secret can be updated to pick up the new monitors.
|
||||
`pool` | yes | Ceph pool into which the RBD image shall be created
|
||||
`imageFormat` | no | RBD image format. Defaults to `2`. See [man pages](http://docs.ceph.com/docs/mimic/man/8/rbd/#cmdoption-rbd-image-format)
|
||||
`imageFeatures` | no | RBD image features. Available for `imageFormat=2`. CSI RBD currently supports only `layering` feature. See [man pages](http://docs.ceph.com/docs/mimic/man/8/rbd/#cmdoption-rbd-image-feature)
|
||||
`csiProvisionerSecretName`, `csiNodePublishSecretName` | for Kubernetes | name of the Kubernetes Secret object containing Ceph client credentials. Both parameters should have the same value
|
||||
`csiProvisionerSecretNamespace`, `csiNodePublishSecretNamespace` | for Kubernetes | namespaces of the above Secret objects
|
||||
`mounter` | no | if set to `rbd-nbd`, use `rbd-nbd` on nodes that have `rbd-nbd` and `nbd` kernel modules to map rbd images
|
||||
Parameter | Required | Description
|
||||
--------- | -------- | -----------
|
||||
`monitors` | one of `monitors`, `clusterID` or `monValueFromSecret` must be set | Comma separated list of Ceph monitors (e.g. `192.168.100.1:6789,192.168.100.2:6789,192.168.100.3:6789`)
|
||||
`monValueFromSecret` | one of `monitors`, `clusterID` or and `monValueFromSecret` must be set | a string pointing the key in the credential secret, whose value is the mon. This is used for the case when the monitors' IP or hostnames are changed, the secret can be updated to pick up the new monitors.
|
||||
`clusterID` | one of `monitors`, `clusterID` or `monValueFromSecret` must be set | String representing a Ceph cluster, must be unique across all Ceph clusters in use for provisioning, cannot be greater than 36 bytes in length, and should remain immutable for the lifetime of the Ceph cluster in use
|
||||
`pool` | yes | Ceph pool into which the RBD image shall be created
|
||||
`imageFormat` | no | RBD image format. Defaults to `2`. See [man pages](http://docs.ceph.com/docs/mimic/man/8/rbd/#cmdoption-rbd-image-format)
|
||||
`imageFeatures` | no | RBD image features. Available for `imageFormat=2`. CSI RBD currently supports only `layering` feature. See [man pages](http://docs.ceph.com/docs/mimic/man/8/rbd/#cmdoption-rbd-image-feature)
|
||||
`csi.storage.k8s.io/provisioner-secret-name`, `csi.storage.k8s.io/node-publish-secret-name` | for Kubernetes | name of the Kubernetes Secret object containing Ceph client credentials. Both parameters should have the same value
|
||||
`csi.storage.k8s.io/provisioner-secret-namespace`, `csi.storage.k8s.io/node-publish-secret-namespace` | for Kubernetes | namespaces of the above Secret objects
|
||||
`mounter`| no | if set to `rbd-nbd`, use `rbd-nbd` on nodes that have `rbd-nbd` and `nbd` kernel modules to map rbd images
|
||||
|
||||
**Required secrets:**
|
||||
Admin credentials are required for provisioning new RBD images
|
||||
`ADMIN_NAME`: `ADMIN_PASSWORD` - note that the key of the key-value pair is the name of the client with admin privileges, and the value is its password
|
||||
NOTE: If `clusterID` parameter is used, then an accompanying Ceph cluster
|
||||
configuration secret or config files needs to be provided to the running pods.
|
||||
Refer to [Cluster ID based configuration](../examples/README.md#cluster-id-based-configuration)
|
||||
for more information. A suggested way to populate the clusterID is to use the
|
||||
output of `ceph fsid` of the Ceph cluster to be used for provisioning.
|
||||
|
||||
**Required secrets:**
|
||||
|
||||
Admin credentials are required for provisioning new RBD images `ADMIN_NAME`:
|
||||
`ADMIN_PASSWORD` - note that the key of the key-value pair is the name of the
|
||||
client with admin privileges, and the value is its password
|
||||
|
||||
If clusterID is specified, then a secret with various keys and values as
|
||||
specified in `examples/rbd/template-ceph-cluster-ID-secret.yaml` needs to be
|
||||
created, with the secret name matching the string value provided as the
|
||||
`clusterID`.
|
||||
|
||||
## Deployment with Kubernetes
|
||||
|
||||
Requires Kubernetes 1.11
|
||||
|
||||
Your Kubernetes cluster must allow privileged pods (i.e. `--allow-privileged` flag must be set to true for both the API server and the kubelet). Moreover, as stated in the [mount propagation docs](https://kubernetes.io/docs/concepts/storage/volumes/#mount-propagation), the Docker daemon of the cluster nodes must allow shared mounts.
|
||||
Your Kubernetes cluster must allow privileged pods (i.e. `--allow-privileged`
|
||||
flag must be set to true for both the API server and the kubelet). Moreover, as
|
||||
stated in the [mount propagation
|
||||
docs](https://kubernetes.io/docs/concepts/storage/volumes/#mount-propagation),
|
||||
the Docker daemon of the cluster nodes must allow shared mounts.
|
||||
|
||||
YAML manifests are located in `deploy/rbd/kubernetes`.
|
||||
|
||||
@ -72,7 +98,9 @@ kubectl create -f csi-provisioner-rbac.yaml
|
||||
kubectl create -f csi-nodeplugin-rbac.yaml
|
||||
```
|
||||
|
||||
Those manifests deploy service accounts, cluster roles and cluster role bindings. These are shared for both RBD and CephFS CSI plugins, as they require the same permissions.
|
||||
Those manifests deploy service accounts, cluster roles and cluster role
|
||||
bindings. These are shared for both RBD and CephFS CSI plugins, as they require
|
||||
the same permissions.
|
||||
|
||||
**Deploy CSI sidecar containers:**
|
||||
|
||||
@ -81,7 +109,8 @@ kubectl create -f csi-rbdplugin-attacher.yaml
|
||||
kubectl create -f csi-rbdplugin-provisioner.yaml
|
||||
```
|
||||
|
||||
Deploys stateful sets for external-attacher and external-provisioner sidecar containers for CSI RBD.
|
||||
Deploys stateful sets for external-attacher and external-provisioner
|
||||
sidecar containers for CSI RBD.
|
||||
|
||||
**Deploy RBD CSI driver:**
|
||||
|
||||
@ -93,7 +122,7 @@ Deploys a daemon set with two containers: CSI driver-registrar and the CSI RBD d
|
||||
|
||||
## Verifying the deployment in Kubernetes
|
||||
|
||||
After successfuly completing the steps above, you should see output similar to this:
|
||||
After successfully completing the steps above, you should see output similar to this:
|
||||
|
||||
```bash
|
||||
$ kubectl get all
|
||||
|
@ -1,10 +1,24 @@
|
||||
# How to test RBD and CephFS plugins with Kubernetes 1.11
|
||||
# How to test RBD and CephFS plugins with Kubernetes 1.13
|
||||
|
||||
Both `rbd` and `cephfs` directories contain `plugin-deploy.sh` and `plugin-teardown.sh` helper scripts. You can use those to help you deploy/tear down RBACs, sidecar containers and the plugin in one go. By default, they look for the YAML manifests in `../../deploy/{rbd,cephfs}/kubernetes`. You can override this path by running `$ ./plugin-deploy.sh /path/to/my/manifests`.
|
||||
Both `rbd` and `cephfs` directories contain `plugin-deploy.sh` and
|
||||
`plugin-teardown.sh` helper scripts. You can use those to help you
|
||||
deploy/teardown RBACs, sidecar containers and the plugin in one go.
|
||||
By default, they look for the YAML manifests in
|
||||
`../../deploy/{rbd,cephfs}/kubernetes`.
|
||||
You can override this path by running `$ ./plugin-deploy.sh /path/to/my/manifests`.
|
||||
|
||||
Once the plugin is successfully deployed, you'll need to customize `storageclass.yaml` and `secret.yaml` manifests to reflect your Ceph cluster setup. Please consult the documentation for info about available parameters.
|
||||
Once the plugin is successfully deployed, you'll need to customize
|
||||
`storageclass.yaml` and `secret.yaml` manifests to reflect your Ceph cluster
|
||||
setup.
|
||||
Please consult the documentation for info about available parameters.
|
||||
|
||||
After configuring the secrets, monitors, etc. you can deploy a testing Pod mounting a RBD image / CephFS volume:
|
||||
**NOTE:** See section
|
||||
[Cluster ID based configuration](#cluster-id-based-configuration) if using
|
||||
the `clusterID` instead of `monitors` or `monValueFromSecret` option in the
|
||||
storage class for RBD based provisioning before proceeding.
|
||||
|
||||
After configuring the secrets, monitors, etc. you can deploy a
|
||||
testing Pod mounting a RBD image / CephFS volume:
|
||||
|
||||
```bash
|
||||
kubectl create -f secret.yaml
|
||||
@ -20,42 +34,221 @@ Other helper scripts:
|
||||
|
||||
## How to test RBD Snapshot feature
|
||||
|
||||
Before continuing, make sure you enabled the required [feature gate](https://kubernetes-csi.github.io/docs/snapshot-restore-feature.html#snapshot-apis) in your Kubernetes cluster.
|
||||
Before continuing, make sure you enabled the required
|
||||
feature gate `VolumeSnapshotDataSource=true` in your Kubernetes cluster.
|
||||
|
||||
In the `examples/rbd` directory you will find four files related to snapshots: [csi-snapshotter-rbac.yaml](./rbd/csi-snapshotter-rbac.yaml), [csi-snapshotter.yaml](./rbd/csi-snapshotter.yaml), [snapshotclass.yaml](./rbd/snapshotclass.yaml) and [snapshot.yaml](./rbd/snapshot.yaml).
|
||||
In the `examples/rbd` directory you will find two files related to snapshots:
|
||||
[snapshotclass.yaml](./rbd/snapshotclass.yaml) and
|
||||
[snapshot.yaml](./rbd/snapshot.yaml).
|
||||
|
||||
Once you created your RBD volume, you'll need to customize at least `snapshotclass.yaml` and make sure the `monitors` and `pool` parameters match your Ceph cluster setup. If you followed the documentation to create the rbdplugin, you shouldn't have to edit any other file. If you didn't, make sure every parameters in `csi-snapshotter.yaml` reflect your configuration.
|
||||
Once you created your RBD volume, you'll need to customize at least
|
||||
`snapshotclass.yaml` and make sure the `monitors` and `pool` parameters match
|
||||
your Ceph cluster setup.
|
||||
If you followed the documentation to create the rbdplugin, you shouldn't
|
||||
have to edit any other file.
|
||||
|
||||
After configuring everything you needed, deploy the csi-snapshotter:
|
||||
After configuring everything you needed, deploy the snapshot class:
|
||||
|
||||
```bash
|
||||
kubectl create -f csi-snapshotter-rbac.yaml
|
||||
kubectl create -f csi-snapshotter.yaml
|
||||
kubectl create -f snapshotclass.yaml
|
||||
```
|
||||
|
||||
Verify that the snapshot class was created:
|
||||
|
||||
```console
|
||||
$ kubectl get volumesnapshotclass
|
||||
NAME AGE
|
||||
csi-rbdplugin-snapclass 4s
|
||||
```
|
||||
|
||||
Create a snapshot from the existing PVC:
|
||||
|
||||
```bash
|
||||
kubectl create -f snapshot.yaml
|
||||
```
|
||||
|
||||
To verify if your volume snapshot has successfully been created, run the following:
|
||||
|
||||
```bash
|
||||
$ kubectl get volumesnapshotclass
|
||||
NAME AGE
|
||||
csi-rbdplugin-snapclass 4s
|
||||
```console
|
||||
$ kubectl get volumesnapshot
|
||||
NAME AGE
|
||||
rbd-pvc-snapshot 6s
|
||||
```
|
||||
|
||||
To be sure everything is OK you can run `rbd snap ls [your-pvc-name]` inside one of your Ceph pod.
|
||||
|
||||
## How to test RBD Block Volume
|
||||
|
||||
Before continuing, make sure you enabled the required [feature gate](https://kubernetes-csi.github.io/docs/raw-block.html#usage) in your Kubernetes cluster.
|
||||
|
||||
In order to create a PVC in Block mode, you need to specify `volumeMode: Block` in the PVC spec.
|
||||
In the `examples/rbd` directory, you can test RBD block volume by running:
|
||||
To check the status of the snapshot, run the following:
|
||||
|
||||
```bash
|
||||
kubectl create -f pvc-block.yaml
|
||||
kubectl create -f pod-block.yaml
|
||||
$ kubectl describe volumesnapshot rbd-pvc-snapshot
|
||||
|
||||
Name: rbd-pvc-snapshot
|
||||
Namespace: default
|
||||
Labels: <none>
|
||||
Annotations: <none>
|
||||
API Version: snapshot.storage.k8s.io/v1alpha1
|
||||
Kind: VolumeSnapshot
|
||||
Metadata:
|
||||
Creation Timestamp: 2019-02-06T08:52:34Z
|
||||
Finalizers:
|
||||
snapshot.storage.kubernetes.io/volumesnapshot-protection
|
||||
Generation: 5
|
||||
Resource Version: 84239
|
||||
Self Link: /apis/snapshot.storage.k8s.io/v1alpha1/namespaces/default/volumesnapshots/rbd-pvc-snapshot
|
||||
UID: 8b9b5740-29ec-11e9-8e0f-b8ca3aad030b
|
||||
Spec:
|
||||
Snapshot Class Name: csi-rbdplugin-snapclass
|
||||
Snapshot Content Name: snapcontent-8b9b5740-29ec-11e9-8e0f-b8ca3aad030b
|
||||
Source:
|
||||
API Group: <nil>
|
||||
Kind: PersistentVolumeClaim
|
||||
Name: rbd-pvc
|
||||
Status:
|
||||
Creation Time: 2019-02-06T08:52:34Z
|
||||
Ready To Use: true
|
||||
Restore Size: 1Gi
|
||||
Events: <none>
|
||||
```
|
||||
|
||||
To be sure everything is OK you can run `rbd snap ls [your-pvc-name]` inside
|
||||
one of your Ceph pod.
|
||||
|
||||
To restore the snapshot to a new PVC, deploy
|
||||
[pvc-restore.yaml](./rbd/pvc-restore.yaml) and a testing pod
|
||||
[pod-restore.yaml](./rbd/pod-restore.yaml):
|
||||
|
||||
```bash
|
||||
kubectl create -f pvc-restore.yaml
|
||||
kubectl create -f pod-restore.yaml
|
||||
```
|
||||
|
||||
## How to test RBD MULTI_NODE_MULTI_WRITER BLOCK feature
|
||||
|
||||
Requires feature-gates: `BlockVolume=true` `CSIBlockVolume=true`
|
||||
|
||||
*NOTE* The MULTI_NODE_MULTI_WRITER capability is only available for
|
||||
Volumes that are of access_type `block`
|
||||
|
||||
*WARNING* This feature is strictly for workloads that know how to deal
|
||||
with concurrent access to the Volume (eg Active/Passive applications).
|
||||
Using RWX modes on non clustered file systems with applications trying
|
||||
to simultaneously access the Volume will likely result in data corruption!
|
||||
|
||||
Following are examples for issuing a request for a `Block`
|
||||
`ReadWriteMany` Claim, and using the resultant Claim for a POD
|
||||
|
||||
```yaml
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: block-pvc
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteMany
|
||||
volumeMode: Block
|
||||
resources:
|
||||
requests:
|
||||
storage: 1Gi
|
||||
storageClassName: csi-rbd
|
||||
```
|
||||
|
||||
Create a POD that uses this PVC:
|
||||
|
||||
```yaml
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: my-pod
|
||||
spec:
|
||||
containers:
|
||||
- name: my-container
|
||||
image: debian
|
||||
command: ["/bin/bash", "-c"]
|
||||
args: [ "tail -f /dev/null" ]
|
||||
volumeDevices:
|
||||
- devicePath: /dev/rbdblock
|
||||
name: my-volume
|
||||
imagePullPolicy: IfNotPresent
|
||||
volumes:
|
||||
- name: my-volume
|
||||
persistentVolumeClaim:
|
||||
claimName: block-pvc
|
||||
|
||||
```
|
||||
|
||||
Now, we can create a second POD (ensure the POD is scheduled on a different
|
||||
node; multiwriter single node works without this feature) that also uses this
|
||||
PVC at the same time, again wait for the pod to enter running state, and verify
|
||||
the block device is available.
|
||||
|
||||
```yaml
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: another-pod
|
||||
spec:
|
||||
containers:
|
||||
- name: my-container
|
||||
image: debian
|
||||
command: ["/bin/bash", "-c"]
|
||||
args: [ "tail -f /dev/null" ]
|
||||
volumeDevices:
|
||||
- devicePath: /dev/rbdblock
|
||||
name: my-volume
|
||||
imagePullPolicy: IfNotPresent
|
||||
volumes:
|
||||
- name: my-volume
|
||||
persistentVolumeClaim:
|
||||
claimName: block-pvc
|
||||
```
|
||||
|
||||
Wait for the PODs to enter Running state, check that our block device
|
||||
is available in the container at `/dev/rdbblock` in both containers:
|
||||
|
||||
```bash
|
||||
$ kubectl exec -it my-pod -- fdisk -l /dev/rbdblock
|
||||
Disk /dev/rbdblock: 1 GiB, 1073741824 bytes, 2097152 sectors
|
||||
Units: sectors of 1 * 512 = 512 bytes
|
||||
Sector size (logical/physical): 512 bytes / 512 bytes
|
||||
I/O size (minimum/optimal): 4194304 bytes / 4194304 bytes
|
||||
```
|
||||
|
||||
```bash
|
||||
$ kubectl exec -it another-pod -- fdisk -l /dev/rbdblock
|
||||
Disk /dev/rbdblock: 1 GiB, 1073741824 bytes, 2097152 sectors
|
||||
Units: sectors of 1 * 512 = 512 bytes
|
||||
Sector size (logical/physical): 512 bytes / 512 bytes
|
||||
I/O size (minimum/optimal): 4194304 bytes / 4194304 bytes
|
||||
```
|
||||
|
||||
## Cluster ID based configuration
|
||||
|
||||
Before creating a storage class that uses the option `clusterID` to refer to a
|
||||
Ceph cluster, the following actions need to be completed.
|
||||
|
||||
Get the following information from the Ceph cluster,
|
||||
|
||||
* Admin ID and key, that has privileges to perform CRUD operations on the Ceph
|
||||
cluster and pools of choice
|
||||
* Key is typically the output of, `ceph auth get-key client.admin` where
|
||||
`admin` is the Admin ID
|
||||
* Used to substitute admin/user id and key values in the files below
|
||||
* Ceph monitor list
|
||||
* Typically in the output of `ceph mon dump`
|
||||
* Used to prepare comma separated MON list where required in the files below
|
||||
* Ceph Cluster fsid
|
||||
* If choosing to use the Ceph cluster fsid as the unique value of clusterID,
|
||||
* Output of `ceph fsid`
|
||||
* Used to substitute `<cluster-id>` references in the files below
|
||||
|
||||
Update the template
|
||||
[template-ceph-cluster-ID-secret.yaml](./rbd/template-ceph-cluster-ID-secret.yaml)
|
||||
with values from
|
||||
a Ceph cluster and replace `<cluster-id>` with the chosen clusterID to create
|
||||
the following secret,
|
||||
|
||||
* `kubectl create -f rbd/template-ceph-cluster-ID-secret.yaml`
|
||||
|
||||
Storage class and snapshot class, using `<cluster-id>` as the value for the
|
||||
option `clusterID`, can now be created on the cluster.
|
||||
|
||||
Remaining steps to test functionality remains the same as mentioned in the
|
||||
sections above.
|
||||
|
@ -1,3 +1,4 @@
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
@ -15,13 +16,13 @@ spec:
|
||||
app: web-server
|
||||
spec:
|
||||
containers:
|
||||
- name: web-server
|
||||
image: nginx
|
||||
volumeMounts:
|
||||
- name: mypvc
|
||||
mountPath: /var/lib/www/html
|
||||
- name: web-server
|
||||
image: nginx
|
||||
volumeMounts:
|
||||
- name: mypvc
|
||||
mountPath: /var/lib/www/html
|
||||
volumes:
|
||||
- name: mypvc
|
||||
persistentVolumeClaim:
|
||||
claimName: csi-cephfs-pvc
|
||||
readOnly: false
|
||||
- name: mypvc
|
||||
persistentVolumeClaim:
|
||||
claimName: csi-cephfs-pvc
|
||||
readOnly: false
|
||||
|
@ -4,7 +4,7 @@ CONTAINER_NAME=csi-cephfsplugin
|
||||
POD_NAME=$(kubectl get pods -l app=$CONTAINER_NAME -o=name | head -n 1)
|
||||
|
||||
function get_pod_status() {
|
||||
echo -n $(kubectl get $POD_NAME -o jsonpath="{.status.phase}")
|
||||
echo -n "$(kubectl get "$POD_NAME" -o jsonpath="{.status.phase}")"
|
||||
}
|
||||
|
||||
while [[ "$(get_pod_status)" != "Running" ]]; do
|
||||
@ -12,4 +12,4 @@ while [[ "$(get_pod_status)" != "Running" ]]; do
|
||||
echo "Waiting for $POD_NAME (status $(get_pod_status))"
|
||||
done
|
||||
|
||||
kubectl exec -it ${POD_NAME#*/} -c $CONTAINER_NAME bash
|
||||
kubectl exec -it "${POD_NAME#*/}" -c "$CONTAINER_NAME" bash
|
||||
|
@ -4,7 +4,7 @@ CONTAINER_NAME=csi-cephfsplugin
|
||||
POD_NAME=$(kubectl get pods -l app=$CONTAINER_NAME -o=name | head -n 1)
|
||||
|
||||
function get_pod_status() {
|
||||
echo -n $(kubectl get $POD_NAME -o jsonpath="{.status.phase}")
|
||||
echo -n "$(kubectl get "$POD_NAME" -o jsonpath="{.status.phase}")"
|
||||
}
|
||||
|
||||
while [[ "$(get_pod_status)" != "Running" ]]; do
|
||||
@ -12,4 +12,4 @@ while [[ "$(get_pod_status)" != "Running" ]]; do
|
||||
echo "Waiting for $POD_NAME (status $(get_pod_status))"
|
||||
done
|
||||
|
||||
kubectl logs -f $POD_NAME -c $CONTAINER_NAME
|
||||
kubectl logs -f "$POD_NAME" -c "$CONTAINER_NAME"
|
||||
|
@ -10,6 +10,6 @@ cd "$deployment_base" || exit 1
|
||||
|
||||
objects=(csi-attacher-rbac csi-provisioner-rbac csi-nodeplugin-rbac csi-cephfsplugin-attacher csi-cephfsplugin-provisioner csi-cephfsplugin)
|
||||
|
||||
for obj in ${objects[@]}; do
|
||||
for obj in "${objects[@]}"; do
|
||||
kubectl create -f "./$obj.yaml"
|
||||
done
|
||||
|
@ -10,6 +10,6 @@ cd "$deployment_base" || exit 1
|
||||
|
||||
objects=(csi-cephfsplugin-attacher csi-cephfsplugin-provisioner csi-cephfsplugin csi-attacher-rbac csi-provisioner-rbac csi-nodeplugin-rbac)
|
||||
|
||||
for obj in ${objects[@]}; do
|
||||
for obj in "${objects[@]}"; do
|
||||
kubectl delete -f "./$obj.yaml"
|
||||
done
|
||||
|
@ -1,17 +1,17 @@
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: csicephfs-demo-pod
|
||||
spec:
|
||||
containers:
|
||||
- name: web-server
|
||||
image: nginx
|
||||
volumeMounts:
|
||||
- name: mypvc
|
||||
mountPath: /var/lib/www
|
||||
- name: web-server
|
||||
image: nginx
|
||||
volumeMounts:
|
||||
- name: mypvc
|
||||
mountPath: /var/lib/www
|
||||
volumes:
|
||||
- name: mypvc
|
||||
persistentVolumeClaim:
|
||||
claimName: csi-cephfs-pvc
|
||||
readOnly: false
|
||||
|
||||
- name: mypvc
|
||||
persistentVolumeClaim:
|
||||
claimName: csi-cephfs-pvc
|
||||
readOnly: false
|
||||
|
@ -1,10 +1,11 @@
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: csi-cephfs-pvc
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteMany
|
||||
- ReadWriteMany
|
||||
resources:
|
||||
requests:
|
||||
storage: 5Gi
|
||||
|
@ -1,3 +1,4 @@
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
|
@ -1,11 +1,12 @@
|
||||
---
|
||||
apiVersion: storage.k8s.io/v1
|
||||
kind: StorageClass
|
||||
metadata:
|
||||
name: csi-cephfs
|
||||
provisioner: csi-cephfsplugin
|
||||
provisioner: cephfs.csi.ceph.com
|
||||
parameters:
|
||||
# Comma separated list of Ceph monitors
|
||||
# if using FQDN, make sure csi plugin's dns policy is appropriate.
|
||||
# if using FQDN, make sure csi plugin's dns policy is appropriate.
|
||||
monitors: mon1:port,mon2:port,...
|
||||
|
||||
# For provisionVolume: "true":
|
||||
@ -13,7 +14,8 @@ parameters:
|
||||
# Requires admin credentials (adminID, adminKey).
|
||||
# For provisionVolume: "false":
|
||||
# It is assumed the volume already exists and the user is expected
|
||||
# to provide path to that volume (rootPath) and user credentials (userID, userKey).
|
||||
# to provide path to that volume (rootPath) and user credentials
|
||||
# (userID, userKey).
|
||||
provisionVolume: "true"
|
||||
|
||||
# Ceph pool into which the volume shall be created
|
||||
@ -25,13 +27,16 @@ parameters:
|
||||
# rootPath: /absolute/path
|
||||
|
||||
# The secrets have to contain user and/or Ceph admin credentials.
|
||||
csiProvisionerSecretName: csi-cephfs-secret
|
||||
csiProvisionerSecretNamespace: default
|
||||
csiNodeStageSecretName: csi-cephfs-secret
|
||||
csiNodeStageSecretNamespace: default
|
||||
csi.storage.k8s.io/provisioner-secret-name: csi-cephfs-secret
|
||||
csi.storage.k8s.io/provisioner-secret-namespace: default
|
||||
csi.storage.k8s.io/node-stage-secret-name: csi-cephfs-secret
|
||||
csi.storage.k8s.io/node-stage-secret-namespace: default
|
||||
|
||||
# (optional) The driver can use either ceph-fuse (fuse) or ceph kernel client (kernel)
|
||||
# If omitted, default volume mounter will be used - this is determined by probing for ceph-fuse
|
||||
# or by setting the default mounter explicitly via --volumemounter command-line argument.
|
||||
# (optional) The driver can use either ceph-fuse (fuse) or
|
||||
# ceph kernelclient (kernel).
|
||||
# If omitted, default volume mounter will be used - this is
|
||||
# determined by probing for ceph-fuse
|
||||
# or by setting the default mounter explicitly via
|
||||
# --volumemounter command-line argument.
|
||||
# mounter: kernel
|
||||
reclaimPolicy: Delete
|
||||
|
@ -1,45 +0,0 @@
|
||||
kind: ClusterRole
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: external-snapshotter-runner
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["persistentvolumes"]
|
||||
verbs: ["get", "list", "watch", "create", "delete"]
|
||||
- apiGroups: [""]
|
||||
resources: ["persistentvolumeclaims"]
|
||||
verbs: ["get", "list", "watch", "update"]
|
||||
- apiGroups: ["storage.k8s.io"]
|
||||
resources: ["storageclasses"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["events"]
|
||||
verbs: ["list", "watch", "create", "update", "patch"]
|
||||
- apiGroups: [""]
|
||||
resources: ["secrets"]
|
||||
verbs: ["get", "list"]
|
||||
- apiGroups: ["snapshot.storage.k8s.io"]
|
||||
resources: ["volumesnapshotclasses"]
|
||||
verbs: ["get", "list", "watch"]
|
||||
- apiGroups: ["snapshot.storage.k8s.io"]
|
||||
resources: ["volumesnapshotcontents"]
|
||||
verbs: ["create", "get", "list", "watch", "update", "delete"]
|
||||
- apiGroups: ["snapshot.storage.k8s.io"]
|
||||
resources: ["volumesnapshots"]
|
||||
verbs: ["get", "list", "watch", "update"]
|
||||
- apiGroups: ["apiextensions.k8s.io"]
|
||||
resources: ["customresourcedefinitions"]
|
||||
verbs: ["create"]
|
||||
---
|
||||
kind: ClusterRoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
metadata:
|
||||
name: csi-snapshotter-role
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: csi-snapshotter
|
||||
namespace: default
|
||||
roleRef:
|
||||
kind: ClusterRole
|
||||
name: external-snapshotter-runner
|
||||
apiGroup: rbac.authorization.k8s.io
|
@ -1,56 +0,0 @@
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: csi-snapshotter
|
||||
---
|
||||
kind: Service
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: csi-snapshotter
|
||||
labels:
|
||||
app: csi-snapshotter
|
||||
spec:
|
||||
selector:
|
||||
app: csi-snapshotter
|
||||
ports:
|
||||
- name: dummy
|
||||
port: 12345
|
||||
---
|
||||
kind: StatefulSet
|
||||
apiVersion: apps/v1
|
||||
metadata:
|
||||
name: csi-snapshotter
|
||||
spec:
|
||||
serviceName: "csi-snapshotter"
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: csi-snapshotter
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: csi-snapshotter
|
||||
spec:
|
||||
serviceAccount: csi-snapshotter
|
||||
containers:
|
||||
- name: csi-snapshotter
|
||||
image: quay.io/k8scsi/csi-snapshotter:v0.4.0
|
||||
args:
|
||||
- "--csi-address=$(ADDRESS)"
|
||||
- "--connection-timeout=15s"
|
||||
- "--v=5"
|
||||
env:
|
||||
- name: ADDRESS
|
||||
value: /csi/csi.sock
|
||||
imagePullPolicy: Always
|
||||
securityContext:
|
||||
privileged: true
|
||||
volumeMounts:
|
||||
- name: socket-dir
|
||||
mountPath: /csi
|
||||
imagePullPolicy: Always
|
||||
volumes:
|
||||
- hostPath:
|
||||
path: /var/lib/kubelet/plugins/rbd.csi.ceph.com
|
||||
type: DirectoryOrCreate
|
||||
name: socket-dir
|
@ -4,7 +4,7 @@ CONTAINER_NAME=csi-rbdplugin
|
||||
POD_NAME=$(kubectl get pods -l app=$CONTAINER_NAME -o=name | head -n 1)
|
||||
|
||||
function get_pod_status() {
|
||||
echo -n $(kubectl get $POD_NAME -o jsonpath="{.status.phase}")
|
||||
echo -n "$(kubectl get "$POD_NAME" -o jsonpath="{.status.phase}")"
|
||||
}
|
||||
|
||||
while [[ "$(get_pod_status)" != "Running" ]]; do
|
||||
@ -12,4 +12,4 @@ while [[ "$(get_pod_status)" != "Running" ]]; do
|
||||
echo "Waiting for $POD_NAME (status $(get_pod_status))"
|
||||
done
|
||||
|
||||
kubectl exec -it ${POD_NAME#*/} -c $CONTAINER_NAME bash
|
||||
kubectl exec -it "${POD_NAME#*/}" -c "$CONTAINER_NAME" bash
|
||||
|
@ -4,7 +4,7 @@ CONTAINER_NAME=csi-rbdplugin
|
||||
POD_NAME=$(kubectl get pods -l app=$CONTAINER_NAME -o=name | head -n 1)
|
||||
|
||||
function get_pod_status() {
|
||||
echo -n $(kubectl get $POD_NAME -o jsonpath="{.status.phase}")
|
||||
echo -n "$(kubectl get "$POD_NAME" -o jsonpath="{.status.phase}")"
|
||||
}
|
||||
|
||||
while [[ "$(get_pod_status)" != "Running" ]]; do
|
||||
@ -12,4 +12,4 @@ while [[ "$(get_pod_status)" != "Running" ]]; do
|
||||
echo "Waiting for $POD_NAME (status $(get_pod_status))"
|
||||
done
|
||||
|
||||
kubectl logs -f $POD_NAME -c $CONTAINER_NAME
|
||||
kubectl logs -f "$POD_NAME" -c "$CONTAINER_NAME"
|
||||
|
@ -10,6 +10,6 @@ cd "$deployment_base" || exit 1
|
||||
|
||||
objects=(csi-attacher-rbac csi-provisioner-rbac csi-nodeplugin-rbac csi-rbdplugin-attacher csi-rbdplugin-provisioner csi-rbdplugin)
|
||||
|
||||
for obj in ${objects[@]}; do
|
||||
for obj in "${objects[@]}"; do
|
||||
kubectl create -f "./$obj.yaml"
|
||||
done
|
||||
|
@ -10,6 +10,6 @@ cd "$deployment_base" || exit 1
|
||||
|
||||
objects=(csi-rbdplugin-attacher csi-rbdplugin-provisioner csi-rbdplugin csi-attacher-rbac csi-provisioner-rbac csi-nodeplugin-rbac)
|
||||
|
||||
for obj in ${objects[@]}; do
|
||||
for obj in "${objects[@]}"; do
|
||||
kubectl delete -f "./$obj.yaml"
|
||||
done
|
||||
|
@ -1,16 +0,0 @@
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: csirbd-block-pod
|
||||
spec:
|
||||
containers:
|
||||
- name: web-server
|
||||
image: nginx
|
||||
volumeDevices:
|
||||
- name: data
|
||||
devicePath: /dev/vda
|
||||
volumes:
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: rbd-block-pvc
|
||||
readOnly: false
|
17
examples/rbd/pod-restore.yaml
Normal file
17
examples/rbd/pod-restore.yaml
Normal file
@ -0,0 +1,17 @@
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: csirbd-restore-demo-pod
|
||||
spec:
|
||||
containers:
|
||||
- name: web-server
|
||||
image: nginx
|
||||
volumeMounts:
|
||||
- name: mypvc
|
||||
mountPath: /var/lib/www/html
|
||||
volumes:
|
||||
- name: mypvc
|
||||
persistentVolumeClaim:
|
||||
claimName: rbd-pvc-restore
|
||||
readOnly: false
|
@ -1,17 +1,17 @@
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: csirbd-demo-pod
|
||||
spec:
|
||||
containers:
|
||||
- name: web-server
|
||||
image: nginx
|
||||
volumeMounts:
|
||||
- name: mypvc
|
||||
mountPath: /var/lib/www/html
|
||||
- name: web-server
|
||||
image: nginx
|
||||
volumeMounts:
|
||||
- name: mypvc
|
||||
mountPath: /var/lib/www/html
|
||||
volumes:
|
||||
- name: mypvc
|
||||
persistentVolumeClaim:
|
||||
claimName: rbd-pvc
|
||||
readOnly: false
|
||||
|
||||
- name: mypvc
|
||||
persistentVolumeClaim:
|
||||
claimName: rbd-pvc
|
||||
readOnly: false
|
||||
|
16
examples/rbd/pvc-restore.yaml
Normal file
16
examples/rbd/pvc-restore.yaml
Normal file
@ -0,0 +1,16 @@
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: rbd-pvc-restore
|
||||
spec:
|
||||
storageClassName: csi-rbd
|
||||
dataSource:
|
||||
name: rbd-pvc-snapshot
|
||||
kind: VolumeSnapshot
|
||||
apiGroup: snapshot.storage.k8s.io
|
||||
accessModes:
|
||||
- ReadWriteMany
|
||||
resources:
|
||||
requests:
|
||||
storage: 1Gi
|
@ -1,10 +1,11 @@
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: rbd-pvc
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 1Gi
|
||||
|
18
examples/rbd/raw-block-pod.yaml
Normal file
18
examples/rbd/raw-block-pod.yaml
Normal file
@ -0,0 +1,18 @@
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: pod-with-raw-block-volume
|
||||
spec:
|
||||
containers:
|
||||
- name: fc-container
|
||||
image: fedora:26
|
||||
command: ["/bin/sh", "-c"]
|
||||
args: ["tail -f /dev/null"]
|
||||
volumeDevices:
|
||||
- name: data
|
||||
devicePath: /dev/xvda
|
||||
volumes:
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: raw-block-pvc
|
@ -1,11 +1,12 @@
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: rbd-block-pvc
|
||||
name: raw-block-pvc
|
||||
spec:
|
||||
volumeMode: Block
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
- ReadWriteOnce
|
||||
volumeMode: Block
|
||||
resources:
|
||||
requests:
|
||||
storage: 1Gi
|
@ -1,8 +1,9 @@
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
name: csi-rbd-secret
|
||||
namespace: default
|
||||
namespace: default
|
||||
data:
|
||||
# Key value corresponds to a user name defined in ceph cluster
|
||||
admin: BASE64-ENCODED-PASSWORD
|
||||
@ -10,4 +11,4 @@ data:
|
||||
kubernetes: BASE64-ENCODED-PASSWORD
|
||||
# if monValueFromSecret is set to "monitors", uncomment the
|
||||
# following and set the mon there
|
||||
#monitors: BASE64-ENCODED-Comma-Delimited-Mons
|
||||
# monitors: BASE64-ENCODED-Comma-Delimited-Mons
|
||||
|
@ -1,3 +1,4 @@
|
||||
---
|
||||
apiVersion: snapshot.storage.k8s.io/v1alpha1
|
||||
kind: VolumeSnapshot
|
||||
metadata:
|
||||
|
@ -1,10 +1,23 @@
|
||||
---
|
||||
apiVersion: snapshot.storage.k8s.io/v1alpha1
|
||||
kind: VolumeSnapshotClass
|
||||
metadata:
|
||||
name: csi-rbdplugin-snapclass
|
||||
snapshotter: csi-rbdplugin
|
||||
snapshotter: rbd.csi.ceph.com
|
||||
parameters:
|
||||
pool: rbd
|
||||
# Comma separated list of Ceph monitors
|
||||
# if using FQDN, make sure csi plugin's dns policy is appropriate.
|
||||
monitors: mon1:port,mon2:port,...
|
||||
csiSnapshotterSecretName: csi-rbd-secret
|
||||
csiSnapshotterSecretNamespace: default
|
||||
# OR,
|
||||
# String representing a Ceph cluster to provision storage from.
|
||||
# Should be unique across all Ceph clusters in use for provisioning,
|
||||
# cannot be greater than 36 bytes in length, and should remain immutable for
|
||||
# the lifetime of the StorageClass in use.
|
||||
# If using clusterID, ensure to create a secret, as in
|
||||
# template-ceph-cluster-ID-secret.yaml, to accompany the string chosen to
|
||||
# represent the Ceph cluster in clusterID
|
||||
# clusterID: <cluster-id>
|
||||
|
||||
csi.storage.k8s.io/snapshotter-secret-name: csi-rbd-secret
|
||||
csi.storage.k8s.io/snapshotter-secret-namespace: default
|
||||
|
@ -1,37 +1,53 @@
|
||||
---
|
||||
apiVersion: storage.k8s.io/v1
|
||||
kind: StorageClass
|
||||
metadata:
|
||||
name: csi-rbd
|
||||
provisioner: csi-rbdplugin
|
||||
provisioner: rbd.csi.ceph.com
|
||||
parameters:
|
||||
# Comma separated list of Ceph monitors
|
||||
# if using FQDN, make sure csi plugin's dns policy is appropriate.
|
||||
monitors: mon1:port,mon2:port,...
|
||||
# Comma separated list of Ceph monitors
|
||||
# if using FQDN, make sure csi plugin's dns policy is appropriate.
|
||||
monitors: mon1:port,mon2:port,...
|
||||
# OR,
|
||||
# String representing a Ceph cluster to provision storage from.
|
||||
# Should be unique across all Ceph clusters in use for provisioning,
|
||||
# cannot be greater than 36 bytes in length, and should remain immutable for
|
||||
# the lifetime of the StorageClass in use.
|
||||
# If using clusterID, ensure to create a secret, as in
|
||||
# template-ceph-cluster-ID-secret.yaml, to accompany the string chosen to
|
||||
# represent the Ceph cluster in clusterID
|
||||
# clusterID: <cluster-id>
|
||||
# OR,
|
||||
# if "monitors" parameter is not set, driver to get monitors from same
|
||||
# secret as admin/user credentials. "monValueFromSecret" provides the
|
||||
# key in the secret whose value is the mons
|
||||
# monValueFromSecret: "monitors"
|
||||
|
||||
# if "monitors" parameter is not set, driver to get monitors from same
|
||||
# secret as admin/user credentials. "monValueFromSecret" provides the
|
||||
# key in the secret whose value is the mons
|
||||
#monValueFromSecret: "monitors"
|
||||
# Ceph pool into which the RBD image shall be created
|
||||
pool: rbd
|
||||
|
||||
|
||||
# Ceph pool into which the RBD image shall be created
|
||||
pool: rbd
|
||||
# RBD image format. Defaults to "2".
|
||||
imageFormat: "2"
|
||||
|
||||
# RBD image format. Defaults to "2".
|
||||
imageFormat: "2"
|
||||
# RBD image features. Available for imageFormat: "2"
|
||||
# CSI RBD currently supports only `layering` feature.
|
||||
imageFeatures: layering
|
||||
|
||||
# RBD image features. Available for imageFormat: "2". CSI RBD currently supports only `layering` feature.
|
||||
imageFeatures: layering
|
||||
|
||||
# The secrets have to contain Ceph admin credentials.
|
||||
csiProvisionerSecretName: csi-rbd-secret
|
||||
csiProvisionerSecretNamespace: default
|
||||
csiNodePublishSecretName: csi-rbd-secret
|
||||
csiNodePublishSecretNamespace: default
|
||||
# The secrets have to contain Ceph admin credentials.
|
||||
# NOTE: If using "clusterID" instead of "monitors" above, the following
|
||||
# secrets MAY be added to the ceph-cluster-<cluster-id> secret and skipped
|
||||
# here
|
||||
csi.storage.k8s.io/provisioner-secret-name: csi-rbd-secret
|
||||
csi.storage.k8s.io/provisioner-secret-namespace: default
|
||||
csi.storage.k8s.io/node-publish-secret-name: csi-rbd-secret
|
||||
csi.storage.k8s.io/node-publish-secret-namespace: default
|
||||
|
||||
# Ceph users for operating RBD
|
||||
adminid: admin
|
||||
userid: kubernetes
|
||||
# uncomment the following to use rbd-nbd as mounter on supported nodes
|
||||
#mounter: rbd-nbd
|
||||
# Ceph users for operating RBD
|
||||
# NOTE: If using "clusterID" instead of "monitors" above, the following
|
||||
# IDs MAY be added to the ceph-cluster-<cluster-id> secret and skipped
|
||||
# here
|
||||
adminid: admin
|
||||
userid: kubernetes
|
||||
# uncomment the following to use rbd-nbd as mounter on supported nodes
|
||||
# mounter: rbd-nbd
|
||||
reclaimPolicy: Delete
|
||||
|
36
examples/rbd/template-ceph-cluster-ID-secret.yaml
Normal file
36
examples/rbd/template-ceph-cluster-ID-secret.yaml
Normal file
@ -0,0 +1,36 @@
|
||||
---
|
||||
# This is a template secret that helps define a Ceph cluster configuration
|
||||
# as required by the CSI driver. This is used when a StorageClass has the
|
||||
# "clusterID" defined as one of the parameters, to provide the CSI instance
|
||||
# Ceph cluster configuration information.
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
metadata:
|
||||
# The <cluster-id> is used by the CSI plugin to uniquely identify and use a
|
||||
# Ceph cluster, the value MUST match the value provided as `clusterID` in the
|
||||
# StorageClass
|
||||
name: ceph-cluster-<cluster-id>
|
||||
namespace: default
|
||||
data:
|
||||
# Base64 encoded and comma separated Ceph cluster monitor list
|
||||
# - Typically output of: `echo -n "mon1:port,mon2:port,..." | base64`
|
||||
monitors: <BASE64-ENCODED-MONLIST>
|
||||
# Base64 encoded and comma separated list of pool names from which volumes
|
||||
# can be provisioned
|
||||
pools: <BASE64-ENCODED-POOLIST>
|
||||
# Base64 encoded admin ID to use for provisioning
|
||||
# - Typically output of: `echo -n "<admin-id>" | base64`
|
||||
# Substitute the entire string including angle braces, with the base64 value
|
||||
adminid: <BASE64-ENCODED-ID>
|
||||
# Base64 encoded key of the provisioner admin ID
|
||||
# - Output of: `ceph auth get-key client.<admin-id> | base64`
|
||||
# Substitute the entire string including angle braces, with the base64 value
|
||||
adminkey: <BASE64-ENCODED-PASSWORD>
|
||||
# Base64 encoded user ID to use for publishing
|
||||
# - Typically output of: `echo -n "<admin-id>" | base64`
|
||||
# Substitute the entire string including angle braces, with the base64 value
|
||||
userid: <BASE64-ENCODED-ID>
|
||||
# Base64 encoded key of the publisher user ID
|
||||
# - Output of: `ceph auth get-key client.<admin-id> | base64`
|
||||
# Substitute the entire string including angle braces, with the base64 value
|
||||
userkey: <BASE64-ENCODED-PASSWORD>
|
33
examples/rbd/template-csi-rbdplugin-patch.yaml
Normal file
33
examples/rbd/template-csi-rbdplugin-patch.yaml
Normal file
@ -0,0 +1,33 @@
|
||||
---
|
||||
# This is a patch to the existing daemonset deployment of CSI rbdplugin.
|
||||
#
|
||||
# This is to be used when using `clusterID` instead of monitors or
|
||||
# monValueFromSecret in the StorageClass to specify the Ceph cluster to
|
||||
# provision storage from, AND when the value of `--configroot` option to the
|
||||
# CSI pods is NOT "k8s_objects".
|
||||
#
|
||||
# This patch file, patches in the specified secret for the 'clusterID' as a
|
||||
# volume, instead of the Ceph CSI plugin actively fetching and using kubernetes
|
||||
# secrets.
|
||||
#
|
||||
# Post substituting the <cluster-id> in all places execute,
|
||||
# `kubectl patch daemonset csi-rbdplugin --patch\
|
||||
# "$(cat template-csi-rbdplugin-patch.yaml)"`
|
||||
# to patch the daemonset deployment.
|
||||
#
|
||||
# `kubectl patch statefulset csi-rbdplugin-provisioner --patch\
|
||||
# "$(cat template-csi-rbdplugin-patch.yaml)"`
|
||||
# to patch the statefulset deployment.
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: csi-rbdplugin
|
||||
volumeMounts:
|
||||
- name: ceph-cluster-<cluster-id>
|
||||
mountPath: "/etc/csi-config/ceph-cluster-<cluster-id>"
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: ceph-cluster-<cluster-id>
|
||||
secret:
|
||||
secretName: ceph-cluster-<cluster-id>
|
@ -17,114 +17,32 @@ limitations under the License.
|
||||
package cephfs
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path"
|
||||
"text/template"
|
||||
)
|
||||
|
||||
const cephConfig = `[global]
|
||||
mon_host = {{.Monitors}}
|
||||
var cephConfig = []byte(`[global]
|
||||
auth_cluster_required = cephx
|
||||
auth_service_required = cephx
|
||||
auth_client_required = cephx
|
||||
|
||||
# Workaround for http://tracker.ceph.com/issues/23446
|
||||
fuse_set_user_groups = false
|
||||
`
|
||||
|
||||
const cephKeyring = `[client.{{.UserId}}]
|
||||
key = {{.Key}}
|
||||
`
|
||||
|
||||
const cephSecret = `{{.Key}}`
|
||||
`)
|
||||
|
||||
const (
|
||||
cephConfigRoot = "/etc/ceph"
|
||||
cephConfigFileNameFmt = "ceph.share.%s.conf"
|
||||
cephKeyringFileNameFmt = "ceph.share.%s.client.%s.keyring"
|
||||
cephSecretFileNameFmt = "ceph.share.%s.client.%s.secret"
|
||||
cephConfigRoot = "/etc/ceph"
|
||||
cephConfigPath = "/etc/ceph/ceph.conf"
|
||||
)
|
||||
|
||||
var (
|
||||
cephConfigTempl *template.Template
|
||||
cephKeyringTempl *template.Template
|
||||
cephSecretTempl *template.Template
|
||||
)
|
||||
|
||||
func init() {
|
||||
fm := map[string]interface{}{
|
||||
"perms": func(readOnly bool) string {
|
||||
if readOnly {
|
||||
return "r"
|
||||
}
|
||||
|
||||
return "rw"
|
||||
},
|
||||
}
|
||||
|
||||
cephConfigTempl = template.Must(template.New("config").Parse(cephConfig))
|
||||
cephKeyringTempl = template.Must(template.New("keyring").Funcs(fm).Parse(cephKeyring))
|
||||
cephSecretTempl = template.Must(template.New("secret").Parse(cephSecret))
|
||||
func createCephConfigRoot() error {
|
||||
return os.MkdirAll(cephConfigRoot, 0755) // #nosec
|
||||
}
|
||||
|
||||
type cephConfigWriter interface {
|
||||
writeToFile() error
|
||||
}
|
||||
|
||||
type cephConfigData struct {
|
||||
Monitors string
|
||||
VolumeID volumeID
|
||||
}
|
||||
|
||||
func writeCephTemplate(fileName string, m os.FileMode, t *template.Template, data interface{}) error {
|
||||
if err := os.MkdirAll(cephConfigRoot, 0755); err != nil {
|
||||
func writeCephConfig() error {
|
||||
if err := createCephConfigRoot(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
f, err := os.OpenFile(path.Join(cephConfigRoot, fileName), os.O_CREATE|os.O_EXCL|os.O_WRONLY, m)
|
||||
if err != nil {
|
||||
if os.IsExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
defer f.Close()
|
||||
|
||||
return t.Execute(f, data)
|
||||
}
|
||||
|
||||
func (d *cephConfigData) writeToFile() error {
|
||||
return writeCephTemplate(fmt.Sprintf(cephConfigFileNameFmt, d.VolumeID), 0640, cephConfigTempl, d)
|
||||
}
|
||||
|
||||
type cephKeyringData struct {
|
||||
UserId, Key string
|
||||
VolumeID volumeID
|
||||
}
|
||||
|
||||
func (d *cephKeyringData) writeToFile() error {
|
||||
return writeCephTemplate(fmt.Sprintf(cephKeyringFileNameFmt, d.VolumeID, d.UserId), 0600, cephKeyringTempl, d)
|
||||
}
|
||||
|
||||
type cephSecretData struct {
|
||||
UserId, Key string
|
||||
VolumeID volumeID
|
||||
}
|
||||
|
||||
func (d *cephSecretData) writeToFile() error {
|
||||
return writeCephTemplate(fmt.Sprintf(cephSecretFileNameFmt, d.VolumeID, d.UserId), 0600, cephSecretTempl, d)
|
||||
}
|
||||
|
||||
func getCephSecretPath(volId volumeID, userId string) string {
|
||||
return path.Join(cephConfigRoot, fmt.Sprintf(cephSecretFileNameFmt, volId, userId))
|
||||
}
|
||||
|
||||
func getCephKeyringPath(volId volumeID, userId string) string {
|
||||
return path.Join(cephConfigRoot, fmt.Sprintf(cephKeyringFileNameFmt, volId, userId))
|
||||
}
|
||||
|
||||
func getCephConfPath(volId volumeID) string {
|
||||
return path.Join(cephConfigRoot, fmt.Sprintf(cephConfigFileNameFmt, volId))
|
||||
return ioutil.WriteFile(cephConfigPath, cephConfig, 0640)
|
||||
}
|
||||
|
@ -17,10 +17,7 @@ limitations under the License.
|
||||
package cephfs
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
)
|
||||
|
||||
const (
|
||||
@ -47,76 +44,65 @@ func (ent *cephEntity) toCredentials() *credentials {
|
||||
}
|
||||
}
|
||||
|
||||
func getCephUserName(volId volumeID) string {
|
||||
return cephUserPrefix + string(volId)
|
||||
func getCephUserName(volID volumeID) string {
|
||||
return cephUserPrefix + string(volID)
|
||||
}
|
||||
|
||||
func getCephUser(adminCr *credentials, volId volumeID) (*cephEntity, error) {
|
||||
entityName := cephEntityClientPrefix + getCephUserName(volId)
|
||||
|
||||
func getSingleCephEntity(args ...string) (*cephEntity, error) {
|
||||
var ents []cephEntity
|
||||
args := [...]string{
|
||||
"auth", "-f", "json", "-c", getCephConfPath(volId), "-n", cephEntityClientPrefix + adminCr.id,
|
||||
"get", entityName,
|
||||
}
|
||||
|
||||
out, err := execCommand("ceph", args[:]...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cephfs: ceph failed with following error: %s\ncephfs: ceph output: %s", err, out)
|
||||
}
|
||||
|
||||
// Workaround for output from `ceph auth get`
|
||||
// Contains non-json data: "exported keyring for ENTITY\n\n"
|
||||
offset := bytes.Index(out, []byte("[{"))
|
||||
|
||||
if json.NewDecoder(bytes.NewReader(out[offset:])).Decode(&ents); err != nil {
|
||||
return nil, fmt.Errorf("failed to decode json: %v", err)
|
||||
if err := execCommandJSON(&ents, "ceph", args...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(ents) != 1 {
|
||||
return nil, fmt.Errorf("got unexpected number of entities for %s: expected 1, got %d", entityName, len(ents))
|
||||
return nil, fmt.Errorf("got unexpected number of entities: expected 1, got %d", len(ents))
|
||||
}
|
||||
|
||||
return &ents[0], nil
|
||||
}
|
||||
|
||||
func createCephUser(volOptions *volumeOptions, adminCr *credentials, volId volumeID) (*cephEntity, error) {
|
||||
caps := cephEntityCaps{
|
||||
Mds: fmt.Sprintf("allow rw path=%s", getVolumeRootPathCeph(volId)),
|
||||
Mon: "allow r",
|
||||
Osd: fmt.Sprintf("allow rw pool=%s namespace=%s", volOptions.Pool, getVolumeNamespace(volId)),
|
||||
}
|
||||
|
||||
var ents []cephEntity
|
||||
args := [...]string{
|
||||
"auth", "-f", "json", "-c", getCephConfPath(volId), "-n", cephEntityClientPrefix + adminCr.id,
|
||||
"get-or-create", cephEntityClientPrefix + getCephUserName(volId),
|
||||
"mds", caps.Mds,
|
||||
"mon", caps.Mon,
|
||||
"osd", caps.Osd,
|
||||
}
|
||||
|
||||
if err := execCommandJSON(&ents, "ceph", args[:]...); err != nil {
|
||||
return nil, fmt.Errorf("error creating ceph user: %v", err)
|
||||
}
|
||||
|
||||
return &ents[0], nil
|
||||
func genUserIDs(adminCr *credentials, volID volumeID) (adminID, userID string) {
|
||||
return cephEntityClientPrefix + adminCr.id, cephEntityClientPrefix + getCephUserName(volID)
|
||||
}
|
||||
|
||||
func deleteCephUser(adminCr *credentials, volId volumeID) error {
|
||||
userId := getCephUserName(volId)
|
||||
func getCephUser(volOptions *volumeOptions, adminCr *credentials, volID volumeID) (*cephEntity, error) {
|
||||
adminID, userID := genUserIDs(adminCr, volID)
|
||||
|
||||
args := [...]string{
|
||||
"-c", getCephConfPath(volId), "-n", cephEntityClientPrefix + adminCr.id,
|
||||
"auth", "rm", cephEntityClientPrefix + userId,
|
||||
}
|
||||
|
||||
if err := execCommandAndValidate("ceph", args[:]...); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
os.Remove(getCephKeyringPath(volId, userId))
|
||||
os.Remove(getCephSecretPath(volId, userId))
|
||||
|
||||
return nil
|
||||
return getSingleCephEntity(
|
||||
"-m", volOptions.Monitors,
|
||||
"-n", adminID,
|
||||
"--key="+adminCr.key,
|
||||
"-c", cephConfigPath,
|
||||
"-f", "json",
|
||||
"auth", "get", userID,
|
||||
)
|
||||
}
|
||||
|
||||
func createCephUser(volOptions *volumeOptions, adminCr *credentials, volID volumeID) (*cephEntity, error) {
|
||||
adminID, userID := genUserIDs(adminCr, volID)
|
||||
|
||||
return getSingleCephEntity(
|
||||
"-m", volOptions.Monitors,
|
||||
"-n", adminID,
|
||||
"--key="+adminCr.key,
|
||||
"-c", cephConfigPath,
|
||||
"-f", "json",
|
||||
"auth", "get-or-create", userID,
|
||||
// User capabilities
|
||||
"mds", fmt.Sprintf("allow rw path=%s", getVolumeRootPathCeph(volID)),
|
||||
"mon", "allow r",
|
||||
"osd", fmt.Sprintf("allow rw pool=%s namespace=%s", volOptions.Pool, getVolumeNamespace(volID)),
|
||||
)
|
||||
}
|
||||
|
||||
func deleteCephUser(volOptions *volumeOptions, adminCr *credentials, volID volumeID) error {
|
||||
adminID, userID := genUserIDs(adminCr, volID)
|
||||
|
||||
return execCommandErr("ceph",
|
||||
"-m", volOptions.Monitors,
|
||||
"-n", adminID,
|
||||
"--key="+adminCr.key,
|
||||
"-c", cephConfigPath,
|
||||
"auth", "rm", userID,
|
||||
)
|
||||
}
|
||||
|
@ -17,18 +17,20 @@ limitations under the License.
|
||||
package cephfs
|
||||
|
||||
import (
|
||||
"github.com/golang/glog"
|
||||
csicommon "github.com/ceph/ceph-csi/pkg/csi-common"
|
||||
"github.com/ceph/ceph-csi/pkg/util"
|
||||
|
||||
"github.com/container-storage-interface/spec/lib/go/csi"
|
||||
"golang.org/x/net/context"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
|
||||
"github.com/container-storage-interface/spec/lib/go/csi/v0"
|
||||
"github.com/kubernetes-csi/drivers/pkg/csi-common"
|
||||
|
||||
"github.com/ceph/ceph-csi/pkg/util"
|
||||
"k8s.io/klog"
|
||||
"k8s.io/kubernetes/pkg/util/keymutex"
|
||||
)
|
||||
|
||||
type controllerServer struct {
|
||||
// ControllerServer struct of CEPH CSI driver with supported methods of CSI
|
||||
// controller server spec.
|
||||
type ControllerServer struct {
|
||||
*csicommon.DefaultControllerServer
|
||||
MetadataStore util.CachePersister
|
||||
}
|
||||
@ -38,129 +40,152 @@ type controllerCacheEntry struct {
|
||||
VolumeID volumeID
|
||||
}
|
||||
|
||||
func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) {
|
||||
var (
|
||||
mtxControllerVolumeID = keymutex.NewHashed(0)
|
||||
)
|
||||
|
||||
// CreateVolume creates the volume in backend and store the volume metadata
|
||||
func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error) {
|
||||
if err := cs.validateCreateVolumeRequest(req); err != nil {
|
||||
glog.Errorf("CreateVolumeRequest validation failed: %v", err)
|
||||
klog.Errorf("CreateVolumeRequest validation failed: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Configuration
|
||||
|
||||
volOptions, err := newVolumeOptions(req.GetParameters())
|
||||
secret := req.GetSecrets()
|
||||
volOptions, err := newVolumeOptions(req.GetParameters(), secret)
|
||||
if err != nil {
|
||||
glog.Errorf("validation of volume options failed: %v", err)
|
||||
klog.Errorf("validation of volume options failed: %v", err)
|
||||
return nil, status.Error(codes.InvalidArgument, err.Error())
|
||||
}
|
||||
|
||||
volId := newVolumeID()
|
||||
volID := makeVolumeID(req.GetName())
|
||||
|
||||
conf := cephConfigData{Monitors: volOptions.Monitors, VolumeID: volId}
|
||||
if err = conf.writeToFile(); err != nil {
|
||||
glog.Errorf("failed to write ceph config file to %s: %v", getCephConfPath(volId), err)
|
||||
return nil, status.Error(codes.Internal, err.Error())
|
||||
}
|
||||
mtxControllerVolumeID.LockKey(string(volID))
|
||||
defer mustUnlock(mtxControllerVolumeID, string(volID))
|
||||
|
||||
// Create a volume in case the user didn't provide one
|
||||
|
||||
if volOptions.ProvisionVolume {
|
||||
// Admin credentials are required
|
||||
cr, err := getAdminCredentials(req.GetControllerCreateSecrets())
|
||||
cr, err := getAdminCredentials(secret)
|
||||
if err != nil {
|
||||
return nil, status.Error(codes.InvalidArgument, err.Error())
|
||||
}
|
||||
|
||||
if err = storeCephCredentials(volId, cr); err != nil {
|
||||
glog.Errorf("failed to store admin credentials for '%s': %v", cr.id, err)
|
||||
if err = createVolume(volOptions, cr, volID, req.GetCapacityRange().GetRequiredBytes()); err != nil {
|
||||
klog.Errorf("failed to create volume %s: %v", req.GetName(), err)
|
||||
return nil, status.Error(codes.Internal, err.Error())
|
||||
}
|
||||
|
||||
if err = createVolume(volOptions, cr, volId, req.GetCapacityRange().GetRequiredBytes()); err != nil {
|
||||
glog.Errorf("failed to create volume %s: %v", req.GetName(), err)
|
||||
if _, err = createCephUser(volOptions, cr, volID); err != nil {
|
||||
klog.Errorf("failed to create ceph user for volume %s: %v", req.GetName(), err)
|
||||
return nil, status.Error(codes.Internal, err.Error())
|
||||
}
|
||||
|
||||
if _, err = createCephUser(volOptions, cr, volId); err != nil {
|
||||
glog.Errorf("failed to create ceph user for volume %s: %v", req.GetName(), err)
|
||||
return nil, status.Error(codes.Internal, err.Error())
|
||||
}
|
||||
|
||||
glog.Infof("cephfs: successfully created volume %s", volId)
|
||||
klog.Infof("cephfs: successfully created volume %s", volID)
|
||||
} else {
|
||||
glog.Infof("cephfs: volume %s is provisioned statically", volId)
|
||||
klog.Infof("cephfs: volume %s is provisioned statically", volID)
|
||||
}
|
||||
|
||||
ce := &controllerCacheEntry{VolOptions: *volOptions, VolumeID: volId}
|
||||
if err := cs.MetadataStore.Create(string(volId), ce); err != nil {
|
||||
glog.Errorf("failed to store a cache entry for volume %s: %v", volId, err)
|
||||
ce := &controllerCacheEntry{VolOptions: *volOptions, VolumeID: volID}
|
||||
if err := cs.MetadataStore.Create(string(volID), ce); err != nil {
|
||||
klog.Errorf("failed to store a cache entry for volume %s: %v", volID, err)
|
||||
return nil, status.Error(codes.Internal, err.Error())
|
||||
}
|
||||
|
||||
return &csi.CreateVolumeResponse{
|
||||
Volume: &csi.Volume{
|
||||
Id: string(volId),
|
||||
VolumeId: string(volID),
|
||||
CapacityBytes: req.GetCapacityRange().GetRequiredBytes(),
|
||||
Attributes: req.GetParameters(),
|
||||
VolumeContext: req.GetParameters(),
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (cs *controllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) {
|
||||
if err := cs.validateDeleteVolumeRequest(req); err != nil {
|
||||
glog.Errorf("DeleteVolumeRequest validation failed: %v", err)
|
||||
// DeleteVolume deletes the volume in backend
|
||||
// and removes the volume metadata from store
|
||||
// nolint: gocyclo
|
||||
func (cs *ControllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error) {
|
||||
if err := cs.validateDeleteVolumeRequest(); err != nil {
|
||||
klog.Errorf("DeleteVolumeRequest validation failed: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var (
|
||||
volId = volumeID(req.GetVolumeId())
|
||||
err error
|
||||
volID = volumeID(req.GetVolumeId())
|
||||
secrets = req.GetSecrets()
|
||||
)
|
||||
|
||||
ce := &controllerCacheEntry{}
|
||||
if err := cs.MetadataStore.Get(string(volId), ce); err != nil {
|
||||
if err := cs.MetadataStore.Get(string(volID), ce); err != nil {
|
||||
if err, ok := err.(*util.CacheEntryNotFound); ok {
|
||||
klog.Infof("cephfs: metadata for volume %s not found, assuming the volume to be already deleted (%v)", volID, err)
|
||||
return &csi.DeleteVolumeResponse{}, nil
|
||||
}
|
||||
|
||||
return nil, status.Error(codes.Internal, err.Error())
|
||||
}
|
||||
|
||||
if !ce.VolOptions.ProvisionVolume {
|
||||
// DeleteVolume() is forbidden for statically provisioned volumes!
|
||||
|
||||
glog.Warningf("volume %s is provisioned statically, aborting delete", volId)
|
||||
klog.Warningf("volume %s is provisioned statically, aborting delete", volID)
|
||||
return &csi.DeleteVolumeResponse{}, nil
|
||||
}
|
||||
|
||||
// mons may have changed since create volume,
|
||||
// retrieve the latest mons and override old mons
|
||||
if mon, secretsErr := getMonValFromSecret(secrets); secretsErr == nil && len(mon) > 0 {
|
||||
klog.Infof("overriding monitors [%q] with [%q] for volume %s", ce.VolOptions.Monitors, mon, volID)
|
||||
ce.VolOptions.Monitors = mon
|
||||
}
|
||||
|
||||
// Deleting a volume requires admin credentials
|
||||
|
||||
cr, err := getAdminCredentials(req.GetControllerDeleteSecrets())
|
||||
cr, err := getAdminCredentials(secrets)
|
||||
if err != nil {
|
||||
glog.Errorf("failed to retrieve admin credentials: %v", err)
|
||||
klog.Errorf("failed to retrieve admin credentials: %v", err)
|
||||
return nil, status.Error(codes.InvalidArgument, err.Error())
|
||||
}
|
||||
|
||||
if err = purgeVolume(volId, cr, &ce.VolOptions); err != nil {
|
||||
glog.Errorf("failed to delete volume %s: %v", volId, err)
|
||||
mtxControllerVolumeID.LockKey(string(volID))
|
||||
defer mustUnlock(mtxControllerVolumeID, string(volID))
|
||||
|
||||
if err = purgeVolume(volID, cr, &ce.VolOptions); err != nil {
|
||||
klog.Errorf("failed to delete volume %s: %v", volID, err)
|
||||
return nil, status.Error(codes.Internal, err.Error())
|
||||
}
|
||||
|
||||
if err = deleteCephUser(cr, volId); err != nil {
|
||||
glog.Errorf("failed to delete ceph user for volume %s: %v", volId, err)
|
||||
if err = deleteCephUser(&ce.VolOptions, cr, volID); err != nil {
|
||||
klog.Errorf("failed to delete ceph user for volume %s: %v", volID, err)
|
||||
return nil, status.Error(codes.Internal, err.Error())
|
||||
}
|
||||
|
||||
if err := cs.MetadataStore.Delete(string(volId)); err != nil {
|
||||
if err = cs.MetadataStore.Delete(string(volID)); err != nil {
|
||||
return nil, status.Error(codes.Internal, err.Error())
|
||||
}
|
||||
|
||||
glog.Infof("cephfs: successfully deleted volume %s", volId)
|
||||
klog.Infof("cephfs: successfully deleted volume %s", volID)
|
||||
|
||||
return &csi.DeleteVolumeResponse{}, nil
|
||||
}
|
||||
|
||||
func (cs *controllerServer) ValidateVolumeCapabilities(
|
||||
// ValidateVolumeCapabilities checks whether the volume capabilities requested
|
||||
// are supported.
|
||||
func (cs *ControllerServer) ValidateVolumeCapabilities(
|
||||
ctx context.Context,
|
||||
req *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error) {
|
||||
// Cephfs doesn't support Block volume
|
||||
for _, cap := range req.VolumeCapabilities {
|
||||
if cap.GetBlock() != nil {
|
||||
return &csi.ValidateVolumeCapabilitiesResponse{Supported: false, Message: ""}, nil
|
||||
return &csi.ValidateVolumeCapabilitiesResponse{Message: ""}, nil
|
||||
}
|
||||
}
|
||||
return &csi.ValidateVolumeCapabilitiesResponse{Supported: true}, nil
|
||||
return &csi.ValidateVolumeCapabilitiesResponse{
|
||||
Confirmed: &csi.ValidateVolumeCapabilitiesResponse_Confirmed{
|
||||
VolumeCapabilities: req.VolumeCapabilities,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
@ -19,10 +19,11 @@ package cephfs
|
||||
import "fmt"
|
||||
|
||||
const (
|
||||
credUserId = "userID"
|
||||
credUserID = "userID"
|
||||
credUserKey = "userKey"
|
||||
credAdminId = "adminID"
|
||||
credAdminID = "adminID"
|
||||
credAdminKey = "adminKey"
|
||||
credMonitors = "monitors"
|
||||
)
|
||||
|
||||
type credentials struct {
|
||||
@ -48,9 +49,16 @@ func getCredentials(idField, keyField string, secrets map[string]string) (*crede
|
||||
}
|
||||
|
||||
func getUserCredentials(secrets map[string]string) (*credentials, error) {
|
||||
return getCredentials(credUserId, credUserKey, secrets)
|
||||
return getCredentials(credUserID, credUserKey, secrets)
|
||||
}
|
||||
|
||||
func getAdminCredentials(secrets map[string]string) (*credentials, error) {
|
||||
return getCredentials(credAdminId, credAdminKey, secrets)
|
||||
return getCredentials(credAdminID, credAdminKey, secrets)
|
||||
}
|
||||
|
||||
func getMonValFromSecret(secrets map[string]string) (string, error) {
|
||||
if mons, ok := secrets[credMonitors]; ok {
|
||||
return mons, nil
|
||||
}
|
||||
return "", fmt.Errorf("missing %q", credMonitors)
|
||||
}
|
||||
|
@ -17,76 +17,78 @@ limitations under the License.
|
||||
package cephfs
|
||||
|
||||
import (
|
||||
"github.com/golang/glog"
|
||||
|
||||
"github.com/container-storage-interface/spec/lib/go/csi/v0"
|
||||
csicommon "github.com/kubernetes-csi/drivers/pkg/csi-common"
|
||||
"k8s.io/klog"
|
||||
|
||||
csicommon "github.com/ceph/ceph-csi/pkg/csi-common"
|
||||
"github.com/ceph/ceph-csi/pkg/util"
|
||||
|
||||
"github.com/container-storage-interface/spec/lib/go/csi"
|
||||
)
|
||||
|
||||
const (
|
||||
// Version of the cephfs csi driver
|
||||
Version = "0.3.0"
|
||||
|
||||
// version of ceph driver
|
||||
version = "1.0.0"
|
||||
)
|
||||
|
||||
var (
|
||||
// PluginFolder is the kubelet plugin directory for cephfs plugin
|
||||
PluginFolder = "/var/lib/kubelet/plugins/"
|
||||
)
|
||||
// PluginFolder defines the location of ceph plugin
|
||||
var PluginFolder = "/var/lib/kubelet/plugins/"
|
||||
|
||||
type cephfsDriver struct {
|
||||
driver *csicommon.CSIDriver
|
||||
// Driver contains the default identity,node and controller struct
|
||||
type Driver struct {
|
||||
cd *csicommon.CSIDriver
|
||||
|
||||
is *identityServer
|
||||
ns *nodeServer
|
||||
cs *controllerServer
|
||||
|
||||
caps []*csi.VolumeCapability_AccessMode
|
||||
cscaps []*csi.ControllerServiceCapability
|
||||
is *IdentityServer
|
||||
ns *NodeServer
|
||||
cs *ControllerServer
|
||||
}
|
||||
|
||||
var (
|
||||
driver *cephfsDriver
|
||||
// DefaultVolumeMounter for mounting volumes
|
||||
DefaultVolumeMounter string
|
||||
)
|
||||
|
||||
// NewCephFSDriver provides a remote csi cephfs driver object.
|
||||
func NewCephFSDriver() *cephfsDriver {
|
||||
return &cephfsDriver{}
|
||||
// NewDriver returns new ceph driver
|
||||
func NewDriver() *Driver {
|
||||
return &Driver{}
|
||||
}
|
||||
|
||||
func NewIdentityServer(d *csicommon.CSIDriver) *identityServer {
|
||||
return &identityServer{
|
||||
// NewIdentityServer initialize a identity server for ceph CSI driver
|
||||
func NewIdentityServer(d *csicommon.CSIDriver) *IdentityServer {
|
||||
return &IdentityServer{
|
||||
DefaultIdentityServer: csicommon.NewDefaultIdentityServer(d),
|
||||
}
|
||||
}
|
||||
|
||||
func NewControllerServer(d *csicommon.CSIDriver, cachePersister util.CachePersister) *controllerServer {
|
||||
return &controllerServer{
|
||||
// NewControllerServer initialize a controller server for ceph CSI driver
|
||||
func NewControllerServer(d *csicommon.CSIDriver, cachePersister util.CachePersister) *ControllerServer {
|
||||
return &ControllerServer{
|
||||
DefaultControllerServer: csicommon.NewDefaultControllerServer(d),
|
||||
MetadataStore: cachePersister,
|
||||
}
|
||||
}
|
||||
|
||||
func NewNodeServer(d *csicommon.CSIDriver) *nodeServer {
|
||||
return &nodeServer{
|
||||
// NewNodeServer initialize a node server for ceph CSI driver.
|
||||
func NewNodeServer(d *csicommon.CSIDriver) *NodeServer {
|
||||
return &NodeServer{
|
||||
DefaultNodeServer: csicommon.NewDefaultNodeServer(d),
|
||||
}
|
||||
}
|
||||
|
||||
func (fs *cephfsDriver) Run(driverName, nodeId, endpoint, volumeMounter string, cachePersister util.CachePersister) {
|
||||
glog.Infof("Driver: %v version: %v", driverName, Version)
|
||||
// Run start a non-blocking grpc controller,node and identityserver for
|
||||
// ceph CSI driver which can serve multiple parallel requests
|
||||
func (fs *Driver) Run(driverName, nodeID, endpoint, volumeMounter, mountCacheDir string, cachePersister util.CachePersister) {
|
||||
klog.Infof("Driver: %v version: %v", driverName, version)
|
||||
|
||||
// Configuration
|
||||
|
||||
if err := loadAvailableMounters(); err != nil {
|
||||
glog.Fatalf("cephfs: failed to load ceph mounters: %v", err)
|
||||
klog.Fatalf("cephfs: failed to load ceph mounters: %v", err)
|
||||
}
|
||||
|
||||
if volumeMounter != "" {
|
||||
if err := validateMounter(volumeMounter); err != nil {
|
||||
glog.Fatalln(err)
|
||||
klog.Fatalln(err)
|
||||
} else {
|
||||
DefaultVolumeMounter = volumeMounter
|
||||
}
|
||||
@ -97,29 +99,40 @@ func (fs *cephfsDriver) Run(driverName, nodeId, endpoint, volumeMounter string,
|
||||
DefaultVolumeMounter = availableMounters[0]
|
||||
}
|
||||
|
||||
glog.Infof("cephfs: setting default volume mounter to %s", DefaultVolumeMounter)
|
||||
klog.Infof("cephfs: setting default volume mounter to %s", DefaultVolumeMounter)
|
||||
|
||||
// Initialize default library driver
|
||||
|
||||
fs.driver = csicommon.NewCSIDriver(driverName, Version, nodeId)
|
||||
if fs.driver == nil {
|
||||
glog.Fatalln("Failed to initialize CSI driver")
|
||||
if err := writeCephConfig(); err != nil {
|
||||
klog.Fatalf("failed to write ceph configuration file: %v", err)
|
||||
}
|
||||
|
||||
fs.driver.AddControllerServiceCapabilities([]csi.ControllerServiceCapability_RPC_Type{
|
||||
initVolumeMountCache(driverName, mountCacheDir, cachePersister)
|
||||
if mountCacheDir != "" {
|
||||
if err := remountCachedVolumes(); err != nil {
|
||||
klog.Warningf("failed to remount cached volumes: %v", err)
|
||||
//ignore remount fail
|
||||
}
|
||||
}
|
||||
// Initialize default library driver
|
||||
|
||||
fs.cd = csicommon.NewCSIDriver(driverName, version, nodeID)
|
||||
if fs.cd == nil {
|
||||
klog.Fatalln("failed to initialize CSI driver")
|
||||
}
|
||||
|
||||
fs.cd.AddControllerServiceCapabilities([]csi.ControllerServiceCapability_RPC_Type{
|
||||
csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME,
|
||||
})
|
||||
|
||||
fs.driver.AddVolumeCapabilityAccessModes([]csi.VolumeCapability_AccessMode_Mode{
|
||||
fs.cd.AddVolumeCapabilityAccessModes([]csi.VolumeCapability_AccessMode_Mode{
|
||||
csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER,
|
||||
})
|
||||
|
||||
// Create gRPC servers
|
||||
|
||||
fs.is = NewIdentityServer(fs.driver)
|
||||
fs.ns = NewNodeServer(fs.driver)
|
||||
fs.is = NewIdentityServer(fs.cd)
|
||||
fs.ns = NewNodeServer(fs.cd)
|
||||
|
||||
fs.cs = NewControllerServer(fs.driver, cachePersister)
|
||||
fs.cs = NewControllerServer(fs.cd, cachePersister)
|
||||
|
||||
server := csicommon.NewNonBlockingGRPCServer()
|
||||
server.Start(endpoint, fs.is, fs.cs, fs.ns)
|
||||
|
@ -19,15 +19,19 @@ package cephfs
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/container-storage-interface/spec/lib/go/csi/v0"
|
||||
"github.com/kubernetes-csi/drivers/pkg/csi-common"
|
||||
csicommon "github.com/ceph/ceph-csi/pkg/csi-common"
|
||||
|
||||
"github.com/container-storage-interface/spec/lib/go/csi"
|
||||
)
|
||||
|
||||
type identityServer struct {
|
||||
// IdentityServer struct of ceph CSI driver with supported methods of CSI
|
||||
// identity server spec.
|
||||
type IdentityServer struct {
|
||||
*csicommon.DefaultIdentityServer
|
||||
}
|
||||
|
||||
func (is *identityServer) GetPluginCapabilities(ctx context.Context, req *csi.GetPluginCapabilitiesRequest) (*csi.GetPluginCapabilitiesResponse, error) {
|
||||
// GetPluginCapabilities returns available capabilities of the ceph driver
|
||||
func (is *IdentityServer) GetPluginCapabilities(ctx context.Context, req *csi.GetPluginCapabilitiesRequest) (*csi.GetPluginCapabilitiesResponse, error) {
|
||||
return &csi.GetPluginCapabilitiesResponse{
|
||||
Capabilities: []*csi.PluginCapability{
|
||||
{
|
||||
|
311
pkg/cephfs/mountcache.go
Normal file
311
pkg/cephfs/mountcache.go
Normal file
@ -0,0 +1,311 @@
|
||||
package cephfs
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"os"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/ceph/ceph-csi/pkg/util"
|
||||
"github.com/pkg/errors"
|
||||
"k8s.io/klog"
|
||||
)
|
||||
|
||||
type volumeMountCacheEntry struct {
|
||||
DriverVersion string `json:"driverVersion"`
|
||||
|
||||
VolumeID string `json:"volumeID"`
|
||||
Secrets map[string]string `json:"secrets"`
|
||||
StagingPath string `json:"stagingPath"`
|
||||
TargetPaths map[string]bool `json:"targetPaths"`
|
||||
CreateTime time.Time `json:"createTime"`
|
||||
}
|
||||
|
||||
type volumeMountCacheMap struct {
|
||||
volumes map[string]volumeMountCacheEntry
|
||||
nodeCacheStore util.NodeCache
|
||||
metadataStore util.CachePersister
|
||||
}
|
||||
|
||||
var (
|
||||
volumeMountCachePrefix = "cephfs-mount-cache-"
|
||||
volumeMountCache volumeMountCacheMap
|
||||
volumeMountCacheMtx sync.Mutex
|
||||
)
|
||||
|
||||
func initVolumeMountCache(driverName string, mountCacheDir string, cachePersister util.CachePersister) {
|
||||
volumeMountCache.volumes = make(map[string]volumeMountCacheEntry)
|
||||
|
||||
volumeMountCache.metadataStore = cachePersister
|
||||
volumeMountCache.nodeCacheStore.BasePath = mountCacheDir
|
||||
volumeMountCache.nodeCacheStore.CacheDir = driverName
|
||||
klog.Infof("mount-cache: name: %s, version: %s, mountCacheDir: %s", driverName, version, mountCacheDir)
|
||||
}
|
||||
|
||||
func remountCachedVolumes() error {
|
||||
if err := os.MkdirAll(volumeMountCache.nodeCacheStore.BasePath, 0755); err != nil {
|
||||
klog.Errorf("mount-cache: failed to create %s: %v", volumeMountCache.nodeCacheStore.BasePath, err)
|
||||
return err
|
||||
}
|
||||
var remountFailCount, remountSuccCount int64
|
||||
me := &volumeMountCacheEntry{}
|
||||
ce := &controllerCacheEntry{}
|
||||
err := volumeMountCache.nodeCacheStore.ForAll(volumeMountCachePrefix, me, func(identifier string) error {
|
||||
volID := me.VolumeID
|
||||
if err := volumeMountCache.metadataStore.Get(volID, ce); err != nil {
|
||||
if err, ok := err.(*util.CacheEntryNotFound); ok {
|
||||
klog.Infof("mount-cache: metadata not found, assuming the volume %s to be already deleted (%v)", volID, err)
|
||||
if err := volumeMountCache.nodeCacheStore.Delete(genVolumeMountCacheFileName(volID)); err == nil {
|
||||
klog.Infof("mount-cache: metadata not found, delete volume cache entry for volume %s", volID)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if err := mountOneCacheEntry(ce, me); err == nil {
|
||||
remountSuccCount++
|
||||
volumeMountCache.volumes[me.VolumeID] = *me
|
||||
klog.Infof("mount-cache: successfully remounted volume %s", volID)
|
||||
} else {
|
||||
remountFailCount++
|
||||
klog.Errorf("mount-cache: failed to remount volume %s", volID)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
klog.Infof("mount-cache: metastore list cache fail %v", err)
|
||||
return err
|
||||
}
|
||||
if remountFailCount > 0 {
|
||||
klog.Infof("mount-cache: successfully remounted %d volumes, failed to remount %d volumes", remountSuccCount, remountFailCount)
|
||||
} else {
|
||||
klog.Infof("mount-cache: successfully remounted %d volumes", remountSuccCount)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func mountOneCacheEntry(ce *controllerCacheEntry, me *volumeMountCacheEntry) error {
|
||||
volumeMountCacheMtx.Lock()
|
||||
defer volumeMountCacheMtx.Unlock()
|
||||
|
||||
var (
|
||||
err error
|
||||
cr *credentials
|
||||
)
|
||||
volID := ce.VolumeID
|
||||
volOptions := ce.VolOptions
|
||||
|
||||
if volOptions.ProvisionVolume {
|
||||
volOptions.RootPath = getVolumeRootPathCeph(volID)
|
||||
cr, err = getAdminCredentials(decodeCredentials(me.Secrets))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var entity *cephEntity
|
||||
entity, err = getCephUser(&volOptions, cr, volID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cr = entity.toCredentials()
|
||||
} else {
|
||||
cr, err = getUserCredentials(decodeCredentials(me.Secrets))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
err = cleanupMountPoint(me.StagingPath)
|
||||
if err != nil {
|
||||
klog.Infof("mount-cache: failed to cleanup volume mount point %s, remove it: %s %v", volID, me.StagingPath, err)
|
||||
return err
|
||||
}
|
||||
|
||||
isMnt, err := isMountPoint(me.StagingPath)
|
||||
if err != nil {
|
||||
isMnt = false
|
||||
klog.Infof("mount-cache: failed to check volume mounted %s: %s %v", volID, me.StagingPath, err)
|
||||
}
|
||||
|
||||
if !isMnt {
|
||||
m, err := newMounter(&volOptions)
|
||||
if err != nil {
|
||||
klog.Errorf("mount-cache: failed to create mounter for volume %s: %v", volID, err)
|
||||
return err
|
||||
}
|
||||
if err := m.mount(me.StagingPath, cr, &volOptions); err != nil {
|
||||
klog.Errorf("mount-cache: failed to mount volume %s: %v", volID, err)
|
||||
return err
|
||||
}
|
||||
}
|
||||
for targetPath, readOnly := range me.TargetPaths {
|
||||
if err := cleanupMountPoint(targetPath); err == nil {
|
||||
if err := bindMount(me.StagingPath, targetPath, readOnly); err != nil {
|
||||
klog.Errorf("mount-cache: failed to bind-mount volume %s: %s %s %v %v",
|
||||
volID, me.StagingPath, targetPath, readOnly, err)
|
||||
} else {
|
||||
klog.Infof("mount-cache: successfully bind-mounted volume %s: %s %s %v",
|
||||
volID, me.StagingPath, targetPath, readOnly)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func cleanupMountPoint(mountPoint string) error {
|
||||
if _, err := os.Stat(mountPoint); err != nil {
|
||||
if isCorruptedMnt(err) {
|
||||
klog.Infof("mount-cache: corrupted mount point %s, need unmount", mountPoint)
|
||||
err := execCommandErr("umount", mountPoint)
|
||||
if err != nil {
|
||||
klog.Infof("mount-cache: failed to umount %s %v", mountPoint, err)
|
||||
//ignore error return err
|
||||
}
|
||||
}
|
||||
}
|
||||
if _, err := os.Stat(mountPoint); err != nil {
|
||||
klog.Errorf("mount-cache: failed to stat mount point %s %v", mountPoint, err)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func isCorruptedMnt(err error) bool {
|
||||
var underlyingError error
|
||||
switch pe := err.(type) {
|
||||
case nil:
|
||||
return false
|
||||
case *os.PathError:
|
||||
underlyingError = pe.Err
|
||||
case *os.LinkError:
|
||||
underlyingError = pe.Err
|
||||
case *os.SyscallError:
|
||||
underlyingError = pe.Err
|
||||
default:
|
||||
return false
|
||||
}
|
||||
|
||||
CorruptedErrors := []error{
|
||||
syscall.ENOTCONN, syscall.ESTALE, syscall.EIO, syscall.EACCES}
|
||||
|
||||
for _, v := range CorruptedErrors {
|
||||
if underlyingError == v {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func genVolumeMountCacheFileName(volID string) string {
|
||||
cachePath := volumeMountCachePrefix + volID
|
||||
return cachePath
|
||||
}
|
||||
func (mc *volumeMountCacheMap) isEnable() bool {
|
||||
//if mount cache dir unset, disable state
|
||||
return mc.nodeCacheStore.BasePath != ""
|
||||
}
|
||||
|
||||
func (mc *volumeMountCacheMap) nodeStageVolume(volID string, stagingTargetPath string, secrets map[string]string) error {
|
||||
if !mc.isEnable() {
|
||||
return nil
|
||||
}
|
||||
volumeMountCacheMtx.Lock()
|
||||
defer volumeMountCacheMtx.Unlock()
|
||||
|
||||
lastTargetPaths := make(map[string]bool)
|
||||
me, ok := volumeMountCache.volumes[volID]
|
||||
if ok {
|
||||
if me.StagingPath == stagingTargetPath {
|
||||
klog.Warningf("mount-cache: node unexpected restage volume for volume %s", volID)
|
||||
return nil
|
||||
}
|
||||
lastTargetPaths = me.TargetPaths
|
||||
klog.Warningf("mount-cache: node stage volume ignore last cache entry for volume %s", volID)
|
||||
}
|
||||
|
||||
me = volumeMountCacheEntry{DriverVersion: version}
|
||||
|
||||
me.VolumeID = volID
|
||||
me.Secrets = encodeCredentials(secrets)
|
||||
me.StagingPath = stagingTargetPath
|
||||
me.TargetPaths = lastTargetPaths
|
||||
|
||||
me.CreateTime = time.Now()
|
||||
volumeMountCache.volumes[volID] = me
|
||||
return mc.nodeCacheStore.Create(genVolumeMountCacheFileName(volID), me)
|
||||
}
|
||||
|
||||
func (mc *volumeMountCacheMap) nodeUnStageVolume(volID string) error {
|
||||
if !mc.isEnable() {
|
||||
return nil
|
||||
}
|
||||
volumeMountCacheMtx.Lock()
|
||||
defer volumeMountCacheMtx.Unlock()
|
||||
delete(volumeMountCache.volumes, volID)
|
||||
return mc.nodeCacheStore.Delete(genVolumeMountCacheFileName(volID))
|
||||
}
|
||||
|
||||
func (mc *volumeMountCacheMap) nodePublishVolume(volID string, targetPath string, readOnly bool) error {
|
||||
if !mc.isEnable() {
|
||||
return nil
|
||||
}
|
||||
volumeMountCacheMtx.Lock()
|
||||
defer volumeMountCacheMtx.Unlock()
|
||||
|
||||
_, ok := volumeMountCache.volumes[volID]
|
||||
if !ok {
|
||||
return errors.New("mount-cache: node publish volume failed to find cache entry for volume")
|
||||
}
|
||||
volumeMountCache.volumes[volID].TargetPaths[targetPath] = readOnly
|
||||
return mc.updateNodeCache(volID)
|
||||
}
|
||||
|
||||
func (mc *volumeMountCacheMap) nodeUnPublishVolume(volID string, targetPath string) error {
|
||||
if !mc.isEnable() {
|
||||
return nil
|
||||
}
|
||||
volumeMountCacheMtx.Lock()
|
||||
defer volumeMountCacheMtx.Unlock()
|
||||
|
||||
_, ok := volumeMountCache.volumes[volID]
|
||||
if !ok {
|
||||
return errors.New("mount-cache: node unpublish volume failed to find cache entry for volume")
|
||||
}
|
||||
delete(volumeMountCache.volumes[volID].TargetPaths, targetPath)
|
||||
return mc.updateNodeCache(volID)
|
||||
}
|
||||
|
||||
func (mc *volumeMountCacheMap) updateNodeCache(volID string) error {
|
||||
me := volumeMountCache.volumes[volID]
|
||||
if err := volumeMountCache.nodeCacheStore.Delete(genVolumeMountCacheFileName(volID)); err == nil {
|
||||
klog.Infof("mount-cache: metadata not found, delete mount cache failed for volume %s", volID)
|
||||
}
|
||||
return mc.nodeCacheStore.Create(genVolumeMountCacheFileName(volID), me)
|
||||
}
|
||||
|
||||
func encodeCredentials(input map[string]string) (output map[string]string) {
|
||||
output = make(map[string]string)
|
||||
for key, value := range input {
|
||||
nKey := base64.StdEncoding.EncodeToString([]byte(key))
|
||||
nValue := base64.StdEncoding.EncodeToString([]byte(value))
|
||||
output[nKey] = nValue
|
||||
}
|
||||
return output
|
||||
}
|
||||
|
||||
func decodeCredentials(input map[string]string) (output map[string]string) {
|
||||
output = make(map[string]string)
|
||||
for key, value := range input {
|
||||
nKey, err := base64.StdEncoding.DecodeString(key)
|
||||
if err != nil {
|
||||
klog.Errorf("mount-cache: decode secret fail")
|
||||
continue
|
||||
}
|
||||
nValue, err := base64.StdEncoding.DecodeString(value)
|
||||
if err != nil {
|
||||
klog.Errorf("mount-cache: decode secret fail")
|
||||
continue
|
||||
}
|
||||
output[string(nKey)] = string(nValue)
|
||||
}
|
||||
return output
|
||||
}
|
38
pkg/cephfs/mountcache_test.go
Normal file
38
pkg/cephfs/mountcache_test.go
Normal file
@ -0,0 +1,38 @@
|
||||
package cephfs
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func init() {
|
||||
}
|
||||
|
||||
func TestMountOneCacheEntry(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRemountHisMountedPath(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestNodeStageVolume(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestNodeUnStageVolume(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestNodePublishVolume(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestNodeUnpublishVolume(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestEncodeDecodeCredentials(t *testing.T) {
|
||||
secrets := make(map[string]string)
|
||||
secrets["user_1"] = "value_1"
|
||||
enSecrets := encodeCredentials(secrets)
|
||||
deSecrets := decodeCredentials(enSecrets)
|
||||
for key, value := range secrets {
|
||||
if deSecrets[key] != value {
|
||||
t.Errorf("key %s of credentials's value %s change after decode %s ", key, value, deSecrets[key])
|
||||
}
|
||||
}
|
||||
}
|
@ -21,63 +21,65 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/golang/glog"
|
||||
csicommon "github.com/ceph/ceph-csi/pkg/csi-common"
|
||||
|
||||
"github.com/container-storage-interface/spec/lib/go/csi"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
|
||||
"github.com/container-storage-interface/spec/lib/go/csi/v0"
|
||||
"github.com/kubernetes-csi/drivers/pkg/csi-common"
|
||||
"k8s.io/klog"
|
||||
"k8s.io/kubernetes/pkg/util/keymutex"
|
||||
)
|
||||
|
||||
type nodeServer struct {
|
||||
// NodeServer struct of ceph CSI driver with supported methods of CSI
|
||||
// node server spec.
|
||||
type NodeServer struct {
|
||||
*csicommon.DefaultNodeServer
|
||||
}
|
||||
|
||||
func getCredentialsForVolume(volOptions *volumeOptions, volId volumeID, req *csi.NodeStageVolumeRequest) (*credentials, error) {
|
||||
var (
|
||||
mtxNodeVolumeID = keymutex.NewHashed(0)
|
||||
)
|
||||
|
||||
func getCredentialsForVolume(volOptions *volumeOptions, volID volumeID, req *csi.NodeStageVolumeRequest) (*credentials, error) {
|
||||
var (
|
||||
userCr *credentials
|
||||
err error
|
||||
cr *credentials
|
||||
secrets = req.GetSecrets()
|
||||
)
|
||||
|
||||
if volOptions.ProvisionVolume {
|
||||
// The volume is provisioned dynamically, get the credentials directly from Ceph
|
||||
|
||||
// First, store admin credentials - those are needed for retrieving the user credentials
|
||||
// First, get admin credentials - those are needed for retrieving the user credentials
|
||||
|
||||
adminCr, err := getAdminCredentials(req.GetNodeStageSecrets())
|
||||
adminCr, err := getAdminCredentials(secrets)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get admin credentials from node stage secrets: %v", err)
|
||||
}
|
||||
|
||||
if err = storeCephCredentials(volId, adminCr); err != nil {
|
||||
return nil, fmt.Errorf("failed to store ceph admin credentials: %v", err)
|
||||
}
|
||||
|
||||
// Then get the ceph user
|
||||
|
||||
entity, err := getCephUser(adminCr, volId)
|
||||
entity, err := getCephUser(volOptions, adminCr, volID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get ceph user: %v", err)
|
||||
}
|
||||
|
||||
userCr = entity.toCredentials()
|
||||
cr = entity.toCredentials()
|
||||
} else {
|
||||
// The volume is pre-made, credentials are in node stage secrets
|
||||
|
||||
userCr, err = getUserCredentials(req.GetNodeStageSecrets())
|
||||
userCr, err := getUserCredentials(req.GetSecrets())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get user credentials from node stage secrets: %v", err)
|
||||
}
|
||||
|
||||
cr = userCr
|
||||
}
|
||||
|
||||
if err = storeCephCredentials(volId, userCr); err != nil {
|
||||
return nil, fmt.Errorf("failed to store ceph user credentials: %v", err)
|
||||
}
|
||||
|
||||
return userCr, nil
|
||||
return cr, nil
|
||||
}
|
||||
|
||||
func (ns *nodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) {
|
||||
// NodeStageVolume mounts the volume to a staging path on the node.
|
||||
func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error) {
|
||||
if err := validateNodeStageVolumeRequest(req); err != nil {
|
||||
return nil, status.Error(codes.InvalidArgument, err.Error())
|
||||
}
|
||||
@ -85,70 +87,82 @@ func (ns *nodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVol
|
||||
// Configuration
|
||||
|
||||
stagingTargetPath := req.GetStagingTargetPath()
|
||||
volId := volumeID(req.GetVolumeId())
|
||||
volID := volumeID(req.GetVolumeId())
|
||||
|
||||
volOptions, err := newVolumeOptions(req.GetVolumeAttributes())
|
||||
volOptions, err := newVolumeOptions(req.GetVolumeContext(), req.GetSecrets())
|
||||
if err != nil {
|
||||
glog.Errorf("error reading volume options for volume %s: %v", volId, err)
|
||||
klog.Errorf("error reading volume options for volume %s: %v", volID, err)
|
||||
return nil, status.Error(codes.InvalidArgument, err.Error())
|
||||
}
|
||||
|
||||
if volOptions.ProvisionVolume {
|
||||
// Dynamically provisioned volumes don't have their root path set, do it here
|
||||
volOptions.RootPath = getVolumeRootPathCeph(volId)
|
||||
volOptions.RootPath = getVolumeRootPathCeph(volID)
|
||||
}
|
||||
|
||||
if err = createMountPoint(stagingTargetPath); err != nil {
|
||||
glog.Errorf("failed to create staging mount point at %s for volume %s: %v", stagingTargetPath, volId, err)
|
||||
klog.Errorf("failed to create staging mount point at %s for volume %s: %v", stagingTargetPath, volID, err)
|
||||
return nil, status.Error(codes.Internal, err.Error())
|
||||
}
|
||||
|
||||
cephConf := cephConfigData{Monitors: volOptions.Monitors, VolumeID: volId}
|
||||
if err = cephConf.writeToFile(); err != nil {
|
||||
glog.Errorf("failed to write ceph config file to %s for volume %s: %v", getCephConfPath(volId), volId, err)
|
||||
return nil, status.Error(codes.Internal, err.Error())
|
||||
}
|
||||
mtxNodeVolumeID.LockKey(string(volID))
|
||||
defer mustUnlock(mtxNodeVolumeID, string(volID))
|
||||
|
||||
// Check if the volume is already mounted
|
||||
|
||||
isMnt, err := isMountPoint(stagingTargetPath)
|
||||
|
||||
if err != nil {
|
||||
glog.Errorf("stat failed: %v", err)
|
||||
klog.Errorf("stat failed: %v", err)
|
||||
return nil, status.Error(codes.Internal, err.Error())
|
||||
}
|
||||
|
||||
if isMnt {
|
||||
glog.Infof("cephfs: volume %s is already mounted to %s, skipping", volId, stagingTargetPath)
|
||||
klog.Infof("cephfs: volume %s is already mounted to %s, skipping", volID, stagingTargetPath)
|
||||
return &csi.NodeStageVolumeResponse{}, nil
|
||||
}
|
||||
|
||||
// It's not, mount now
|
||||
|
||||
cr, err := getCredentialsForVolume(volOptions, volId, req)
|
||||
if err != nil {
|
||||
glog.Errorf("failed to get ceph credentials for volume %s: %v", volId, err)
|
||||
return nil, status.Error(codes.Internal, err.Error())
|
||||
if err = ns.mount(volOptions, req); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
m, err := newMounter(volOptions)
|
||||
if err != nil {
|
||||
glog.Errorf("failed to create mounter for volume %s: %v", volId, err)
|
||||
}
|
||||
|
||||
glog.V(4).Infof("cephfs: mounting volume %s with %s", volId, m.name())
|
||||
|
||||
if err = m.mount(stagingTargetPath, cr, volOptions, volId); err != nil {
|
||||
glog.Errorf("failed to mount volume %s: %v", volId, err)
|
||||
return nil, status.Error(codes.Internal, err.Error())
|
||||
}
|
||||
|
||||
glog.Infof("cephfs: successfully mounted volume %s to %s", volId, stagingTargetPath)
|
||||
klog.Infof("cephfs: successfully mounted volume %s to %s", volID, stagingTargetPath)
|
||||
|
||||
return &csi.NodeStageVolumeResponse{}, nil
|
||||
}
|
||||
|
||||
func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) {
|
||||
func (*NodeServer) mount(volOptions *volumeOptions, req *csi.NodeStageVolumeRequest) error {
|
||||
stagingTargetPath := req.GetStagingTargetPath()
|
||||
volID := volumeID(req.GetVolumeId())
|
||||
|
||||
cr, err := getCredentialsForVolume(volOptions, volID, req)
|
||||
if err != nil {
|
||||
klog.Errorf("failed to get ceph credentials for volume %s: %v", volID, err)
|
||||
return status.Error(codes.Internal, err.Error())
|
||||
}
|
||||
|
||||
m, err := newMounter(volOptions)
|
||||
if err != nil {
|
||||
klog.Errorf("failed to create mounter for volume %s: %v", volID, err)
|
||||
return status.Error(codes.Internal, err.Error())
|
||||
}
|
||||
|
||||
klog.V(4).Infof("cephfs: mounting volume %s with %s", volID, m.name())
|
||||
|
||||
if err = m.mount(stagingTargetPath, cr, volOptions); err != nil {
|
||||
klog.Errorf("failed to mount volume %s: %v", volID, err)
|
||||
return status.Error(codes.Internal, err.Error())
|
||||
}
|
||||
if err := volumeMountCache.nodeStageVolume(req.GetVolumeId(), stagingTargetPath, req.GetSecrets()); err != nil {
|
||||
klog.Warningf("mount-cache: failed to stage volume %s %s: %v", volID, stagingTargetPath, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// NodePublishVolume mounts the volume mounted to the staging path to the target
|
||||
// path
|
||||
func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error) {
|
||||
if err := validateNodePublishVolumeRequest(req); err != nil {
|
||||
return nil, status.Error(codes.InvalidArgument, err.Error())
|
||||
}
|
||||
@ -156,10 +170,10 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis
|
||||
// Configuration
|
||||
|
||||
targetPath := req.GetTargetPath()
|
||||
volId := req.GetVolumeId()
|
||||
volID := req.GetVolumeId()
|
||||
|
||||
if err := createMountPoint(targetPath); err != nil {
|
||||
glog.Errorf("failed to create mount point at %s: %v", targetPath, err)
|
||||
klog.Errorf("failed to create mount point at %s: %v", targetPath, err)
|
||||
return nil, status.Error(codes.Internal, err.Error())
|
||||
}
|
||||
|
||||
@ -168,66 +182,89 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis
|
||||
isMnt, err := isMountPoint(targetPath)
|
||||
|
||||
if err != nil {
|
||||
glog.Errorf("stat failed: %v", err)
|
||||
klog.Errorf("stat failed: %v", err)
|
||||
return nil, status.Error(codes.Internal, err.Error())
|
||||
}
|
||||
|
||||
if isMnt {
|
||||
glog.Infof("cephfs: volume %s is already bind-mounted to %s", volId, targetPath)
|
||||
klog.Infof("cephfs: volume %s is already bind-mounted to %s", volID, targetPath)
|
||||
return &csi.NodePublishVolumeResponse{}, nil
|
||||
}
|
||||
|
||||
// It's not, mount now
|
||||
|
||||
if err = bindMount(req.GetStagingTargetPath(), req.GetTargetPath(), req.GetReadonly()); err != nil {
|
||||
glog.Errorf("failed to bind-mount volume %s: %v", volId, err)
|
||||
klog.Errorf("failed to bind-mount volume %s: %v", volID, err)
|
||||
return nil, status.Error(codes.Internal, err.Error())
|
||||
}
|
||||
|
||||
glog.Infof("cephfs: successfully bind-mounted volume %s to %s", volId, targetPath)
|
||||
if err := volumeMountCache.nodePublishVolume(volID, targetPath, req.GetReadonly()); err != nil {
|
||||
klog.Warningf("mount-cache: failed to publish volume %s %s: %v", volID, targetPath, err)
|
||||
}
|
||||
|
||||
klog.Infof("cephfs: successfully bind-mounted volume %s to %s", volID, targetPath)
|
||||
|
||||
return &csi.NodePublishVolumeResponse{}, nil
|
||||
}
|
||||
|
||||
func (ns *nodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) {
|
||||
if err := validateNodeUnpublishVolumeRequest(req); err != nil {
|
||||
// NodeUnpublishVolume unmounts the volume from the target path
|
||||
func (ns *NodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error) {
|
||||
var err error
|
||||
if err = validateNodeUnpublishVolumeRequest(req); err != nil {
|
||||
return nil, status.Error(codes.InvalidArgument, err.Error())
|
||||
}
|
||||
|
||||
targetPath := req.GetTargetPath()
|
||||
|
||||
volID := req.GetVolumeId()
|
||||
if err = volumeMountCache.nodeUnPublishVolume(volID, targetPath); err != nil {
|
||||
klog.Warningf("mount-cache: failed to unpublish volume %s %s: %v", volID, targetPath, err)
|
||||
}
|
||||
|
||||
// Unmount the bind-mount
|
||||
if err := unmountVolume(targetPath); err != nil {
|
||||
if err = unmountVolume(targetPath); err != nil {
|
||||
return nil, status.Error(codes.Internal, err.Error())
|
||||
}
|
||||
|
||||
os.Remove(targetPath)
|
||||
if err = os.Remove(targetPath); err != nil {
|
||||
return nil, status.Error(codes.Internal, err.Error())
|
||||
}
|
||||
|
||||
glog.Infof("cephfs: successfully unbinded volume %s from %s", req.GetVolumeId(), targetPath)
|
||||
klog.Infof("cephfs: successfully unbinded volume %s from %s", req.GetVolumeId(), targetPath)
|
||||
|
||||
return &csi.NodeUnpublishVolumeResponse{}, nil
|
||||
}
|
||||
|
||||
func (ns *nodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error) {
|
||||
if err := validateNodeUnstageVolumeRequest(req); err != nil {
|
||||
// NodeUnstageVolume unstages the volume from the staging path
|
||||
func (ns *NodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error) {
|
||||
var err error
|
||||
if err = validateNodeUnstageVolumeRequest(req); err != nil {
|
||||
return nil, status.Error(codes.InvalidArgument, err.Error())
|
||||
}
|
||||
|
||||
stagingTargetPath := req.GetStagingTargetPath()
|
||||
|
||||
volID := req.GetVolumeId()
|
||||
if err = volumeMountCache.nodeUnStageVolume(volID); err != nil {
|
||||
klog.Warningf("mount-cache: failed to unstage volume %s %s: %v", volID, stagingTargetPath, err)
|
||||
}
|
||||
|
||||
// Unmount the volume
|
||||
if err := unmountVolume(stagingTargetPath); err != nil {
|
||||
if err = unmountVolume(stagingTargetPath); err != nil {
|
||||
return nil, status.Error(codes.Internal, err.Error())
|
||||
}
|
||||
|
||||
os.Remove(stagingTargetPath)
|
||||
if err = os.Remove(stagingTargetPath); err != nil {
|
||||
return nil, status.Error(codes.Internal, err.Error())
|
||||
}
|
||||
|
||||
glog.Infof("cephfs: successfully umounted volume %s from %s", req.GetVolumeId(), stagingTargetPath)
|
||||
klog.Infof("cephfs: successfully unmounted volume %s from %s", req.GetVolumeId(), stagingTargetPath)
|
||||
|
||||
return &csi.NodeUnstageVolumeResponse{}, nil
|
||||
}
|
||||
|
||||
func (ns *nodeServer) NodeGetCapabilities(ctx context.Context, req *csi.NodeGetCapabilitiesRequest) (*csi.NodeGetCapabilitiesResponse, error) {
|
||||
// NodeGetCapabilities returns the supported capabilities of the node server
|
||||
func (ns *NodeServer) NodeGetCapabilities(ctx context.Context, req *csi.NodeGetCapabilitiesRequest) (*csi.NodeGetCapabilitiesResponse, error) {
|
||||
return &csi.NodeGetCapabilitiesResponse{
|
||||
Capabilities: []*csi.NodeServiceCapability{
|
||||
{
|
||||
|
@ -19,48 +19,70 @@ package cephfs
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
|
||||
"github.com/golang/glog"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
"k8s.io/klog"
|
||||
|
||||
"github.com/container-storage-interface/spec/lib/go/csi/v0"
|
||||
"github.com/pborman/uuid"
|
||||
"github.com/ceph/ceph-csi/pkg/util"
|
||||
"github.com/container-storage-interface/spec/lib/go/csi"
|
||||
"k8s.io/kubernetes/pkg/util/keymutex"
|
||||
"k8s.io/kubernetes/pkg/util/mount"
|
||||
)
|
||||
|
||||
type volumeID string
|
||||
|
||||
func newVolumeID() volumeID {
|
||||
return volumeID("csi-cephfs-" + uuid.NewUUID().String())
|
||||
func mustUnlock(m keymutex.KeyMutex, key string) {
|
||||
if err := m.UnlockKey(key); err != nil {
|
||||
klog.Fatalf("failed to unlock mutex for %s: %v", key, err)
|
||||
}
|
||||
}
|
||||
|
||||
func execCommand(command string, args ...string) ([]byte, error) {
|
||||
glog.V(4).Infof("cephfs: EXEC %s %s", command, args)
|
||||
|
||||
cmd := exec.Command(command, args...)
|
||||
return cmd.CombinedOutput()
|
||||
func makeVolumeID(volName string) volumeID {
|
||||
return volumeID("csi-cephfs-" + volName)
|
||||
}
|
||||
|
||||
func execCommandAndValidate(program string, args ...string) error {
|
||||
out, err := execCommand(program, args...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cephfs: %s failed with following error: %s\ncephfs: %s output: %s", program, err, program, out)
|
||||
func execCommand(program string, args ...string) (stdout, stderr []byte, err error) {
|
||||
var (
|
||||
cmd = exec.Command(program, args...) // nolint: gosec
|
||||
sanitizedArgs = util.StripSecretInArgs(args)
|
||||
stdoutBuf bytes.Buffer
|
||||
stderrBuf bytes.Buffer
|
||||
)
|
||||
|
||||
cmd.Stdout = &stdoutBuf
|
||||
cmd.Stderr = &stderrBuf
|
||||
|
||||
klog.V(4).Infof("cephfs: EXEC %s %s", program, sanitizedArgs)
|
||||
|
||||
if err := cmd.Run(); err != nil {
|
||||
return nil, nil, fmt.Errorf("an error occurred while running (%d) %s %v: %v: %s",
|
||||
cmd.Process.Pid, program, sanitizedArgs, err, stderrBuf.Bytes())
|
||||
}
|
||||
|
||||
return nil
|
||||
return stdoutBuf.Bytes(), stderrBuf.Bytes(), nil
|
||||
}
|
||||
|
||||
func execCommandErr(program string, args ...string) error {
|
||||
_, _, err := execCommand(program, args...)
|
||||
return err
|
||||
}
|
||||
|
||||
func execCommandJSON(v interface{}, program string, args ...string) error {
|
||||
out, err := execCommand(program, args...)
|
||||
|
||||
stdout, _, err := execCommand(program, args...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cephfs: %s failed with following error: %s\ncephfs: %s output: %s", program, err, program, out)
|
||||
return err
|
||||
}
|
||||
|
||||
return json.NewDecoder(bytes.NewReader(out)).Decode(v)
|
||||
if err = json.Unmarshal(stdout, v); err != nil {
|
||||
return fmt.Errorf("failed to unmarshal JSON for %s %v: %s: %v", program, util.StripSecretInArgs(args), stdout, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Used in isMountPoint()
|
||||
@ -75,33 +97,13 @@ func isMountPoint(p string) (bool, error) {
|
||||
return !notMnt, nil
|
||||
}
|
||||
|
||||
func storeCephCredentials(volId volumeID, cr *credentials) error {
|
||||
keyringData := cephKeyringData{
|
||||
UserId: cr.id,
|
||||
Key: cr.key,
|
||||
VolumeID: volId,
|
||||
}
|
||||
|
||||
if err := keyringData.writeToFile(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
secret := cephSecretData{
|
||||
UserId: cr.id,
|
||||
Key: cr.key,
|
||||
VolumeID: volId,
|
||||
}
|
||||
|
||||
err := secret.writeToFile()
|
||||
|
||||
return err
|
||||
func pathExists(p string) bool {
|
||||
_, err := os.Stat(p)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
//
|
||||
// Controller service request validation
|
||||
//
|
||||
|
||||
func (cs *controllerServer) validateCreateVolumeRequest(req *csi.CreateVolumeRequest) error {
|
||||
func (cs *ControllerServer) validateCreateVolumeRequest(req *csi.CreateVolumeRequest) error {
|
||||
if err := cs.Driver.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME); err != nil {
|
||||
return fmt.Errorf("invalid CreateVolumeRequest: %v", err)
|
||||
}
|
||||
@ -120,10 +122,11 @@ func (cs *controllerServer) validateCreateVolumeRequest(req *csi.CreateVolumeReq
|
||||
return status.Error(codes.Unimplemented, "block volume not supported")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (cs *controllerServer) validateDeleteVolumeRequest(req *csi.DeleteVolumeRequest) error {
|
||||
func (cs *ControllerServer) validateDeleteVolumeRequest() error {
|
||||
if err := cs.Driver.ValidateControllerServiceRequest(csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME); err != nil {
|
||||
return fmt.Errorf("invalid DeleteVolumeRequest: %v", err)
|
||||
}
|
||||
@ -131,25 +134,22 @@ func (cs *controllerServer) validateDeleteVolumeRequest(req *csi.DeleteVolumeReq
|
||||
return nil
|
||||
}
|
||||
|
||||
//
|
||||
// Node service request validation
|
||||
//
|
||||
|
||||
func validateNodeStageVolumeRequest(req *csi.NodeStageVolumeRequest) error {
|
||||
if req.GetVolumeCapability() == nil {
|
||||
return fmt.Errorf("volume capability missing in request")
|
||||
return errors.New("volume capability missing in request")
|
||||
}
|
||||
|
||||
if req.GetVolumeId() == "" {
|
||||
return fmt.Errorf("volume ID missing in request")
|
||||
return errors.New("volume ID missing in request")
|
||||
}
|
||||
|
||||
if req.GetStagingTargetPath() == "" {
|
||||
return fmt.Errorf("staging target path missing in request")
|
||||
return errors.New("staging target path missing in request")
|
||||
}
|
||||
|
||||
if req.GetNodeStageSecrets() == nil || len(req.GetNodeStageSecrets()) == 0 {
|
||||
return fmt.Errorf("stage secrets cannot be nil or empty")
|
||||
if req.GetSecrets() == nil || len(req.GetSecrets()) == 0 {
|
||||
return errors.New("stage secrets cannot be nil or empty")
|
||||
}
|
||||
|
||||
return nil
|
||||
@ -157,11 +157,11 @@ func validateNodeStageVolumeRequest(req *csi.NodeStageVolumeRequest) error {
|
||||
|
||||
func validateNodeUnstageVolumeRequest(req *csi.NodeUnstageVolumeRequest) error {
|
||||
if req.GetVolumeId() == "" {
|
||||
return fmt.Errorf("volume ID missing in request")
|
||||
return errors.New("volume ID missing in request")
|
||||
}
|
||||
|
||||
if req.GetStagingTargetPath() == "" {
|
||||
return fmt.Errorf("staging target path missing in request")
|
||||
return errors.New("staging target path missing in request")
|
||||
}
|
||||
|
||||
return nil
|
||||
@ -169,15 +169,15 @@ func validateNodeUnstageVolumeRequest(req *csi.NodeUnstageVolumeRequest) error {
|
||||
|
||||
func validateNodePublishVolumeRequest(req *csi.NodePublishVolumeRequest) error {
|
||||
if req.GetVolumeCapability() == nil {
|
||||
return fmt.Errorf("volume capability missing in request")
|
||||
return errors.New("volume capability missing in request")
|
||||
}
|
||||
|
||||
if req.GetVolumeId() == "" {
|
||||
return fmt.Errorf("volume ID missing in request")
|
||||
return errors.New("volume ID missing in request")
|
||||
}
|
||||
|
||||
if req.GetTargetPath() == "" {
|
||||
return fmt.Errorf("varget path missing in request")
|
||||
return errors.New("target path missing in request")
|
||||
}
|
||||
|
||||
return nil
|
||||
@ -185,11 +185,11 @@ func validateNodePublishVolumeRequest(req *csi.NodePublishVolumeRequest) error {
|
||||
|
||||
func validateNodeUnpublishVolumeRequest(req *csi.NodeUnpublishVolumeRequest) error {
|
||||
if req.GetVolumeId() == "" {
|
||||
return fmt.Errorf("volume ID missing in request")
|
||||
return errors.New("volume ID missing in request")
|
||||
}
|
||||
|
||||
if req.GetTargetPath() == "" {
|
||||
return fmt.Errorf("target path missing in request")
|
||||
return errors.New("target path missing in request")
|
||||
}
|
||||
|
||||
return nil
|
||||
|
@ -20,6 +20,8 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
|
||||
"k8s.io/klog"
|
||||
)
|
||||
|
||||
const (
|
||||
@ -28,113 +30,127 @@ const (
|
||||
namespacePrefix = "ns-"
|
||||
)
|
||||
|
||||
var (
|
||||
cephRootPrefix = PluginFolder + "/controller/volumes/root-"
|
||||
)
|
||||
|
||||
func getCephRootPathLocal(volId volumeID) string {
|
||||
return cephRootPrefix + string(volId)
|
||||
func getCephRootPathLocal(volID volumeID) string {
|
||||
return fmt.Sprintf("%s/controller/volumes/root-%s", PluginFolder, string(volID))
|
||||
}
|
||||
|
||||
func getCephRootVolumePathLocal(volId volumeID) string {
|
||||
return path.Join(getCephRootPathLocal(volId), cephVolumesRoot, string(volId))
|
||||
func getCephRootVolumePathLocal(volID volumeID) string {
|
||||
return path.Join(getCephRootPathLocal(volID), cephVolumesRoot, string(volID))
|
||||
}
|
||||
|
||||
func getVolumeRootPathCeph(volId volumeID) string {
|
||||
return path.Join("/", cephVolumesRoot, string(volId))
|
||||
func getVolumeRootPathCeph(volID volumeID) string {
|
||||
return path.Join("/", cephVolumesRoot, string(volID))
|
||||
}
|
||||
|
||||
func getVolumeNamespace(volId volumeID) string {
|
||||
return namespacePrefix + string(volId)
|
||||
func getVolumeNamespace(volID volumeID) string {
|
||||
return namespacePrefix + string(volID)
|
||||
}
|
||||
|
||||
func setVolumeAttribute(root, attrName, attrValue string) error {
|
||||
return execCommandAndValidate("setfattr", "-n", attrName, "-v", attrValue, root)
|
||||
return execCommandErr("setfattr", "-n", attrName, "-v", attrValue, root)
|
||||
}
|
||||
|
||||
func createVolume(volOptions *volumeOptions, adminCr *credentials, volId volumeID, bytesQuota int64) error {
|
||||
cephRoot := getCephRootPathLocal(volId)
|
||||
|
||||
if err := createMountPoint(cephRoot); err != nil {
|
||||
func createVolume(volOptions *volumeOptions, adminCr *credentials, volID volumeID, bytesQuota int64) error {
|
||||
if err := mountCephRoot(volID, volOptions, adminCr); err != nil {
|
||||
return err
|
||||
}
|
||||
defer unmountCephRoot(volID)
|
||||
|
||||
// RootPath is not set for a dynamically provisioned volume
|
||||
// Access to cephfs's / is required
|
||||
volOptions.RootPath = "/"
|
||||
var (
|
||||
volRoot = getCephRootVolumePathLocal(volID)
|
||||
volRootCreating = volRoot + "-creating"
|
||||
)
|
||||
|
||||
m, err := newMounter(volOptions)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create mounter: %v", err)
|
||||
if pathExists(volRoot) {
|
||||
klog.V(4).Infof("cephfs: volume %s already exists, skipping creation", volID)
|
||||
return nil
|
||||
}
|
||||
|
||||
if err = m.mount(cephRoot, adminCr, volOptions, volId); err != nil {
|
||||
return fmt.Errorf("error mounting ceph root: %v", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
unmountVolume(cephRoot)
|
||||
os.Remove(cephRoot)
|
||||
}()
|
||||
|
||||
volOptions.RootPath = getVolumeRootPathCeph(volId)
|
||||
localVolRoot := getCephRootVolumePathLocal(volId)
|
||||
|
||||
if err := createMountPoint(localVolRoot); err != nil {
|
||||
if err := createMountPoint(volRootCreating); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if bytesQuota > 0 {
|
||||
if err := setVolumeAttribute(localVolRoot, "ceph.quota.max_bytes", fmt.Sprintf("%d", bytesQuota)); err != nil {
|
||||
if err := setVolumeAttribute(volRootCreating, "ceph.quota.max_bytes", fmt.Sprintf("%d", bytesQuota)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if err := setVolumeAttribute(localVolRoot, "ceph.dir.layout.pool", volOptions.Pool); err != nil {
|
||||
if err := setVolumeAttribute(volRootCreating, "ceph.dir.layout.pool", volOptions.Pool); err != nil {
|
||||
return fmt.Errorf("%v\ncephfs: Does pool '%s' exist?", err, volOptions.Pool)
|
||||
}
|
||||
|
||||
err = setVolumeAttribute(localVolRoot, "ceph.dir.layout.pool_namespace", getVolumeNamespace(volId))
|
||||
if err := setVolumeAttribute(volRootCreating, "ceph.dir.layout.pool_namespace", getVolumeNamespace(volID)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return err
|
||||
if err := os.Rename(volRootCreating, volRoot); err != nil {
|
||||
return fmt.Errorf("couldn't mark volume %s as created: %v", volID, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func purgeVolume(volId volumeID, adminCr *credentials, volOptions *volumeOptions) error {
|
||||
func purgeVolume(volID volumeID, adminCr *credentials, volOptions *volumeOptions) error {
|
||||
if err := mountCephRoot(volID, volOptions, adminCr); err != nil {
|
||||
return err
|
||||
}
|
||||
defer unmountCephRoot(volID)
|
||||
|
||||
var (
|
||||
cephRoot = getCephRootPathLocal(volId)
|
||||
volRoot = getCephRootVolumePathLocal(volId)
|
||||
volRoot = getCephRootVolumePathLocal(volID)
|
||||
volRootDeleting = volRoot + "-deleting"
|
||||
)
|
||||
|
||||
if err := createMountPoint(cephRoot); err != nil {
|
||||
return err
|
||||
if pathExists(volRoot) {
|
||||
if err := os.Rename(volRoot, volRootDeleting); err != nil {
|
||||
return fmt.Errorf("couldn't mark volume %s for deletion: %v", volID, err)
|
||||
}
|
||||
} else {
|
||||
if !pathExists(volRootDeleting) {
|
||||
klog.V(4).Infof("cephfs: volume %s not found, assuming it to be already deleted", volID)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
if err := os.RemoveAll(volRootDeleting); err != nil {
|
||||
return fmt.Errorf("failed to delete volume %s: %v", volID, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func mountCephRoot(volID volumeID, volOptions *volumeOptions, adminCr *credentials) error {
|
||||
cephRoot := getCephRootPathLocal(volID)
|
||||
|
||||
// Root path is not set for dynamically provisioned volumes
|
||||
// Access to cephfs's / is required
|
||||
volOptions.RootPath = "/"
|
||||
|
||||
if err := createMountPoint(cephRoot); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
m, err := newMounter(volOptions)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create mounter: %v", err)
|
||||
}
|
||||
|
||||
if err = m.mount(cephRoot, adminCr, volOptions, volId); err != nil {
|
||||
if err = m.mount(cephRoot, adminCr, volOptions); err != nil {
|
||||
return fmt.Errorf("error mounting ceph root: %v", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
unmountVolume(volRoot)
|
||||
os.Remove(volRoot)
|
||||
}()
|
||||
|
||||
if err := os.Rename(volRoot, volRootDeleting); err != nil {
|
||||
return fmt.Errorf("coudln't mark volume %s for deletion: %v", volId, err)
|
||||
}
|
||||
|
||||
if err := os.RemoveAll(volRootDeleting); err != nil {
|
||||
return fmt.Errorf("failed to delete volume %s: %v", volId, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func unmountCephRoot(volID volumeID) {
|
||||
cephRoot := getCephRootPathLocal(volID)
|
||||
|
||||
if err := unmountVolume(cephRoot); err != nil {
|
||||
klog.Errorf("failed to unmount %s with error %s", cephRoot, err)
|
||||
} else {
|
||||
if err := os.Remove(cephRoot); err != nil {
|
||||
klog.Errorf("failed to remove %s with error %s", cephRoot, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -17,10 +17,15 @@ limitations under the License.
|
||||
package cephfs
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"sync"
|
||||
|
||||
"k8s.io/klog"
|
||||
)
|
||||
|
||||
const (
|
||||
@ -30,12 +35,20 @@ const (
|
||||
|
||||
var (
|
||||
availableMounters []string
|
||||
|
||||
// maps a mountpoint to PID of its FUSE daemon
|
||||
fusePidMap = make(map[string]int)
|
||||
fusePidMapMtx sync.Mutex
|
||||
|
||||
fusePidRx = regexp.MustCompile(`(?m)^ceph-fuse\[(.+)\]: starting fuse$`)
|
||||
)
|
||||
|
||||
// Load available ceph mounters installed on system into availableMounters
|
||||
// Called from driver.go's Run()
|
||||
func loadAvailableMounters() error {
|
||||
// #nosec
|
||||
fuseMounterProbe := exec.Command("ceph-fuse", "--version")
|
||||
// #nosec
|
||||
kernelMounterProbe := exec.Command("mount.ceph")
|
||||
|
||||
if fuseMounterProbe.Run() == nil {
|
||||
@ -47,14 +60,14 @@ func loadAvailableMounters() error {
|
||||
}
|
||||
|
||||
if len(availableMounters) == 0 {
|
||||
return fmt.Errorf("no ceph mounters found on system")
|
||||
return errors.New("no ceph mounters found on system")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type volumeMounter interface {
|
||||
mount(mountPoint string, cr *credentials, volOptions *volumeOptions, volId volumeID) error
|
||||
mount(mountPoint string, cr *credentials, volOptions *volumeOptions) error
|
||||
name() string
|
||||
}
|
||||
|
||||
@ -98,71 +111,84 @@ func newMounter(volOptions *volumeOptions) (volumeMounter, error) {
|
||||
|
||||
type fuseMounter struct{}
|
||||
|
||||
func mountFuse(mountPoint string, cr *credentials, volOptions *volumeOptions, volId volumeID) error {
|
||||
func mountFuse(mountPoint string, cr *credentials, volOptions *volumeOptions) error {
|
||||
args := [...]string{
|
||||
mountPoint,
|
||||
"-c", getCephConfPath(volId),
|
||||
"-n", cephEntityClientPrefix + cr.id,
|
||||
"--keyring", getCephKeyringPath(volId, cr.id),
|
||||
"-m", volOptions.Monitors,
|
||||
"-c", cephConfigPath,
|
||||
"-n", cephEntityClientPrefix + cr.id, "--key=" + cr.key,
|
||||
"-r", volOptions.RootPath,
|
||||
"-o", "nonempty",
|
||||
}
|
||||
|
||||
out, err := execCommand("ceph-fuse", args[:]...)
|
||||
_, stderr, err := execCommand("ceph-fuse", args[:]...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("cephfs: ceph-fuse failed with following error: %s\ncephfs: ceph-fuse output: %s", err, out)
|
||||
return err
|
||||
}
|
||||
|
||||
if !bytes.Contains(out, []byte("starting fuse")) {
|
||||
return fmt.Errorf("cephfs: ceph-fuse failed:\ncephfs: ceph-fuse output: %s", out)
|
||||
// Parse the output:
|
||||
// We need "starting fuse" meaning the mount is ok
|
||||
// and PID of the ceph-fuse daemon for unmount
|
||||
|
||||
match := fusePidRx.FindSubmatch(stderr)
|
||||
if len(match) != 2 {
|
||||
return fmt.Errorf("ceph-fuse failed: %s", stderr)
|
||||
}
|
||||
|
||||
pid, err := strconv.Atoi(string(match[1]))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to parse FUSE daemon PID: %v", err)
|
||||
}
|
||||
|
||||
fusePidMapMtx.Lock()
|
||||
fusePidMap[mountPoint] = pid
|
||||
fusePidMapMtx.Unlock()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *fuseMounter) mount(mountPoint string, cr *credentials, volOptions *volumeOptions, volId volumeID) error {
|
||||
func (m *fuseMounter) mount(mountPoint string, cr *credentials, volOptions *volumeOptions) error {
|
||||
if err := createMountPoint(mountPoint); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return mountFuse(mountPoint, cr, volOptions, volId)
|
||||
return mountFuse(mountPoint, cr, volOptions)
|
||||
}
|
||||
|
||||
func (m *fuseMounter) name() string { return "Ceph FUSE driver" }
|
||||
|
||||
type kernelMounter struct{}
|
||||
|
||||
func mountKernel(mountPoint string, cr *credentials, volOptions *volumeOptions, volId volumeID) error {
|
||||
if err := execCommandAndValidate("modprobe", "ceph"); err != nil {
|
||||
func mountKernel(mountPoint string, cr *credentials, volOptions *volumeOptions) error {
|
||||
if err := execCommandErr("modprobe", "ceph"); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return execCommandAndValidate("mount",
|
||||
return execCommandErr("mount",
|
||||
"-t", "ceph",
|
||||
fmt.Sprintf("%s:%s", volOptions.Monitors, volOptions.RootPath),
|
||||
mountPoint,
|
||||
"-o",
|
||||
fmt.Sprintf("name=%s,secretfile=%s", cr.id, getCephSecretPath(volId, cr.id)),
|
||||
"-o", fmt.Sprintf("name=%s,secret=%s", cr.id, cr.key),
|
||||
)
|
||||
}
|
||||
|
||||
func (m *kernelMounter) mount(mountPoint string, cr *credentials, volOptions *volumeOptions, volId volumeID) error {
|
||||
func (m *kernelMounter) mount(mountPoint string, cr *credentials, volOptions *volumeOptions) error {
|
||||
if err := createMountPoint(mountPoint); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return mountKernel(mountPoint, cr, volOptions, volId)
|
||||
return mountKernel(mountPoint, cr, volOptions)
|
||||
}
|
||||
|
||||
func (m *kernelMounter) name() string { return "Ceph kernel client" }
|
||||
|
||||
func bindMount(from, to string, readOnly bool) error {
|
||||
if err := execCommandAndValidate("mount", "--bind", from, to); err != nil {
|
||||
if err := execCommandErr("mount", "--bind", from, to); err != nil {
|
||||
return fmt.Errorf("failed to bind-mount %s to %s: %v", from, to, err)
|
||||
}
|
||||
|
||||
if readOnly {
|
||||
if err := execCommandAndValidate("mount", "-o", "remount,ro,bind", to); err != nil {
|
||||
if err := execCommandErr("mount", "-o", "remount,ro,bind", to); err != nil {
|
||||
return fmt.Errorf("failed read-only remount of %s: %v", to, err)
|
||||
}
|
||||
}
|
||||
@ -171,7 +197,29 @@ func bindMount(from, to string, readOnly bool) error {
|
||||
}
|
||||
|
||||
func unmountVolume(mountPoint string) error {
|
||||
return execCommandAndValidate("umount", mountPoint)
|
||||
if err := execCommandErr("umount", mountPoint); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fusePidMapMtx.Lock()
|
||||
pid, ok := fusePidMap[mountPoint]
|
||||
if ok {
|
||||
delete(fusePidMap, mountPoint)
|
||||
}
|
||||
fusePidMapMtx.Unlock()
|
||||
|
||||
if ok {
|
||||
p, err := os.FindProcess(pid)
|
||||
if err != nil {
|
||||
klog.Warningf("failed to find process %d: %v", pid, err)
|
||||
} else {
|
||||
if _, err = p.Wait(); err != nil {
|
||||
klog.Warningf("%d is not a child process: %v", pid, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func createMountPoint(root string) error {
|
||||
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user