mirrors/ceph-csi
1
0
Fork 0
mirror of https://github.com/ceph/ceph-csi.git synced 2024-12-18 02:50:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add E2E for cephfs resize functionality

Browse Source 
Signed-off-by: Humble Chirammal <hchiramm@redhat.com>
This commit is contained in:
Humble Chirammal 2019-11-25 16:39:24 +05:30 committed by mergify[bot]
parent daad8070ac
commit ac09c5553c
13 changed files with 184 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Gopkg.lock generated
View File

@ -1374,6 +1374,7 @@
"k8s.io/api/core/v1",
"k8s.io/api/storage/v1",
"k8s.io/apimachinery/pkg/api/errors",
"k8s.io/apimachinery/pkg/api/resource",
"k8s.io/apimachinery/pkg/apis/meta/v1",
"k8s.io/apimachinery/pkg/util/sets",
"k8s.io/apimachinery/pkg/util/validation",

19
charts/ceph-csi-cephfs/templates/provisioner-deployment.yaml
View File

@ -63,6 +63,25 @@ spec:
mountPath: /csi
resources:
{{ toYaml .Values.nodeplugin.plugin.resources | indent 12 }}
{{- end }}
{{- if semverCompare ">=1.15" .Capabilities.KubeVersion.GitVersion -}}
{{- if .Values.provisioner.resizer.enabled }}
- name: csi-resizer
image: "{{ .Values.provisioner.resizer.image.repository }}:{{ .Values.provisioner.resizer.image.tag }}"
imagePullPolicy: {{ .Values.provisioner.resizer.image.pullPolicy }}
args:
- "--v=5"
- "--csi-address=$(ADDRESS)"
- "--leader-election"
env:
- name: ADDRESS
value: "unix:///csi/{{ .Values.provisionerSocketFile }}"
volumeMounts:
- name: socket-dir
mountPath: /csi
resources:
{{ toYaml .Values.provisioner.resizer.resources | indent 12 }}
{{- end }}
{{- end }}
- name: csi-cephfsplugin
image: "{{ .Values.nodeplugin.plugin.image.repository }}:{{ .Values.nodeplugin.plugin.image.tag }}"

9
charts/ceph-csi-cephfs/templates/provisioner-rules-clusterrole.yaml
View File

@ -19,7 +19,7 @@ rules:
verbs: ["get", "list"]
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "create", "delete"]
verbs: ["get", "list", "watch", "create", "delete","patch"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "update"]
@ -37,4 +37,11 @@ rules:
resources: ["volumeattachments"]
verbs: ["get", "list", "watch", "update", "patch"]
{{- end -}}
{{- if semverCompare ">=1.15" .Capabilities.KubeVersion.GitVersion -}}
{{- if .Values.provisioner.resizer.enabled }}
- apiGroups: [""]
resources: ["persistentvolumeclaims/status"]
verbs: ["update", "patch"]
{{- end -}}
{{- end -}}
{{- end -}}

2
charts/ceph-csi-cephfs/values.yaml
View File

@ -203,7 +203,7 @@ provisioner:
enabled: true
image:
repository: quay.io/k8scsi/csi-resizer
tag: canary
tag: v0.3.0
pullPolicy: IfNotPresent
resources: {}

12
charts/ceph-csi-rbd/values.yaml
View File

@ -205,24 +205,12 @@ provisioner:
pullPolicy: IfNotPresent
resources: {}
resizer:
name: resizer
enabled: true
image:
repository: quay.io/k8scsi/csi-resizer
tag: canary
pullPolicy: IfNotPresent
resources: {}
nodeSelector: {}
tolerations: []
affinity: {}
#########################################################
# Variables for 'internal' use please use with caution! #
#########################################################

1
deploy/cephfs/kubernetes/v1.13/csi-provisioner-rbac.yaml
View File

@ -46,7 +46,6 @@ rules:
- apiGroups: ["storage.k8s.io"]
resources: ["volumeattachments"]
verbs: ["get", "list", "watch", "update"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1

15
deploy/cephfs/kubernetes/v1.14+/csi-cephfsplugin-provisioner.yaml
View File

@ -40,7 +40,7 @@ spec:
args:
- "--csi-address=$(ADDRESS)"
- "--v=5"
- "--timeout=150s"
- "--timeout=60s"
- "--enable-leader-election=true"
- "--leader-election-type=leases"
- "--retry-interval-start=500ms"
@ -51,6 +51,19 @@ spec:
volumeMounts:
- name: socket-dir
mountPath: /csi
- name: csi-resizer
image: quay.io/k8scsi/csi-resizer:v0.3.0
args:
- "--csi-address=$(ADDRESS)"
- "--v=5"
- "--leader-election"
env:
- name: ADDRESS
value: unix:///csi/csi-provisioner.sock
imagePullPolicy: "IfNotPresent"
volumeMounts:
- name: socket-dir
mountPath: /csi
- name: csi-cephfsplugin-attacher
image: quay.io/k8scsi/csi-attacher:v1.2.0
args:

5
deploy/cephfs/kubernetes/v1.14+/csi-provisioner-rbac.yaml
View File

@ -33,7 +33,7 @@ rules:
verbs: ["list", "watch", "create", "update", "patch"]
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "create", "delete"]
verbs: ["get", "list", "watch", "create", "delete", "patch"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "update"]
@ -46,6 +46,9 @@ rules:
- apiGroups: ["storage.k8s.io"]
resources: ["volumeattachments"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: [""]
resources: ["persistentvolumeclaims/status"]
verbs: ["update", "patch"]
---
kind: ClusterRoleBinding

17
e2e/cephfs.go
View File

@ -157,6 +157,23 @@ var _ = Describe("cephfs", func() {
Fail(err.Error())
}
})
By("Resize PVC and check application directory size", func() {
v, err := f.ClientSet.Discovery().ServerVersion()
if err != nil {
e2elog.Logf("failed to get server version with error %v", err)
Fail(err.Error())
}
// Resize 0.3.0 is only supported from v1.15+
if v.Major > "1" || (v.Major == "1" && v.Minor >= "15") {
err := resizePVCAndValidateSize(pvcPath, appPath, f)
if err != nil {
e2elog.Logf("failed to resize PVC %v", err)
Fail(err.Error())
}
}
})
})

110
e2e/utils.go
View File

@ -18,6 +18,7 @@ import (
v1 "k8s.io/api/core/v1"
scv1 "k8s.io/api/storage/v1"
apierrs "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
utilyaml "k8s.io/apimachinery/pkg/util/yaml"
@ -797,8 +798,8 @@ func checkDataPersist(pvcPath, appPath string, f *framework.Framework) error {
if pvc == nil {
return err
}
pvc.Namespace = f.UniqueName
e2elog.Logf("The PVC template %+v", pvc)
app, err := loadApp(appPath)
if err != nil {
@ -839,3 +840,110 @@ func checkDataPersist(pvcPath, appPath string, f *framework.Framework) error {
err = deletePVCAndApp("", f, pvc, app)
return err
}
func expandPVCSize(c kubernetes.Interface, pvc *v1.PersistentVolumeClaim, size string, t int) error {
pvcName := pvc.Name
updatedPVC := pvc.DeepCopy()
var err error
updatedPVC, err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Get(pvcName, metav1.GetOptions{})
if err != nil {
return fmt.Errorf("error fetching pvc %q with %v", pvcName, err)
}
timeout := time.Duration(t) * time.Minute
updatedPVC.Spec.Resources.Requests[v1.ResourceStorage] = resource.MustParse(size)
_, err = c.CoreV1().PersistentVolumeClaims(updatedPVC.Namespace).Update(updatedPVC)
Expect(err).Should(BeNil())
start := time.Now()
e2elog.Logf("Waiting up to %v to be in Resized state", pvc)
return wait.PollImmediate(poll, timeout, func() (bool, error) {
e2elog.Logf("waiting for PVC %s (%d seconds elapsed)", updatedPVC.Name, int(time.Since(start).Seconds()))
updatedPVC, err = c.CoreV1().PersistentVolumeClaims(updatedPVC.Namespace).Get(pvcName, metav1.GetOptions{})
if err != nil {
e2elog.Logf("Error getting pvc in namespace: '%s': %v", updatedPVC.Namespace, err)
if testutils.IsRetryableAPIError(err) {
return false, nil
}
return false, err
}
pvcConditions := updatedPVC.Status.Conditions
if len(pvcConditions) > 0 {
if pvcConditions[0].Type == v1.PersistentVolumeClaimResizing {
return true, nil
}
e2elog.Logf("pvc state %v", pvcConditions[0].Type)
}
return false, nil
})
}
func resizePVCAndValidateSize(pvcPath, appPath string, f *framework.Framework) error {
pvc, err := loadPVC(pvcPath)
if pvc == nil {
return err
}
pvc.Namespace = f.UniqueName
resizePvc, err := loadPVC(pvcPath)
if resizePvc == nil {
return err
}
resizePvc.Namespace = f.UniqueName
app, err := loadApp(appPath)
if err != nil {
return err
}
app.Labels = map[string]string{"app": "resize-pvc"}
app.Namespace = f.UniqueName
err = createPVCAndApp("", f, pvc, app)
if err != nil {
return err
}
opt := metav1.ListOptions{
LabelSelector: "app=resize-pvc",
}
err = checkDirSize(app, f, &opt, "5.0G", deployTimeout)
if err != nil {
return err
}
// resize PVC
err = expandPVCSize(f.ClientSet, resizePvc, "10Gi", deployTimeout)
if err != nil {
return err
}
// wait for application pod to come up after resize
err = waitForPodInRunningState(app.Name, app.Namespace, f.ClientSet, deployTimeout)
if err != nil {
return err
}
err = checkDirSize(app, f, &opt, "10G", deployTimeout)
return err
}
func checkDirSize(app *v1.Pod, f *framework.Framework, opt *metav1.ListOptions, size string, t int) error {
dirPath := app.Spec.Containers[0].VolumeMounts[0].MountPath
timeout := time.Duration(t) * time.Minute
start := time.Now()
return wait.PollImmediate(poll, timeout, func() (bool, error) {
e2elog.Logf("checking directory size %s (%d seconds elapsed)", dirPath, int(time.Since(start).Seconds()))
output, stdErr := execCommandInPod(f, fmt.Sprintf("df -h|grep %s |awk '{print $2}'", dirPath), app.Namespace, opt)
if stdErr != "" {
e2elog.Logf("failed to execute command in app pod %v", stdErr)
return false, nil
}
if !strings.Contains(output, size) {
e2elog.Logf("expected directory size %s found %s information", size, output)
return false, nil
}
return true, nil
})
}

3
examples/cephfs/storageclass.yaml
View File

@ -31,6 +31,8 @@ parameters:
# The secrets have to contain user and/or Ceph admin credentials.
csi.storage.k8s.io/provisioner-secret-name: csi-cephfs-secret
csi.storage.k8s.io/provisioner-secret-namespace: default
csi.storage.k8s.io/controller-expand-secret-name: csi-cephfs-secret
csi.storage.k8s.io/controller-expand-secret-namespace: default
csi.storage.k8s.io/node-stage-secret-name: csi-cephfs-secret
csi.storage.k8s.io/node-stage-secret-namespace: default
@ -40,5 +42,6 @@ parameters:
# determined by probing for ceph-fuse and mount.ceph
# mounter: kernel
reclaimPolicy: Delete
allowVolumeExpansion: true
mountOptions:
- debug

7
pkg/cephfs/controllerserver.go
View File

@ -304,6 +304,13 @@ func (cs *ControllerServer) ControllerExpandVolume(ctx context.Context, req *csi
volID := req.GetVolumeId()
secret := req.GetSecrets()
// lock out parallel delete operations
if acquired := cs.VolumeLocks.TryAcquire(volID); !acquired {
klog.Infof(util.Log(ctx, util.VolumeOperationAlreadyExistsFmt), volID)
return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, volID)
}
defer cs.VolumeLocks.Release(volID)
cr, err := util.NewAdminCredentials(secret)
if err != nil {
return nil, status.Error(codes.InvalidArgument, err.Error())

2
scripts/minikube.sh
View File

@ -63,7 +63,7 @@ CEPHCSI_IMAGE_REPO=${CEPHCSI_IMAGE_REPO:-"quay.io/cephcsi"}
K8S_IMAGE_REPO=${K8S_IMAGE_REPO:-"quay.io/k8scsi"}
#feature-gates for kube
K8S_FEATURE_GATES=${K8S_FEATURE_GATES:-"BlockVolume=true,CSIBlockVolume=true,VolumeSnapshotDataSource=true"}
K8S_FEATURE_GATES=${K8S_FEATURE_GATES:-"BlockVolume=true,CSIBlockVolume=true,VolumeSnapshotDataSource=true,ExpandCSIVolumes=true,ExpandInUsePersistentVolumes=true"}
case "${1:-}" in
up)