From 1a8f8e3c243967b0598ab1c864f4c605420cefa6 Mon Sep 17 00:00:00 2001 From: ShyamsundarR Date: Mon, 6 Apr 2020 16:19:13 -0400 Subject: [PATCH] Add support for erasure coded pools This commit adds support to mention dataPool parameter for the topology constrained pools in the StorageClass, that can be leveraged to mention erasure coded pool names to use for RBD data instead of the replica pools. Signed-off-by: ShyamsundarR --- e2e/README.md | 22 +++++++------- e2e/rbd.go | 54 +++++++++++++++++++++++++++------- e2e/utils.go | 25 ++++++++++++++-- examples/rbd/storageclass.yaml | 23 ++++++++------- pkg/cephfs/fsjournal.go | 2 +- pkg/rbd/rbd_journal.go | 3 +- pkg/util/topology.go | 27 +++++++++-------- pkg/util/topology_test.go | 47 ++++++++++++++++++----------- scripts/install-helm.sh | 22 +++++++++++++- scripts/rook.sh | 28 +++++------------- scripts/travis-helmtest.sh | 1 + 11 files changed, 166 insertions(+), 88 deletions(-) diff --git a/e2e/README.md b/e2e/README.md index 9efff5f17..63ca67c0d 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -30,7 +30,7 @@ cluster. the following parameters are available to configure kubernetes cluster -| flag | description | +| flag | description | | ----------------- | ------------------------------------------------------------- | | up | Starts a local kubernetes cluster and prepare a disk for rook | | down | Stops a running local kubernetes cluster | @@ -45,16 +45,16 @@ the following parameters are available to configure kubernetes cluster following environment variables can be exported to customize kubernetes deployment -| ENV | Description | Default | -| ------------------ | ------------------------------------------------ | ------------------------------------------------------------------ | -| MINIKUBE_VERSION | minikube version to install | latest | -| KUBE_VERSION | kubernetes version to install | v1.14.10 | -| MEMORY | Amount of RAM allocated to the minikube VM in MB | 3000 | -| VM_DRIVER | VM driver to create virtual machine | virtualbox | -| CEPHCSI_IMAGE_REPO | Repo URL to pull cephcsi images | quay.io/cephcsi | -| K8S_IMAGE_REPO | Repo URL to pull kubernetes sidecar images | quay.io/k8scsi | -| K8S_FEATURE_GATES | Feature gates to enable on kubernetes cluster | BlockVolume=true,CSIBlockVolume=true,VolumeSnapshotDataSource=true | -| ROOK_BLOCK_POOL_NAME | Block pool name to create in the rook instance | newrbdpool | +| ENV | Description | Default | +|----------------------|--------------------------------------------------|--------------------------------------------------------------------| +| MINIKUBE_VERSION | minikube version to install | latest | +| KUBE_VERSION | kubernetes version to install | v1.14.10 | +| MEMORY | Amount of RAM allocated to the minikube VM in MB | 3000 | +| VM_DRIVER | VM driver to create virtual machine | virtualbox | +| CEPHCSI_IMAGE_REPO | Repo URL to pull cephcsi images | quay.io/cephcsi | +| K8S_IMAGE_REPO | Repo URL to pull kubernetes sidecar images | quay.io/k8scsi | +| K8S_FEATURE_GATES | Feature gates to enable on kubernetes cluster | BlockVolume=true,CSIBlockVolume=true,VolumeSnapshotDataSource=true | +| ROOK_BLOCK_POOL_NAME | Block pool name to create in the rook instance | newrbdpool | - creating kubernetes cluster diff --git a/e2e/rbd.go b/e2e/rbd.go index 677a8cb72..fc8083066 100644 --- a/e2e/rbd.go +++ b/e2e/rbd.go @@ -23,13 +23,14 @@ var ( rbdDeploymentName = "csi-rbdplugin-provisioner" rbdDaemonsetName = "csi-rbdplugin" // Topology related variables - nodeRegionLabel = "test.failure-domain/region" - regionValue = "testregion" - nodeZoneLabel = "test.failure-domain/zone" - zoneValue = "testzone" - nodeCSIRegionLabel = "topology.rbd.csi.ceph.com/region" - nodeCSIZoneLabel = "topology.rbd.csi.ceph.com/zone" - rbdTopologyPool = "newrbdpool" + nodeRegionLabel = "test.failure-domain/region" + regionValue = "testregion" + nodeZoneLabel = "test.failure-domain/zone" + zoneValue = "testzone" + nodeCSIRegionLabel = "topology.rbd.csi.ceph.com/region" + nodeCSIZoneLabel = "topology.rbd.csi.ceph.com/zone" + rbdTopologyPool = "newrbdpool" + rbdTopologyDataPool = "replicapool" // NOTE: should be different than rbdTopologyPool for test to be effective ) func deployRBDPlugin() { @@ -125,9 +126,9 @@ var _ = Describe("RBD", func() { // deploy RBD CSI BeforeEach(func() { c = f.ClientSet - createNodeLabel(f, nodeRegionLabel, regionValue) - createNodeLabel(f, nodeZoneLabel, zoneValue) if deployRBD { + createNodeLabel(f, nodeRegionLabel, regionValue) + createNodeLabel(f, nodeZoneLabel, zoneValue) if cephCSINamespace != defaultNs { err := createNamespace(c, cephCSINamespace) if err != nil { @@ -519,11 +520,44 @@ var _ = Describe("RBD", func() { Fail(err.Error()) } - // cleanup and undo changes made by the test err = deletePVCAndApp("", f, pvc, app) if err != nil { Fail(err.Error()) } + + By("checking if data pool parameter is honored", func() { + deleteResource(rbdExamplePath + "storageclass.yaml") + topologyConstraint := "[{\"poolName\":\"" + rbdTopologyPool + "\",\"dataPool\":\"" + rbdTopologyDataPool + + "\",\"domainSegments\":" + + "[{\"domainLabel\":\"region\",\"value\":\"" + regionValue + "\"}," + + "{\"domainLabel\":\"zone\",\"value\":\"" + zoneValue + "\"}]}]" + createRBDStorageClass(f.ClientSet, f, + map[string]string{"volumeBindingMode": "WaitForFirstConsumer"}, + map[string]string{"topologyConstrainedPools": topologyConstraint}) + + By("creating an app using a PV from the delayed binding mode StorageClass with a data pool") + pvc, app = createPVCAndAppBinding(pvcPath, appPath, f, 0) + + By("ensuring created PV has its image in the topology specific pool") + err = checkPVCImageInPool(f, pvc, rbdTopologyPool) + if err != nil { + Fail(err.Error()) + } + + By("ensuring created image has the right data pool parameter set") + err = checkPVCDataPoolForImageInPool(f, pvc, rbdTopologyPool, rbdTopologyDataPool) + if err != nil { + Fail(err.Error()) + } + + // cleanup and undo changes made by the test + err = deletePVCAndApp("", f, pvc, app) + if err != nil { + Fail(err.Error()) + } + }) + + // cleanup and undo changes made by the test deleteResource(rbdExamplePath + "storageclass.yaml") createRBDStorageClass(f.ClientSet, f, nil, nil) }) diff --git a/e2e/utils.go b/e2e/utils.go index 02be1c18f..71ea671f0 100644 --- a/e2e/utils.go +++ b/e2e/utils.go @@ -1234,21 +1234,40 @@ func checkNodeHasLabel(c clientset.Interface, labelKey, labelValue string) { } } -func checkPVCImageInPool(f *framework.Framework, pvc *v1.PersistentVolumeClaim, pool string) error { +func getPVCImageInfoInPool(f *framework.Framework, pvc *v1.PersistentVolumeClaim, pool string) (string, error) { imageData, err := getImageInfoFromPVC(pvc.Namespace, pvc.Name, f) if err != nil { - return err + return "", err } opt := metav1.ListOptions{ LabelSelector: "app=rook-ceph-tools", } - _, stdErr := execCommandInPod(f, "rbd info "+pool+"/"+imageData.imageName, rookNamespace, &opt) + stdOut, stdErr := execCommandInPod(f, "rbd info "+pool+"/"+imageData.imageName, rookNamespace, &opt) Expect(stdErr).Should(BeEmpty()) e2elog.Logf("found image %s in pool %s", imageData.imageName, pool) + return stdOut, nil +} + +func checkPVCImageInPool(f *framework.Framework, pvc *v1.PersistentVolumeClaim, pool string) error { + _, err := getPVCImageInfoInPool(f, pvc, pool) + + return err +} + +func checkPVCDataPoolForImageInPool(f *framework.Framework, pvc *v1.PersistentVolumeClaim, pool, dataPool string) error { + stdOut, err := getPVCImageInfoInPool(f, pvc, pool) + if err != nil { + return err + } + + if !strings.Contains(stdOut, "data_pool: "+dataPool) { + return fmt.Errorf("missing data pool value in image info, got info (%s)", stdOut) + } + return nil } diff --git a/examples/rbd/storageclass.yaml b/examples/rbd/storageclass.yaml index 61b179a2b..0b0124574 100644 --- a/examples/rbd/storageclass.yaml +++ b/examples/rbd/storageclass.yaml @@ -58,18 +58,21 @@ parameters: # For further information read TODO # topologyConstrainedPools: | # [{"poolName":"pool0", - # "domainSegments":[ - # {"domainLabel":"region","value":"east"}, - # {"domainLabel":"zone","value":"zone1"}]}, + # "dataPool":"ec-pool0" # optional, erasure-coded pool for data + # "domainSegments":[ + # {"domainLabel":"region","value":"east"}, + # {"domainLabel":"zone","value":"zone1"}]}, # {"poolName":"pool1", - # "domainSegments":[ - # {"domainLabel":"region","value":"east"}, - # {"domainLabel":"zone","value":"zone2"}]}, + # "dataPool":"ec-pool1" # optional, erasure-coded pool for data + # "domainSegments":[ + # {"domainLabel":"region","value":"east"}, + # {"domainLabel":"zone","value":"zone2"}]}, # {"poolName":"pool2", - # "domainSegments":[ - # {"domainLabel":"region","value":"west"}, - # {"domainLabel":"zone","value":"zone1"}]} - # ] + # "dataPool":"ec-pool2" # optional, erasure-coded pool for data + # "domainSegments":[ + # {"domainLabel":"region","value":"west"}, + # {"domainLabel":"zone","value":"zone1"}]} + # ] reclaimPolicy: Delete allowVolumeExpansion: true diff --git a/pkg/cephfs/fsjournal.go b/pkg/cephfs/fsjournal.go index b462cd05d..a3e8ea711 100644 --- a/pkg/cephfs/fsjournal.go +++ b/pkg/cephfs/fsjournal.go @@ -110,7 +110,7 @@ func undoVolReservation(ctx context.Context, volOptions *volumeOptions, vid volu func updateTopologyConstraints(volOpts *volumeOptions) error { // update request based on topology constrained parameters (if present) - poolName, topology, err := util.FindPoolAndTopology(volOpts.TopologyPools, volOpts.TopologyRequirement) + poolName, _, topology, err := util.FindPoolAndTopology(volOpts.TopologyPools, volOpts.TopologyRequirement) if err != nil { return err } diff --git a/pkg/rbd/rbd_journal.go b/pkg/rbd/rbd_journal.go index d202af5d0..d4cb5f126 100644 --- a/pkg/rbd/rbd_journal.go +++ b/pkg/rbd/rbd_journal.go @@ -282,12 +282,13 @@ func updateTopologyConstraints(rbdVol *rbdVolume, rbdSnap *rbdSnapshot) error { return nil } // update request based on topology constrained parameters (if present) - poolName, topology, err := util.FindPoolAndTopology(rbdVol.TopologyPools, rbdVol.TopologyRequirement) + poolName, dataPoolName, topology, err := util.FindPoolAndTopology(rbdVol.TopologyPools, rbdVol.TopologyRequirement) if err != nil { return err } if poolName != "" { rbdVol.Pool = poolName + rbdVol.DataPool = dataPoolName rbdVol.Topology = topology } diff --git a/pkg/util/topology.go b/pkg/util/topology.go index 16f535674..9069f2a89 100644 --- a/pkg/util/topology.go +++ b/pkg/util/topology.go @@ -125,6 +125,7 @@ type topologySegment struct { // TopologyConstrainedPool stores the pool name and a list of its associated topology domain values type TopologyConstrainedPool struct { PoolName string `json:"poolName"` + DataPoolName string `json:"dataPool"` DomainSegments []topologySegment `json:"domainSegments"` } @@ -178,7 +179,7 @@ func MatchTopologyForPool(topologyPools *[]TopologyConstrainedPool, topologyPools, poolName) } - _, topology, err := FindPoolAndTopology(&topologyPool, accessibilityRequirements) + _, _, topology, err := FindPoolAndTopology(&topologyPool, accessibilityRequirements) return topology, err } @@ -188,28 +189,28 @@ func MatchTopologyForPool(topologyPools *[]TopologyConstrainedPool, // The return variables are, image poolname, data poolname, and topology map of // matched requirement func FindPoolAndTopology(topologyPools *[]TopologyConstrainedPool, - accessibilityRequirements *csi.TopologyRequirement) (string, map[string]string, error) { + accessibilityRequirements *csi.TopologyRequirement) (string, string, map[string]string, error) { if topologyPools == nil || accessibilityRequirements == nil { - return "", nil, nil + return "", "", nil, nil } // select pool that fits first topology constraint preferred requirements for _, topology := range accessibilityRequirements.GetPreferred() { - poolName := matchPoolToTopology(topologyPools, topology) - if poolName != "" { - return poolName, topology.GetSegments(), nil + topologyPool := matchPoolToTopology(topologyPools, topology) + if topologyPool.PoolName != "" { + return topologyPool.PoolName, topologyPool.DataPoolName, topology.GetSegments(), nil } } // If preferred mismatches, check requisite for a fit for _, topology := range accessibilityRequirements.GetRequisite() { - poolName := matchPoolToTopology(topologyPools, topology) - if poolName != "" { - return poolName, topology.GetSegments(), nil + topologyPool := matchPoolToTopology(topologyPools, topology) + if topologyPool.PoolName != "" { + return topologyPool.PoolName, topologyPool.DataPoolName, topology.GetSegments(), nil } } - return "", nil, fmt.Errorf("none of the topology constrained pools matched requested "+ + return "", "", nil, fmt.Errorf("none of the topology constrained pools matched requested "+ "topology constraints : pools (%+v) requested topology (%+v)", *topologyPools, *accessibilityRequirements) } @@ -217,7 +218,7 @@ func FindPoolAndTopology(topologyPools *[]TopologyConstrainedPool, // matchPoolToTopology loops through passed in pools, and for each pool checks if all // requested topology segments are present and match the request, returning the first pool // that hence matches (or an empty string if none match) -func matchPoolToTopology(topologyPools *[]TopologyConstrainedPool, topology *csi.Topology) string { +func matchPoolToTopology(topologyPools *[]TopologyConstrainedPool, topology *csi.Topology) TopologyConstrainedPool { domainMap := extractDomainsFromlabels(topology) // check if any pool matches all the domain keys and values @@ -235,10 +236,10 @@ func matchPoolToTopology(topologyPools *[]TopologyConstrainedPool, topology *csi continue } - return topologyPool.PoolName + return topologyPool } - return "" + return TopologyConstrainedPool{} } // extractDomainsFromlabels returns the domain name map, from passed in domain segments, diff --git a/pkg/util/topology_test.go b/pkg/util/topology_test.go index 1c1d1c40c..8e77d61dd 100644 --- a/pkg/util/topology_test.go +++ b/pkg/util/topology_test.go @@ -224,91 +224,91 @@ func TestFindPoolAndTopology(t *testing.T) { return nil } // Test nil values - _, _, err = FindPoolAndTopology(nil, nil) + _, _, _, err = FindPoolAndTopology(nil, nil) if err != nil { t.Errorf("expected success due to nil in-args (%v)", err) } - poolName, _, err := FindPoolAndTopology(&validMultipleTopoPools, nil) + poolName, _, _, err := FindPoolAndTopology(&validMultipleTopoPools, nil) if err != nil || poolName != "" { t.Errorf("expected success due to nil accessibility requirements (err - %v) (poolName - %s)", err, poolName) } - poolName, _, err = FindPoolAndTopology(nil, &validAccReq) + poolName, _, _, err = FindPoolAndTopology(nil, &validAccReq) if err != nil || poolName != "" { t.Errorf("expected success due to nil topology pools (err - %v) (poolName - %s)", err, poolName) } // Test valid accessibility requirement, with invalid topology pools values - _, _, err = FindPoolAndTopology(&emptyTopoPools, &validAccReq) + _, _, _, err = FindPoolAndTopology(&emptyTopoPools, &validAccReq) if err == nil { t.Errorf("expected failure due to empty topology pools") } - _, _, err = FindPoolAndTopology(&emptyPoolNameTopoPools, &validAccReq) + _, _, _, err = FindPoolAndTopology(&emptyPoolNameTopoPools, &validAccReq) if err == nil { t.Errorf("expected failure due to missing pool name in topology pools") } - _, _, err = FindPoolAndTopology(&differentDomainsInTopoPools, &validAccReq) + _, _, _, err = FindPoolAndTopology(&differentDomainsInTopoPools, &validAccReq) if err == nil { t.Errorf("expected failure due to mismatching domains in topology pools") } // Test valid topology pools, with invalid accessibility requirements - _, _, err = FindPoolAndTopology(&validMultipleTopoPools, &emptyAccReq) + _, _, _, err = FindPoolAndTopology(&validMultipleTopoPools, &emptyAccReq) if err == nil { t.Errorf("expected failure due to empty accessibility requirements") } - _, _, err = FindPoolAndTopology(&validSingletonTopoPools, &emptySegmentAccReq) + _, _, _, err = FindPoolAndTopology(&validSingletonTopoPools, &emptySegmentAccReq) if err == nil { t.Errorf("expected failure due to empty segments in accessibility requirements") } - _, _, err = FindPoolAndTopology(&validMultipleTopoPools, &partialHigherSegmentAccReq) + _, _, _, err = FindPoolAndTopology(&validMultipleTopoPools, &partialHigherSegmentAccReq) if err == nil { t.Errorf("expected failure due to partial segments in accessibility requirements") } - _, _, err = FindPoolAndTopology(&validSingletonTopoPools, &partialLowerSegmentAccReq) + _, _, _, err = FindPoolAndTopology(&validSingletonTopoPools, &partialLowerSegmentAccReq) if err == nil { t.Errorf("expected failure due to partial segments in accessibility requirements") } - _, _, err = FindPoolAndTopology(&validMultipleTopoPools, &partialLowerSegmentAccReq) + _, _, _, err = FindPoolAndTopology(&validMultipleTopoPools, &partialLowerSegmentAccReq) if err == nil { t.Errorf("expected failure due to partial segments in accessibility requirements") } - _, _, err = FindPoolAndTopology(&validMultipleTopoPools, &differentSegmentAccReq) + _, _, _, err = FindPoolAndTopology(&validMultipleTopoPools, &differentSegmentAccReq) if err == nil { t.Errorf("expected failure due to mismatching segments in accessibility requirements") } // Test success cases // If a pool is a superset of domains (either empty domain labels or partial), it can be selected - poolName, topoSegment, err := FindPoolAndTopology(&emptyDomainsInTopoPools, &validAccReq) + poolName, _, topoSegment, err := FindPoolAndTopology(&emptyDomainsInTopoPools, &validAccReq) err = checkOutput(err, poolName, topoSegment) if err != nil { t.Errorf("expected success got: (%v)", err) } - poolName, topoSegment, err = FindPoolAndTopology(&partialDomainsInTopoPools, &validAccReq) + poolName, _, topoSegment, err = FindPoolAndTopology(&partialDomainsInTopoPools, &validAccReq) err = checkOutput(err, poolName, topoSegment) if err != nil { t.Errorf("expected success got: (%v)", err) } // match in a singleton topology pools - poolName, topoSegment, err = FindPoolAndTopology(&validSingletonTopoPools, &validAccReq) + poolName, _, topoSegment, err = FindPoolAndTopology(&validSingletonTopoPools, &validAccReq) err = checkOutput(err, poolName, topoSegment) if err != nil { t.Errorf("expected success got: (%v)", err) } // match first in multiple topology pools - poolName, topoSegment, err = FindPoolAndTopology(&validMultipleTopoPools, &validAccReq) + poolName, _, topoSegment, err = FindPoolAndTopology(&validMultipleTopoPools, &validAccReq) err = checkOutput(err, poolName, topoSegment) if err != nil { t.Errorf("expected success got: (%v)", err) @@ -317,12 +317,25 @@ func TestFindPoolAndTopology(t *testing.T) { // match non-first in multiple topology pools switchPoolOrder := []TopologyConstrainedPool{} switchPoolOrder = append(switchPoolOrder, validMultipleTopoPools[1], validMultipleTopoPools[0]) - poolName, topoSegment, err = FindPoolAndTopology(&switchPoolOrder, &validAccReq) + poolName, _, topoSegment, err = FindPoolAndTopology(&switchPoolOrder, &validAccReq) err = checkOutput(err, poolName, topoSegment) if err != nil { t.Errorf("expected success got: (%v)", err) } + // test valid dataPool return + for i := range switchPoolOrder { + switchPoolOrder[i].DataPoolName = "ec-" + switchPoolOrder[i].PoolName + } + poolName, dataPoolName, topoSegment, err := FindPoolAndTopology(&switchPoolOrder, &validAccReq) + err = checkOutput(err, poolName, topoSegment) + if err != nil { + t.Errorf("expected success got: (%v)", err) + } + if dataPoolName != "ec-"+poolName { + t.Errorf("expected data pool to be named ec-%s, got %s", poolName, dataPoolName) + } + // TEST: MatchTopologyForPool // check for non-existent pool _, err = MatchTopologyForPool(&validMultipleTopoPools, &validAccReq, pool1+"fuzz") diff --git a/scripts/install-helm.sh b/scripts/install-helm.sh index 0c4eff357..a6b6a0575 100755 --- a/scripts/install-helm.sh +++ b/scripts/install-helm.sh @@ -12,6 +12,12 @@ RBD_CHART_NAME="ceph-csi-rbd" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" DEPLOY_TIMEOUT=600 +# ceph-csi specific variables +NODE_LABEL_REGION="test.failure-domain/region" +NODE_LABEL_ZONE="test.failure-domain/zone" +REGION_VALUE="testregion" +ZONE_VALUE="testzone" + function check_deployment_status() { LABEL=$1 NAMESPACE=$2 @@ -130,6 +136,13 @@ install_cephcsi_helm_charts() { if [ -z "$NAMESPACE" ]; then NAMESPACE="default" fi + + # label the nodes uniformly for domain information + for node in $(kubectl get node -o jsonpath='{.items[*].metadata.name}'); do + kubectl label node/"${node}" ${NODE_LABEL_REGION}=${REGION_VALUE} + kubectl label node/"${node}" ${NODE_LABEL_ZONE}=${ZONE_VALUE} + done + # install ceph-csi-cephfs and ceph-csi-rbd charts "${HELM}" install "${SCRIPT_DIR}"/../charts/ceph-csi-cephfs --name ${CEPHFS_CHART_NAME} --namespace ${NAMESPACE} --set provisioner.fullnameOverride=csi-cephfsplugin-provisioner --set nodeplugin.fullnameOverride=csi-cephfsplugin --set configMapName=ceph-csi-config --set provisioner.podSecurityPolicy.enabled=true --set nodeplugin.podSecurityPolicy.enabled=true @@ -139,7 +152,7 @@ install_cephcsi_helm_charts() { # deleting configmap as a workaround to avoid configmap already present # issue when installing ceph-csi-rbd kubectl delete cm ceph-csi-config --namespace ${NAMESPACE} - "${HELM}" install "${SCRIPT_DIR}"/../charts/ceph-csi-rbd --name ${RBD_CHART_NAME} --namespace ${NAMESPACE} --set provisioner.fullnameOverride=csi-rbdplugin-provisioner --set nodeplugin.fullnameOverride=csi-rbdplugin --set configMapName=ceph-csi-config --set provisioner.podSecurityPolicy.enabled=true --set nodeplugin.podSecurityPolicy.enabled=true + "${HELM}" install "${SCRIPT_DIR}"/../charts/ceph-csi-rbd --name ${RBD_CHART_NAME} --namespace ${NAMESPACE} --set provisioner.fullnameOverride=csi-rbdplugin-provisioner --set nodeplugin.fullnameOverride=csi-rbdplugin --set configMapName=ceph-csi-config --set provisioner.podSecurityPolicy.enabled=true --set nodeplugin.podSecurityPolicy.enabled=true --set topology.enabled=true --set topology.domainLabels="{${NODE_LABEL_REGION},${NODE_LABEL_ZONE}}" check_deployment_status app=ceph-csi-rbd ${NAMESPACE} check_daemonset_status app=ceph-csi-rbd ${NAMESPACE} @@ -149,6 +162,13 @@ install_cephcsi_helm_charts() { cleanup_cephcsi_helm_charts() { "${HELM}" del --purge ${CEPHFS_CHART_NAME} "${HELM}" del --purge ${RBD_CHART_NAME} + + # remove set labels + for node in $(kubectl get node --no-headers | cut -f 1 -d ' '); do + kubectl label node/"$node" test.failure-domain/region- + kubectl label node/"$node" test.failure-domain/zone- + done + # TODO/LATER we could remove the CSI labels that would have been set as well } helm_reset() { diff --git a/scripts/rook.sh b/scripts/rook.sh index 2ad54a716..c50147cde 100755 --- a/scripts/rook.sh +++ b/scripts/rook.sh @@ -25,7 +25,7 @@ function deploy_rook() { # Check if CephBlockPool is empty if ! kubectl -n rook-ceph get cephblockpools -oyaml | grep 'items: \[\]' &>/dev/null; then - check_rbd_stat + check_rbd_stat "" fi } @@ -44,25 +44,7 @@ function create_block_pool() { kubectl create -f "./newpool.yaml" rm -f "./newpool.yaml" - for ((retry = 0; retry <= ROOK_DEPLOY_TIMEOUT; retry = retry + 5)); do - echo "Checking RBD ($ROOK_BLOCK_POOL_NAME) stats... ${retry}s" && sleep 5 - - TOOLBOX_POD=$(kubectl -n rook-ceph get pods -l app=rook-ceph-tools -o jsonpath='{.items[0].metadata.name}') - TOOLBOX_POD_STATUS=$(kubectl -n rook-ceph get pod "$TOOLBOX_POD" -ojsonpath='{.status.phase}') - [[ "$TOOLBOX_POD_STATUS" != "Running" ]] && \ - { echo "Toolbox POD ($TOOLBOX_POD) status: [$TOOLBOX_POD_STATUS]"; continue; } - - if kubectl exec -n rook-ceph "$TOOLBOX_POD" -it -- rbd pool stats "$ROOK_BLOCK_POOL_NAME" &>/dev/null; then - echo "RBD ($ROOK_BLOCK_POOL_NAME) is successfully created..." - break - fi - done - - if [ "$retry" -gt "$ROOK_DEPLOY_TIMEOUT" ]; then - echo "[Timeout] Failed to get RBD pool $ROOK_BLOCK_POOL_NAME stats" - exit 1 - fi - echo "" + check_rbd_stat "$ROOK_BLOCK_POOL_NAME" } function delete_block_pool() { @@ -122,7 +104,11 @@ function check_mds_stat() { function check_rbd_stat() { for ((retry = 0; retry <= ROOK_DEPLOY_TIMEOUT; retry = retry + 5)); do - RBD_POOL_NAME=$(kubectl -n rook-ceph get cephblockpools -ojsonpath='{.items[0].metadata.name}') + if [ -z "$1" ]; then + RBD_POOL_NAME=$(kubectl -n rook-ceph get cephblockpools -ojsonpath='{.items[0].metadata.name}') + else + RBD_POOL_NAME=$1 + fi echo "Checking RBD ($RBD_POOL_NAME) stats... ${retry}s" && sleep 5 TOOLBOX_POD=$(kubectl -n rook-ceph get pods -l app=rook-ceph-tools -o jsonpath='{.items[0].metadata.name}') diff --git a/scripts/travis-helmtest.sh b/scripts/travis-helmtest.sh index b02d77a82..2c3d384bc 100755 --- a/scripts/travis-helmtest.sh +++ b/scripts/travis-helmtest.sh @@ -6,6 +6,7 @@ set -e export KUBE_VERSION=$1 sudo scripts/minikube.sh up sudo scripts/minikube.sh deploy-rook +sudo scripts/minikube.sh create-block-pool # pull docker images to speed up e2e sudo scripts/minikube.sh cephcsi sudo scripts/minikube.sh k8s-sidecar