mirror of
https://github.com/ceph/ceph-csi.git
synced 2024-12-18 11:00:25 +00:00
419 lines
12 KiB
Go
419 lines
12 KiB
Go
/*
|
|
Copyright 2021 The Ceph-CSI Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package e2e
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"time"
|
|
|
|
snapapi "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1"
|
|
snapclient "github.com/kubernetes-csi/external-snapshotter/client/v6/clientset/versioned/typed/volumesnapshot/v1"
|
|
. "github.com/onsi/gomega" //nolint:golint // e2e uses Expect() and other Gomega functions
|
|
v1 "k8s.io/api/core/v1"
|
|
apierrs "k8s.io/apimachinery/pkg/api/errors"
|
|
"k8s.io/apimachinery/pkg/api/resource"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/util/wait"
|
|
"k8s.io/kubernetes/test/e2e/framework"
|
|
)
|
|
|
|
func getSnapshotClass(path string) snapapi.VolumeSnapshotClass {
|
|
sc := snapapi.VolumeSnapshotClass{}
|
|
err := unmarshal(path, &sc)
|
|
Expect(err).ShouldNot(HaveOccurred())
|
|
|
|
return sc
|
|
}
|
|
|
|
func getSnapshot(path string) snapapi.VolumeSnapshot {
|
|
sc := snapapi.VolumeSnapshot{}
|
|
err := unmarshal(path, &sc)
|
|
Expect(err).ShouldNot(HaveOccurred())
|
|
|
|
return sc
|
|
}
|
|
|
|
func newSnapshotClient() (*snapclient.SnapshotV1Client, error) {
|
|
config, err := framework.LoadConfig()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error creating client: %w", err)
|
|
}
|
|
c, err := snapclient.NewForConfig(config)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error creating snapshot client: %w", err)
|
|
}
|
|
|
|
return c, err
|
|
}
|
|
|
|
func createSnapshot(snap *snapapi.VolumeSnapshot, t int) error {
|
|
sclient, err := newSnapshotClient()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
_, err = sclient.
|
|
VolumeSnapshots(snap.Namespace).
|
|
Create(context.TODO(), snap, metav1.CreateOptions{})
|
|
if err != nil {
|
|
return fmt.Errorf("failed to create volumesnapshot: %w", err)
|
|
}
|
|
framework.Logf("snapshot with name %v created in %v namespace", snap.Name, snap.Namespace)
|
|
|
|
timeout := time.Duration(t) * time.Minute
|
|
name := snap.Name
|
|
start := time.Now()
|
|
framework.Logf("waiting for %v to be in ready state", snap)
|
|
|
|
return wait.PollImmediate(poll, timeout, func() (bool, error) {
|
|
framework.Logf("waiting for snapshot %s (%d seconds elapsed)", snap.Name, int(time.Since(start).Seconds()))
|
|
snaps, err := sclient.
|
|
VolumeSnapshots(snap.Namespace).
|
|
Get(context.TODO(), name, metav1.GetOptions{})
|
|
if err != nil {
|
|
framework.Logf("Error getting snapshot in namespace: '%s': %v", snap.Namespace, err)
|
|
if isRetryableAPIError(err) {
|
|
return false, nil
|
|
}
|
|
if apierrs.IsNotFound(err) {
|
|
return false, nil
|
|
}
|
|
|
|
return false, fmt.Errorf("failed to get volumesnapshot: %w", err)
|
|
}
|
|
if snaps.Status == nil || snaps.Status.ReadyToUse == nil {
|
|
return false, nil
|
|
}
|
|
if *snaps.Status.ReadyToUse {
|
|
return true, nil
|
|
}
|
|
framework.Logf("snapshot %s in %v state", snap.Name, *snaps.Status.ReadyToUse)
|
|
|
|
return false, nil
|
|
})
|
|
}
|
|
|
|
func deleteSnapshot(snap *snapapi.VolumeSnapshot, t int) error {
|
|
sclient, err := newSnapshotClient()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
err = sclient.
|
|
VolumeSnapshots(snap.Namespace).
|
|
Delete(context.TODO(), snap.Name, metav1.DeleteOptions{})
|
|
if err != nil {
|
|
return fmt.Errorf("failed to delete volumesnapshot: %w", err)
|
|
}
|
|
|
|
timeout := time.Duration(t) * time.Minute
|
|
name := snap.Name
|
|
start := time.Now()
|
|
framework.Logf("Waiting up to %v to be deleted", snap)
|
|
|
|
return wait.PollImmediate(poll, timeout, func() (bool, error) {
|
|
framework.Logf("deleting snapshot %s (%d seconds elapsed)", name, int(time.Since(start).Seconds()))
|
|
_, err := sclient.
|
|
VolumeSnapshots(snap.Namespace).
|
|
Get(context.TODO(), name, metav1.GetOptions{})
|
|
if err == nil {
|
|
return false, nil
|
|
}
|
|
|
|
if isRetryableAPIError(err) {
|
|
return false, nil
|
|
}
|
|
if !apierrs.IsNotFound(err) {
|
|
return false, fmt.Errorf(
|
|
"get on deleted snapshot %v failed : other than \"not found\": %w",
|
|
name,
|
|
err)
|
|
}
|
|
|
|
return true, nil
|
|
})
|
|
}
|
|
|
|
func createRBDSnapshotClass(f *framework.Framework) error {
|
|
scPath := fmt.Sprintf("%s/%s", rbdExamplePath, "snapshotclass.yaml")
|
|
sc := getSnapshotClass(scPath)
|
|
|
|
sc.Parameters["csi.storage.k8s.io/snapshotter-secret-namespace"] = cephCSINamespace
|
|
sc.Parameters["csi.storage.k8s.io/snapshotter-secret-name"] = rbdProvisionerSecretName
|
|
|
|
fsID, err := getClusterID(f)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to get clusterID: %w", err)
|
|
}
|
|
sc.Parameters["clusterID"] = fsID
|
|
sclient, err := newSnapshotClient()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
_, err = sclient.VolumeSnapshotClasses().Create(context.TODO(), &sc, metav1.CreateOptions{})
|
|
|
|
return err
|
|
}
|
|
|
|
func deleteRBDSnapshotClass() error {
|
|
scPath := fmt.Sprintf("%s/%s", rbdExamplePath, "snapshotclass.yaml")
|
|
sc := getSnapshotClass(scPath)
|
|
|
|
sclient, err := newSnapshotClient()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
return sclient.VolumeSnapshotClasses().Delete(context.TODO(), sc.Name, metav1.DeleteOptions{})
|
|
}
|
|
|
|
func createCephFSSnapshotClass(f *framework.Framework) error {
|
|
scPath := fmt.Sprintf("%s/%s", cephFSExamplePath, "snapshotclass.yaml")
|
|
sc := getSnapshotClass(scPath)
|
|
sc.Parameters["csi.storage.k8s.io/snapshotter-secret-namespace"] = cephCSINamespace
|
|
sc.Parameters["csi.storage.k8s.io/snapshotter-secret-name"] = cephFSProvisionerSecretName
|
|
|
|
fsID, err := getClusterID(f)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to get clusterID: %w", err)
|
|
}
|
|
sc.Parameters["clusterID"] = fsID
|
|
sclient, err := newSnapshotClient()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
_, err = sclient.VolumeSnapshotClasses().Create(context.TODO(), &sc, metav1.CreateOptions{})
|
|
if err != nil {
|
|
return fmt.Errorf("failed to create volumesnapshotclass: %w", err)
|
|
}
|
|
|
|
return err
|
|
}
|
|
|
|
func createNFSSnapshotClass(f *framework.Framework) error {
|
|
scPath := fmt.Sprintf("%s/%s", nfsExamplePath, "snapshotclass.yaml")
|
|
sc := getSnapshotClass(scPath)
|
|
sc.Parameters["csi.storage.k8s.io/snapshotter-secret-namespace"] = cephCSINamespace
|
|
sc.Parameters["csi.storage.k8s.io/snapshotter-secret-name"] = cephFSProvisionerSecretName
|
|
|
|
fsID, err := getClusterID(f)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to get clusterID: %w", err)
|
|
}
|
|
sc.Parameters["clusterID"] = fsID
|
|
sclient, err := newSnapshotClient()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
timeout := time.Duration(deployTimeout) * time.Minute
|
|
|
|
return wait.PollImmediate(poll, timeout, func() (bool, error) {
|
|
_, err = sclient.VolumeSnapshotClasses().Create(context.TODO(), &sc, metav1.CreateOptions{})
|
|
if err != nil {
|
|
framework.Logf("error creating SnapshotClass %q: %v", sc.Name, err)
|
|
if apierrs.IsAlreadyExists(err) {
|
|
return true, nil
|
|
}
|
|
if isRetryableAPIError(err) {
|
|
return false, nil
|
|
}
|
|
|
|
return false, fmt.Errorf("failed to create SnapshotClass %q: %w", sc.Name, err)
|
|
}
|
|
|
|
return true, nil
|
|
})
|
|
}
|
|
|
|
func deleteNFSSnapshotClass() error {
|
|
scPath := fmt.Sprintf("%s/%s", nfsExamplePath, "snapshotclass.yaml")
|
|
sc := getSnapshotClass(scPath)
|
|
|
|
sclient, err := newSnapshotClient()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
timeout := time.Duration(deployTimeout) * time.Minute
|
|
|
|
return wait.PollImmediate(poll, timeout, func() (bool, error) {
|
|
err = sclient.VolumeSnapshotClasses().Delete(context.TODO(), sc.Name, metav1.DeleteOptions{})
|
|
if err != nil {
|
|
framework.Logf("error deleting SnapshotClass %q: %v", sc.Name, err)
|
|
if apierrs.IsNotFound(err) {
|
|
return true, nil
|
|
}
|
|
if isRetryableAPIError(err) {
|
|
return false, nil
|
|
}
|
|
|
|
return false, fmt.Errorf("failed to delete SnapshotClass %q: %w", sc.Name, err)
|
|
}
|
|
|
|
return true, nil
|
|
})
|
|
}
|
|
|
|
func getVolumeSnapshotContent(namespace, snapshotName string) (*snapapi.VolumeSnapshotContent, error) {
|
|
sclient, err := newSnapshotClient()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
snapshot, err := sclient.
|
|
VolumeSnapshots(namespace).
|
|
Get(context.TODO(), snapshotName, metav1.GetOptions{})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to get volumesnapshot: %w", err)
|
|
}
|
|
|
|
volumeSnapshotContent, err := sclient.
|
|
VolumeSnapshotContents().
|
|
Get(context.TODO(), *snapshot.Status.BoundVolumeSnapshotContentName, metav1.GetOptions{})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to get volumesnapshotcontent: %w", err)
|
|
}
|
|
|
|
return volumeSnapshotContent, nil
|
|
}
|
|
|
|
//nolint:gocyclo,cyclop // reduce complexity
|
|
func validateBiggerPVCFromSnapshot(f *framework.Framework,
|
|
pvcPath,
|
|
appPath,
|
|
snapPath,
|
|
pvcClonePath,
|
|
appClonePath string,
|
|
) error {
|
|
const (
|
|
size = "1Gi"
|
|
newSize = "2Gi"
|
|
)
|
|
pvc, err := loadPVC(pvcPath)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to load PVC: %w", err)
|
|
}
|
|
label := make(map[string]string)
|
|
pvc.Namespace = f.UniqueName
|
|
pvc.Spec.Resources.Requests[v1.ResourceStorage] = resource.MustParse(size)
|
|
app, err := loadApp(appPath)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to load app: %w", err)
|
|
}
|
|
label[appKey] = appLabel
|
|
app.Namespace = f.UniqueName
|
|
app.Labels = label
|
|
opt := metav1.ListOptions{
|
|
LabelSelector: fmt.Sprintf("%s=%s", appKey, label[appKey]),
|
|
}
|
|
err = createPVCAndApp("", f, pvc, app, deployTimeout)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to create pvc and application: %w", err)
|
|
}
|
|
|
|
snap := getSnapshot(snapPath)
|
|
snap.Namespace = f.UniqueName
|
|
snap.Spec.Source.PersistentVolumeClaimName = &pvc.Name
|
|
err = createSnapshot(&snap, deployTimeout)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to create snapshot: %w", err)
|
|
}
|
|
err = deletePVCAndApp("", f, pvc, app)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to delete pvc and application: %w", err)
|
|
}
|
|
pvcClone, err := loadPVC(pvcClonePath)
|
|
if err != nil {
|
|
framework.Failf("failed to load PVC: %v", err)
|
|
}
|
|
pvcClone.Namespace = f.UniqueName
|
|
pvcClone.Spec.DataSource.Name = snap.Name
|
|
pvcClone.Spec.Resources.Requests[v1.ResourceStorage] = resource.MustParse(newSize)
|
|
appClone, err := loadApp(appClonePath)
|
|
if err != nil {
|
|
framework.Failf("failed to load application: %v", err)
|
|
}
|
|
appClone.Namespace = f.UniqueName
|
|
appClone.Labels = label
|
|
err = createPVCAndApp("", f, pvcClone, appClone, deployTimeout)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to create pvc clone and application: %w", err)
|
|
}
|
|
err = deleteSnapshot(&snap, deployTimeout)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to delete snapshot: %w", err)
|
|
}
|
|
if pvcClone.Spec.VolumeMode == nil || *pvcClone.Spec.VolumeMode == v1.PersistentVolumeFilesystem {
|
|
err = checkDirSize(appClone, f, &opt, newSize)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to validate directory size: %w", err)
|
|
}
|
|
}
|
|
|
|
if pvcClone.Spec.VolumeMode != nil && *pvcClone.Spec.VolumeMode == v1.PersistentVolumeBlock {
|
|
err = checkDeviceSize(appClone, f, &opt, newSize)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to validate device size: %w", err)
|
|
}
|
|
|
|
// make sure we had unset snapshot metadata on CreateVolume
|
|
// from snapshot
|
|
var (
|
|
volSnapName string
|
|
volSnapNamespace string
|
|
volSnapContentName string
|
|
stdErr string
|
|
imageList []string
|
|
)
|
|
imageList, err = listRBDImages(f, defaultRBDPool)
|
|
if err != nil {
|
|
framework.Failf("failed to list rbd images: %v", err)
|
|
}
|
|
framework.Logf("list of rbd images: %v", imageList)
|
|
volSnapName, stdErr, err = execCommandInToolBoxPod(f,
|
|
formatImageMetaGetCmd(defaultRBDPool, imageList[0], volSnapNameKey),
|
|
rookNamespace)
|
|
if checkGetKeyError(err, stdErr) {
|
|
framework.Failf("found volume snapshot name %s/%s %s=%s: err=%v stdErr=%q",
|
|
rbdOptions(defaultRBDPool), imageList[0], volSnapNameKey, volSnapName, err, stdErr)
|
|
}
|
|
volSnapNamespace, stdErr, err = execCommandInToolBoxPod(f,
|
|
formatImageMetaGetCmd(defaultRBDPool, imageList[0], volSnapNamespaceKey),
|
|
rookNamespace)
|
|
if checkGetKeyError(err, stdErr) {
|
|
framework.Failf("found volume snapshot namespace %s/%s %s=%s: err=%v stdErr=%q",
|
|
rbdOptions(defaultRBDPool), imageList[0], volSnapNamespaceKey, volSnapNamespace, err, stdErr)
|
|
}
|
|
volSnapContentName, stdErr, err = execCommandInToolBoxPod(f,
|
|
formatImageMetaGetCmd(defaultRBDPool, imageList[0], volSnapContentNameKey),
|
|
rookNamespace)
|
|
if checkGetKeyError(err, stdErr) {
|
|
framework.Failf("found snapshotcontent name %s/%s %s=%s: err=%v stdErr=%q",
|
|
rbdOptions(defaultRBDPool), imageList[0], volSnapContentNameKey,
|
|
volSnapContentName, err, stdErr)
|
|
}
|
|
}
|
|
err = deletePVCAndApp("", f, pvcClone, appClone)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to delete pvc and application: %w", err)
|
|
}
|
|
|
|
return nil
|
|
}
|