mirrors/ceph-csi
1
0
Fork 0
mirror of https://github.com/ceph/ceph-csi.git synced 2024-09-19 15:09:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

e2e: update snapshot restore e2e to check data consistency

Browse Source 
Currently, in rbd snapshot restore and volume clone E2E we
are not checking any data consistency after doing snapshot
restore or volume clone. Hence, this PR writes the data in
the PVC and checks the checksum of the file and verify it with
the snapshot or cloned PVC.

Signed-off-by: Yati Padia <ypadia@redhat.com>
This commit is contained in:
Yati Padia 2020-10-13 11:04:55 +05:30 committed by mergify[bot]
parent bad7d56ef4
commit cfb6abc067
5 changed files with 154 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
e2e/cephfs.go
View File

@ -682,7 +682,13 @@ var _ = Describe("cephfs", func() {
app.Namespace = f.UniqueName
app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvc.Name
wErr := writeDataInPod(app, f)
label := make(map[string]string)
label[appKey] = appLabel
app.Labels = label
opt := metav1.ListOptions{
LabelSelector: fmt.Sprintf("%s=%s", appKey, label[appKey]),
}
wErr := writeDataInPod(app, &opt, f)
if wErr != nil {
e2elog.Failf("failed to write data with error %v", wErr)
}
@ -893,7 +899,13 @@ var _ = Describe("cephfs", func() {
}
app.Namespace = f.UniqueName
app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvc.Name
wErr := writeDataInPod(app, f)
label := make(map[string]string)
label[appKey] = appLabel
app.Labels = label
opt := metav1.ListOptions{
LabelSelector: fmt.Sprintf("%s=%s", appKey, label[appKey]),
}
wErr := writeDataInPod(app, &opt, f)
if wErr != nil {
e2elog.Failf("failed to write data from application %v", wErr)
}

53
e2e/rbd.go
View File

@ -473,6 +473,7 @@ var _ = Describe("RBD", func() {
var wg sync.WaitGroup
totalCount := 10
wgErrs := make([]error, totalCount)
chErrs := make([]error, totalCount)
wg.Add(totalCount)
err := createRBDSnapshotClass(f)
if err != nil {
@ -482,12 +483,28 @@ var _ = Describe("RBD", func() {
if err != nil {
e2elog.Failf("failed to load PVC with error %v", err)
}
label := make(map[string]string)
pvc.Namespace = f.UniqueName
err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout)
if err != nil {
e2elog.Failf("failed to create PVC with error %v", err)
}
app, err := loadApp(appPath)
if err != nil {
e2elog.Failf("failed to load app with error %v", err)
}
// write data in PVC
label[appKey] = appLabel
app.Namespace = f.UniqueName
app.Labels = label
opt := metav1.ListOptions{
LabelSelector: fmt.Sprintf("%s=%s", appKey, label[appKey]),
}
app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvc.Name
checkSum, err := writeDataAndCalChecksum(app, &opt, f)
if err != nil {
e2elog.Failf("failed to calculate checksum with error %v", err)
}
validateRBDImageCount(f, 1)
snap := getSnapshot(snapshotPath)
snap.Namespace = f.UniqueName
@ -533,7 +550,26 @@ var _ = Describe("RBD", func() {
for i := 0; i < totalCount; i++ {
go func(w *sync.WaitGroup, n int, p v1.PersistentVolumeClaim, a v1.Pod) {
name := fmt.Sprintf("%s%d", f.UniqueName, n)
label := make(map[string]string)
label[appKey] = name
a.Labels = label
opt := metav1.ListOptions{
LabelSelector: fmt.Sprintf("%s=%s", appKey, label[appKey]),
}
wgErrs[n] = createPVCAndApp(name, f, &p, &a, deployTimeout)
if wgErrs[n] == nil {
filePath := a.Spec.Containers[0].VolumeMounts[0].MountPath + "/test"
checkSumClone := ""
e2elog.Logf("calculating checksum clone for filepath %s", filePath)
checkSumClone, chErrs[n] = calculateSHA512sum(f, &a, filePath, &opt)
e2elog.Logf("checksum value for the clone is %s with pod name %s", checkSumClone, name)
if chErrs[n] != nil {
e2elog.Logf("failed to calculte checksum for clone with error %s", chErrs[n])
}
if checkSumClone != checkSum {
e2elog.Logf("checksum value didn't match. checksum=%s and checksumclone=%s", checkSum, checkSumClone)
}
}
w.Done()
}(&wg, i, *pvcClone, *appClone)
}
@ -550,6 +586,16 @@ var _ = Describe("RBD", func() {
e2elog.Failf("creating PVCs and applications failed, %d errors were logged", failed)
}
for i, err := range chErrs {
if err != nil {
// not using Failf() as it aborts the test and does not log other errors
e2elog.Logf("failed to calculate checksum (%s%d): %v", f.UniqueName, i, err)
failed++
}
}
if failed != 0 {
e2elog.Failf("calculating checksum failed, %d errors were logged", failed)
}
// total images in cluster is 1 parent rbd image+ total
// snaps+ total clones
totalCloneCount := totalCount + totalCount + 1
@ -671,9 +717,8 @@ var _ = Describe("RBD", func() {
By("create a PVC-PVC clone and bind it to an app", func() {
// pvc clone is only supported from v1.16+
if k8sVersionGreaterEquals(f.ClientSet, 1, 16) {
validatePVCClone(pvcPath, pvcSmartClonePath, appSmartClonePath, f)
validatePVCClone(pvcPath, appPath, pvcSmartClonePath, appSmartClonePath, f)
}
})
By("create a block type PVC and bind it to an app", func() {
@ -689,7 +734,7 @@ var _ = Describe("RBD", func() {
}
// pvc clone is only supported from v1.16+
if v.Major > "1" || (v.Major == "1" && v.Minor >= "16") {
validatePVCClone(rawPvcPath, pvcBlockSmartClonePath, appBlockSmartClonePath, f)
validatePVCClone(rawPvcPath, rawAppPath, pvcBlockSmartClonePath, appBlockSmartClonePath, f)
}
})
By("create/delete multiple PVCs and Apps", func() {

7
e2e/upgrade-cephfs.go
View File

@ -205,7 +205,7 @@ var _ = Describe("CephFS Upgrade Testing", func() {
e2elog.Logf("Calculating checksum of %s", filePath)
checkSum, err = calculateSHA512sum(f, app, filePath, &opt)
if err != nil {
e2elog.Failf("failed to calculate checksum of %s", filePath)
e2elog.Failf("failed to calculate checksum with error %v", err)
}
// pvc clone is only supported from v1.16+
@ -273,10 +273,9 @@ var _ = Describe("CephFS Upgrade Testing", func() {
}
mountPath := appClone.Spec.Containers[0].VolumeMounts[0].MountPath
testFilePath := filepath.Join(mountPath, "testClone")
newCheckSum, err = calculateSHA512sum(f, appClone, testFilePath, &opt)
if err != nil {
e2elog.Failf("failed to calculate checksum of %s", testFilePath)
e2elog.Failf("failed to calculate checksum with error %v", err)
}
if strings.Compare(newCheckSum, checkSum) != 0 {
@ -337,7 +336,7 @@ var _ = Describe("CephFS Upgrade Testing", func() {
testFilePath := filepath.Join(mountPath, "testClone")
newCheckSum, err = calculateSHA512sum(f, appClone, testFilePath, &opt)
if err != nil {
e2elog.Failf("failed to calculate checksum of %s", testFilePath)
e2elog.Failf("failed to calculate checksum with error %v", err)
}
if strings.Compare(newCheckSum, checkSum) != 0 {

6
e2e/upgrade-rbd.go
View File

@ -223,7 +223,7 @@ var _ = Describe("RBD Upgrade Testing", func() {
e2elog.Logf("Calculating checksum of %s", filePath)
checkSum, err = calculateSHA512sum(f, app, filePath, &opt)
if err != nil {
e2elog.Failf("failed to calculate checksum of %s", filePath)
e2elog.Failf("failed to calculate checksum with error %v", err)
}
// pvc clone is only supported from v1.16+
@ -293,7 +293,7 @@ var _ = Describe("RBD Upgrade Testing", func() {
testFilePath := filepath.Join(mountPath, "testClone")
newCheckSum, err := calculateSHA512sum(f, appClone, testFilePath, &opt)
if err != nil {
e2elog.Failf("failed to calculate checksum of %s", testFilePath)
e2elog.Failf("failed to calculate checksum with error %v", err)
}
if strings.Compare(newCheckSum, checkSum) != 0 {
e2elog.Failf("The checksum of files did not match, expected %s received %s ", checkSum, newCheckSum)
@ -342,7 +342,7 @@ var _ = Describe("RBD Upgrade Testing", func() {
testFilePath := filepath.Join(mountPath, "testClone")
newCheckSum, err := calculateSHA512sum(f, appClone, testFilePath, &opt)
if err != nil {
e2elog.Failf("failed to calculate checksum of %s", testFilePath)
e2elog.Failf("failed to calculate checksum with error %v", err)
}
if strings.Compare(newCheckSum, checkSum) != 0 {
e2elog.Failf("The checksum of files did not match, expected %s received %s ", checkSum, newCheckSum)

92
e2e/utils.go
View File

@ -34,6 +34,9 @@ const (
retainPolicy = v1.PersistentVolumeReclaimRetain
// deletePolicy is the default policy in E2E.
deletePolicy = v1.PersistentVolumeReclaimDelete
// Default key and label for Listoptions
appKey = "app"
appLabel = "write-data-in-pod"
)
var (
@ -294,22 +297,19 @@ func validateNormalUserPVCAccess(pvcPath string, f *framework.Framework) error {
}
// writeDataInPod fill zero content to a file in the provided POD volume.
func writeDataInPod(app *v1.Pod, f *framework.Framework) error {
app.Labels = map[string]string{"app": "write-data-in-pod"}
func writeDataInPod(app *v1.Pod, opt *metav1.ListOptions, f *framework.Framework) error {
app.Namespace = f.UniqueName
err := createApp(f.ClientSet, app, deployTimeout)
if err != nil {
return err
}
opt := metav1.ListOptions{
LabelSelector: "app=write-data-in-pod",
}
// write data to PVC. The idea here is to fill some content in the file
// instead of filling and reverifying the md5sum/data integrity
filePath := app.Spec.Containers[0].VolumeMounts[0].MountPath + "/test"
// While writing more data we are encountering issues in E2E timeout, so keeping it low for now
_, writeErr, err := execCommandInPod(f, fmt.Sprintf("dd if=/dev/zero of=%s bs=1M count=10 status=none", filePath), app.Namespace, &opt)
_, writeErr, err := execCommandInPod(f, fmt.Sprintf("dd if=/dev/zero of=%s bs=1M count=10 status=none", filePath), app.Namespace, opt)
if err != nil {
return err
}
@ -471,20 +471,68 @@ func enableTopologyInTemplate(data string) string {
return strings.ReplaceAll(data, "--feature-gates=Topology=false", "--feature-gates=Topology=true")
}
func validatePVCClone(sourcePvcPath, clonePvcPath, clonePvcAppPath string, f *framework.Framework) {
func writeDataAndCalChecksum(app *v1.Pod, opt *metav1.ListOptions, f *framework.Framework) (string, error) {
filePath := app.Spec.Containers[0].VolumeMounts[0].MountPath + "/test"
// write data in PVC
err := writeDataInPod(app, opt, f)
if err != nil {
e2elog.Logf("failed to write data in the pod with error %v", err)
return "", err
}
checkSum, err := calculateSHA512sum(f, app, filePath, opt)
if err != nil {
e2elog.Logf("failed to calculate checksum with error %v", err)
return checkSum, err
}
err = deletePod(app.Name, app.Namespace, f.ClientSet, deployTimeout)
if err != nil {
e2elog.Failf("failed to delete pod with error %v", err)
}
return checkSum, nil
}
// nolint:gocyclo // reduce complexity
func validatePVCClone(sourcePvcPath, sourceAppPath, clonePvcPath, clonePvcAppPath string, f *framework.Framework) {
var wg sync.WaitGroup
totalCount := 10
wgErrs := make([]error, totalCount)
chErrs := make([]error, totalCount)
pvc, err := loadPVC(sourcePvcPath)
if err != nil {
e2elog.Failf("failed to load PVC with error %v", err)
}
label := make(map[string]string)
pvc.Namespace = f.UniqueName
err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout)
if err != nil {
e2elog.Failf("failed to create PVC with error %v", err)
}
app, err := loadApp(sourceAppPath)
if err != nil {
e2elog.Failf("failed to load app with error %v", err)
}
label[appKey] = appLabel
app.Namespace = f.UniqueName
app.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvc.Name
app.Labels = label
opt := metav1.ListOptions{
LabelSelector: fmt.Sprintf("%s=%s", appKey, label[appKey]),
}
checkSum := ""
pvc, err = f.ClientSet.CoreV1().PersistentVolumeClaims(pvc.Namespace).Get(context.TODO(), pvc.Name, metav1.GetOptions{})
if err != nil {
e2elog.Failf("failed to get pvc %v", err)
}
if *pvc.Spec.VolumeMode == v1.PersistentVolumeFilesystem {
checkSum, err = writeDataAndCalChecksum(app, &opt, f)
if err != nil {
e2elog.Failf("failed to calculate checksum with error %v", err)
}
}
// validate created backend rbd images
validateRBDImageCount(f, 1)
pvcClone, err := loadPVC(clonePvcPath)
@ -503,7 +551,26 @@ func validatePVCClone(sourcePvcPath, clonePvcPath, clonePvcAppPath string, f *fr
for i := 0; i < totalCount; i++ {
go func(w *sync.WaitGroup, n int, p v1.PersistentVolumeClaim, a v1.Pod) {
name := fmt.Sprintf("%s%d", f.UniqueName, n)
label := make(map[string]string)
label[appKey] = name
a.Labels = label
opt := metav1.ListOptions{
LabelSelector: fmt.Sprintf("%s=%s", appKey, label[appKey]),
}
wgErrs[n] = createPVCAndApp(name, f, &p, &a, deployTimeout)
if *pvc.Spec.VolumeMode == v1.PersistentVolumeFilesystem && wgErrs[n] == nil {
filePath := a.Spec.Containers[0].VolumeMounts[0].MountPath + "/test"
checkSumClone := ""
e2elog.Logf("Calculating checksum clone for filepath %s", filePath)
checkSumClone, chErrs[n] = calculateSHA512sum(f, &a, filePath, &opt)
e2elog.Logf("checksum for clone is %s", checkSumClone)
if chErrs[n] != nil {
e2elog.Logf("Failed calculating checksum clone %s", chErrs[n])
}
if checkSumClone != checkSum {
e2elog.Logf("checksum didn't match. checksum=%s and checksumclone=%s", checkSum, checkSumClone)
}
}
w.Done()
}(&wg, i, *pvcClone, *appClone)
}
@ -521,6 +588,17 @@ func validatePVCClone(sourcePvcPath, clonePvcPath, clonePvcAppPath string, f *fr
e2elog.Failf("creating PVCs failed, %d errors were logged", failed)
}
for i, err := range chErrs {
if err != nil {
// not using Failf() as it aborts the test and does not log other errors
e2elog.Logf("failed to calculate checksum (%s%d): %v", f.UniqueName, i, err)
failed++
}
}
if failed != 0 {
e2elog.Failf("calculating checksum failed, %d errors were logged", failed)
}
// total images in cluster is 1 parent rbd image+ total
// temporary clone+ total clones
totalCloneCount := totalCount + totalCount + 1