diff --git a/e2e/cephfs.go b/e2e/cephfs.go index f4366c1d2..43171537f 100644 --- a/e2e/cephfs.go +++ b/e2e/cephfs.go @@ -215,6 +215,11 @@ var _ = Describe(cephfsType, func() { if err != nil { framework.Failf("timeout waiting for deployment update %s/%s: %v", cephCSINamespace, cephFSDeploymentName, err) } + + err = createSubvolumegroup(f, fileSystemName, subvolumegroup) + if err != nil { + framework.Failf("%v", err) + } }) AfterEach(func() { @@ -254,10 +259,15 @@ var _ = Describe(cephfsType, func() { } deleteVault() + err = deleteSubvolumegroup(f, fileSystemName, subvolumegroup) + if err != nil { + framework.Failf("%v", err) + } + if deployCephFS { deleteCephfsPlugin() if cephCSINamespace != defaultNs { - err := deleteNamespace(c, cephCSINamespace) + err = deleteNamespace(c, cephCSINamespace) if err != nil { framework.Failf("failed to delete namespace %s: %v", cephCSINamespace, err) } @@ -939,14 +949,24 @@ var _ = Describe(cephfsType, func() { } // re-define configmap with information of multiple clusters. + subvolgrp1 := "subvolgrp1" + subvolgrp2 := "subvolgrp2" clusterInfo := map[string]map[string]string{} clusterID1 := "clusterID-1" clusterID2 := "clusterID-2" clusterInfo[clusterID1] = map[string]string{} - clusterInfo[clusterID1]["subvolumeGroup"] = "subvolgrp1" + clusterInfo[clusterID1]["subvolumeGroup"] = subvolgrp1 clusterInfo[clusterID2] = map[string]string{} - clusterInfo[clusterID2]["subvolumeGroup"] = "subvolgrp2" + clusterInfo[clusterID2]["subvolumeGroup"] = subvolgrp2 + err = createSubvolumegroup(f, fileSystemName, subvolgrp1) + if err != nil { + framework.Failf("%v", err) + } + err = createSubvolumegroup(f, fileSystemName, subvolgrp2) + if err != nil { + framework.Failf("%v", err) + } err = createCustomConfigMap(f.ClientSet, cephFSDirPath, clusterInfo) if err != nil { framework.Failf("failed to create configmap: %v", err) @@ -967,7 +987,7 @@ var _ = Describe(cephfsType, func() { framework.Failf("failed to delete storageclass: %v", err) } // verify subvolume group creation. - err = validateSubvolumegroup(f, "subvolgrp1") + err = validateSubvolumegroup(f, subvolgrp1) if err != nil { framework.Failf("failed to validate subvolume group: %v", err) } @@ -987,10 +1007,18 @@ var _ = Describe(cephfsType, func() { if err != nil { framework.Failf("failed to delete storageclass: %v", err) } - err = validateSubvolumegroup(f, "subvolgrp2") + err = validateSubvolumegroup(f, subvolgrp2) if err != nil { framework.Failf("failed to validate subvolume group: %v", err) } + err = deleteSubvolumegroup(f, fileSystemName, subvolgrp1) + if err != nil { + framework.Failf("%v", err) + } + err = deleteSubvolumegroup(f, fileSystemName, subvolgrp2) + if err != nil { + framework.Failf("%v", err) + } err = deleteConfigMap(cephFSDirPath) if err != nil { framework.Failf("failed to delete configmap: %v", err) diff --git a/e2e/nfs.go b/e2e/nfs.go index 56f9cc7f4..59628c94a 100644 --- a/e2e/nfs.go +++ b/e2e/nfs.go @@ -276,6 +276,11 @@ var _ = Describe("nfs", func() { if err != nil { framework.Failf("failed to create node secret: %v", err) } + + err = createSubvolumegroup(f, fileSystemName, subvolumegroup) + if err != nil { + framework.Failf("%v", err) + } }) AfterEach(func() { @@ -313,10 +318,15 @@ var _ = Describe("nfs", func() { if err != nil { framework.Failf("failed to delete storageclass: %v", err) } + err = deleteSubvolumegroup(f, fileSystemName, subvolumegroup) + if err != nil { + framework.Failf("%v", err) + } + if deployNFS { deleteNFSPlugin() if cephCSINamespace != defaultNs { - err := deleteNamespace(c, cephCSINamespace) + err = deleteNamespace(c, cephCSINamespace) if err != nil { framework.Failf("failed to delete namespace %s: %v", cephCSINamespace, err) } diff --git a/e2e/utils.go b/e2e/utils.go index ff1618bf0..08a8d4b51 100644 --- a/e2e/utils.go +++ b/e2e/utils.go @@ -1853,3 +1853,31 @@ func checkExports(f *framework.Framework, clusterID, clientString string) bool { return true } + +// createSubvolumegroup creates a subvolumegroup. +func createSubvolumegroup(f *framework.Framework, fileSystemName, subvolumegroup string) error { + cmd := fmt.Sprintf("ceph fs subvolumegroup create %s %s", fileSystemName, subvolumegroup) + _, stdErr, err := execCommandInToolBoxPod(f, cmd, rookNamespace) + if err != nil { + return fmt.Errorf("failed to exec command in toolbox: %w", err) + } + if stdErr != "" { + return fmt.Errorf("failed to create subvolumegroup %s : %v", subvolumegroup, stdErr) + } + + return nil +} + +// deleteSubvolumegroup creates a subvolumegroup. +func deleteSubvolumegroup(f *framework.Framework, fileSystemName, subvolumegroup string) error { + cmd := fmt.Sprintf("ceph fs subvolumegroup rm %s %s", fileSystemName, subvolumegroup) + _, stdErr, err := execCommandInToolBoxPod(f, cmd, rookNamespace) + if err != nil { + return fmt.Errorf("failed to exec command in toolbox: %w", err) + } + if stdErr != "" { + return fmt.Errorf("failed to remove subvolumegroup %s : %v", subvolumegroup, stdErr) + } + + return nil +} diff --git a/internal/cephfs/core/metadata.go b/internal/cephfs/core/metadata.go index 1dfd1b5c4..9e2b90d5f 100644 --- a/internal/cephfs/core/metadata.go +++ b/internal/cephfs/core/metadata.go @@ -51,32 +51,6 @@ func (s *subVolumeClient) isUnsupportedSubVolMetadata(err error) bool { return true } -// isSubVolumeGroupCreated returns true if subvolume group is created. -func (s *subVolumeClient) isSubVolumeGroupCreated() bool { - newLocalClusterState(s.clusterID) - clusterAdditionalInfo[s.clusterID].subVolumeGroupsRWMutex.RLock() - defer clusterAdditionalInfo[s.clusterID].subVolumeGroupsRWMutex.RUnlock() - - if clusterAdditionalInfo[s.clusterID].subVolumeGroupsCreated == nil { - return false - } - - return clusterAdditionalInfo[s.clusterID].subVolumeGroupsCreated[s.SubvolumeGroup] -} - -// updateSubVolumeGroupCreated updates subvolume group created map. -// If the map is nil, it creates a new map and updates it. -func (s *subVolumeClient) updateSubVolumeGroupCreated(state bool) { - clusterAdditionalInfo[s.clusterID].subVolumeGroupsRWMutex.Lock() - defer clusterAdditionalInfo[s.clusterID].subVolumeGroupsRWMutex.Unlock() - - if clusterAdditionalInfo[s.clusterID].subVolumeGroupsCreated == nil { - clusterAdditionalInfo[s.clusterID].subVolumeGroupsCreated = make(map[string]bool) - } - - clusterAdditionalInfo[s.clusterID].subVolumeGroupsCreated[s.SubvolumeGroup] = state -} - // setMetadata sets custom metadata on the subvolume in a volume as a // key-value pair. func (s *subVolumeClient) setMetadata(key, value string) error { diff --git a/internal/cephfs/core/volume.go b/internal/cephfs/core/volume.go index 0754b8a23..a92025ec9 100644 --- a/internal/cephfs/core/volume.go +++ b/internal/cephfs/core/volume.go @@ -210,14 +210,6 @@ type localClusterState struct { resizeState operationState subVolMetadataState operationState subVolSnapshotMetadataState operationState - // A cluster can have multiple filesystem for that we need to have a map of - // subvolumegroups to check filesystem is created nor not. - // set true once a subvolumegroup is created - // for corresponding filesystem in a cluster. - subVolumeGroupsCreated map[string]bool - // subVolumeGroupsRWMutex is used to protect subVolumeGroupsCreated map - // against concurrent writes while allowing multiple readers. - subVolumeGroupsRWMutex sync.RWMutex } func newLocalClusterState(clusterID string) { @@ -241,24 +233,6 @@ func (s *subVolumeClient) CreateVolume(ctx context.Context) error { return err } - // create subvolumegroup if not already created for the cluster. - if !s.isSubVolumeGroupCreated() { - opts := fsAdmin.SubVolumeGroupOptions{} - err = ca.CreateSubVolumeGroup(s.FsName, s.SubvolumeGroup, &opts) - if err != nil { - log.ErrorLog( - ctx, - "failed to create subvolume group %s, for the vol %s: %s", - s.SubvolumeGroup, - s.VolID, - err) - - return err - } - log.DebugLog(ctx, "cephfs: created subvolume group %s", s.SubvolumeGroup) - s.updateSubVolumeGroupCreated(true) - } - opts := fsAdmin.SubVolumeOptions{ Size: fsAdmin.ByteCount(s.Size), } @@ -271,12 +245,6 @@ func (s *subVolumeClient) CreateVolume(ctx context.Context) error { if err != nil { log.ErrorLog(ctx, "failed to create subvolume %s in fs %s: %s", s.VolID, s.FsName, err) - if errors.Is(err, rados.ErrNotFound) { - // Reset the subVolumeGroupsCreated so that we can try again to create the - // subvolumegroup in next request if the error is Not Found. - s.updateSubVolumeGroupCreated(false) - } - return err }