mirror of
https://github.com/ceph/ceph-csi.git
synced 2025-06-13 02:33:34 +00:00
cleanup: resolve nlreturn linter issues
nlreturn linter requires a new line before return and branch statements except when the return is alone inside a statement group (such as an if statement) to increase code clarity. This commit addresses such issues. Updates: #1586 Signed-off-by: Rakshith R <rar@redhat.com>
This commit is contained in:
Rakshith R
mergify[bot]
@ -37,6 +37,7 @@ func rbdNodePluginCaps(pool, rbdNamespace string) []string {
|
||||
} else {
|
||||
caps = append(caps, fmt.Sprintf("osd 'profile rbd pool=%s namespace=%s'", pool, rbdNamespace))
|
||||
}
|
||||
|
||||
return caps
|
||||
}
|
||||
|
||||
@ -50,6 +51,7 @@ func rbdProvisionerCaps(pool, rbdNamespace string) []string {
|
||||
} else {
|
||||
caps = append(caps, fmt.Sprintf("osd 'profile rbd pool=%s namespace=%s'", pool, rbdNamespace))
|
||||
}
|
||||
|
||||
return caps
|
||||
}
|
||||
|
||||
@ -62,6 +64,7 @@ func cephFSNodePluginCaps() []string {
|
||||
"osd", "'allow rw tag cephfs *=*'",
|
||||
"mds", "'allow rw'",
|
||||
}
|
||||
|
||||
return caps
|
||||
}
|
||||
|
||||
@ -71,6 +74,7 @@ func cephFSProvisionerCaps() []string {
|
||||
"mgr", "'allow rw'",
|
||||
"osd", "'allow rw tag cephfs metadata=*'",
|
||||
}
|
||||
|
||||
return caps
|
||||
}
|
||||
|
||||
@ -83,11 +87,13 @@ func createCephUser(f *framework.Framework, user string, caps []string) (string,
|
||||
if stdErr != "" {
|
||||
return "", fmt.Errorf("failed to create user %s with error %v", cmd, stdErr)
|
||||
}
|
||||
|
||||
return strings.TrimSpace(stdOut), nil
|
||||
}
|
||||
|
||||
func deleteCephUser(f *framework.Framework, user string) error {
|
||||
cmd := fmt.Sprintf("ceph auth del client.%s", user)
|
||||
_, _, err := execCommandInToolBoxPod(f, cmd, rookNamespace)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
@ -165,6 +165,7 @@ func validateSubvolumePath(f *framework.Framework, pvcName, pvcNamespace, fileSy
|
||||
subVolumePath,
|
||||
subVolumePathInPV)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -354,6 +355,7 @@ var _ = Describe("cephfs", func() {
|
||||
fmt.Printf("Checking prefix on %s\n", subVol)
|
||||
if strings.HasPrefix(subVol.Name, volumeNamePrefix) {
|
||||
foundIt = true
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
@ -34,6 +34,7 @@ func validateSubvolumegroup(f *framework.Framework, subvolgrp string) error {
|
||||
if stdOut != expectedGrpPath {
|
||||
return fmt.Errorf("error unexpected group path. Found: %s", stdOut)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -84,6 +85,7 @@ func createCephfsStorageClass(
|
||||
}
|
||||
sc.Namespace = cephCSINamespace
|
||||
_, err = c.StorageV1().StorageClasses().Create(context.TODO(), &sc, metav1.CreateOptions{})
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
@ -102,6 +104,7 @@ func createCephfsSecret(f *framework.Framework, secretName, userName, userKey st
|
||||
delete(sc.StringData, "userKey")
|
||||
sc.Namespace = cephCSINamespace
|
||||
_, err = f.ClientSet.CoreV1().Secrets(cephCSINamespace).Create(context.TODO(), &sc, metav1.CreateOptions{})
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
@ -110,6 +113,7 @@ func unmountCephFSVolume(f *framework.Framework, appName, pvcName string) error
|
||||
pod, err := f.ClientSet.CoreV1().Pods(f.UniqueName).Get(context.TODO(), appName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
e2elog.Logf("Error occurred getting pod %s in namespace %s", appName, f.UniqueName)
|
||||
|
||||
return fmt.Errorf("failed to get pod: %w", err)
|
||||
}
|
||||
pvc, err := f.ClientSet.CoreV1().
|
||||
@ -117,6 +121,7 @@ func unmountCephFSVolume(f *framework.Framework, appName, pvcName string) error
|
||||
Get(context.TODO(), pvcName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
e2elog.Logf("Error occurred getting PVC %s in namespace %s", pvcName, f.UniqueName)
|
||||
|
||||
return fmt.Errorf("failed to get pvc: %w", err)
|
||||
}
|
||||
cmd := fmt.Sprintf(
|
||||
@ -133,6 +138,7 @@ func unmountCephFSVolume(f *framework.Framework, appName, pvcName string) error
|
||||
if stdErr != "" {
|
||||
e2elog.Logf("StdErr occurred: %s", stdErr)
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
@ -150,6 +156,7 @@ func deleteBackingCephFSVolume(f *framework.Framework, pvc *v1.PersistentVolumeC
|
||||
if stdErr != "" {
|
||||
return fmt.Errorf("error deleting backing volume %s %v", imageData.imageName, stdErr)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -174,6 +181,7 @@ func listCephFSSubVolumes(f *framework.Framework, filesystem, groupname string)
|
||||
if err != nil {
|
||||
return subVols, err
|
||||
}
|
||||
|
||||
return subVols, nil
|
||||
}
|
||||
|
||||
@ -187,6 +195,7 @@ func getSubvolumePath(f *framework.Framework, filesystem, subvolgrp, subvolume s
|
||||
if stdErr != "" {
|
||||
return "", fmt.Errorf("failed to getpath for subvolume %s with error %s", subvolume, stdErr)
|
||||
}
|
||||
|
||||
return strings.TrimSpace(stdOut), nil
|
||||
}
|
||||
|
||||
@ -211,6 +220,7 @@ func getSnapName(snapNamespace, snapName string) (string, error) {
|
||||
snapID := snapIDRegex.FindString(*sc.Status.SnapshotHandle)
|
||||
snapshotName := fmt.Sprintf("csi-snap-%s", snapID)
|
||||
e2elog.Logf("snapshotName= %s", snapshotName)
|
||||
|
||||
return snapshotName, nil
|
||||
}
|
||||
|
||||
@ -239,5 +249,6 @@ func deleteBackingCephFSSubvolumeSnapshot(
|
||||
if stdErr != "" {
|
||||
return fmt.Errorf("error deleting backing snapshot %s %v", snapshotName, stdErr)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -21,6 +21,7 @@ func deleteConfigMap(pluginPath string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -120,5 +121,6 @@ func createCustomConfigMap(c kubernetes.Interface, pluginPath string, subvolgrpI
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to update configmap: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -104,5 +104,6 @@ func (vc *vaultConfig) getPassphrase(f *framework.Framework, key string) (string
|
||||
LabelSelector: "app=vault",
|
||||
}
|
||||
stdOut, stdErr := execCommandInPodAndAllowFail(f, cmd, cephCSINamespace, &opt)
|
||||
|
||||
return strings.TrimSpace(stdOut), strings.TrimSpace(stdErr)
|
||||
}
|
||||
|
||||
@ -33,6 +33,7 @@ func logsCSIPods(label string, c clientset.Interface) {
|
||||
podList, err := c.CoreV1().Pods(cephCSINamespace).List(context.TODO(), opt)
|
||||
if err != nil {
|
||||
e2elog.Logf("failed to list pods with selector %s %v", label, err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
@ -72,5 +73,6 @@ func getPreviousPodLogs(c clientset.Interface, namespace, podName, containerName
|
||||
if strings.Contains(string(logs), "Internal Error") {
|
||||
return "", fmt.Errorf("fetched log contains \"Internal Error\": %q", string(logs))
|
||||
}
|
||||
|
||||
return string(logs), err
|
||||
}
|
||||
|
||||
@ -37,8 +37,10 @@ func createNamespace(c kubernetes.Interface, name string) error {
|
||||
if isRetryableAPIError(err) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
return false, err
|
||||
}
|
||||
|
||||
return true, nil
|
||||
})
|
||||
}
|
||||
@ -49,6 +51,7 @@ func deleteNamespace(c kubernetes.Interface, name string) error {
|
||||
if err != nil && !apierrs.IsNotFound(err) {
|
||||
return fmt.Errorf("failed to delete namespace: %w", err)
|
||||
}
|
||||
|
||||
return wait.PollImmediate(poll, timeout, func() (bool, error) {
|
||||
_, err = c.CoreV1().Namespaces().Get(context.TODO(), name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
@ -59,8 +62,10 @@ func deleteNamespace(c kubernetes.Interface, name string) error {
|
||||
if isRetryableAPIError(err) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
return false, err
|
||||
}
|
||||
|
||||
return false, nil
|
||||
})
|
||||
}
|
||||
@ -70,5 +75,6 @@ func replaceNamespaceInTemplate(filePath string) (string, error) {
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return strings.ReplaceAll(string(read), "namespace: default", fmt.Sprintf("namespace: %s", cephCSINamespace)), nil
|
||||
}
|
||||
|
||||
@ -21,6 +21,7 @@ func createNodeLabel(f *framework.Framework, labelKey, labelValue string) error
|
||||
for i := range nodes.Items {
|
||||
framework.AddOrUpdateLabelOnNode(f.ClientSet, nodes.Items[i].Name, labelKey, labelValue)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -32,6 +33,7 @@ func deleteNodeLabel(c kubernetes.Interface, labelKey string) error {
|
||||
for i := range nodes.Items {
|
||||
framework.RemoveLabelOffNode(c, nodes.Items[i].Name, labelKey)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -43,6 +45,7 @@ func checkNodeHasLabel(c kubernetes.Interface, labelKey, labelValue string) erro
|
||||
for i := range nodes.Items {
|
||||
framework.ExpectNodeHasLabel(c, nodes.Items[i].Name, labelKey, labelValue)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
28
e2e/pod.go
28
e2e/pod.go
@ -24,14 +24,17 @@ func getDaemonSetLabelSelector(f *framework.Framework, ns, daemonSetName string)
|
||||
ds, err := f.ClientSet.AppsV1().DaemonSets(ns).Get(context.TODO(), daemonSetName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
e2elog.Logf("Error getting daemonsets with name %s in namespace %s", daemonSetName, ns)
|
||||
|
||||
return "", err
|
||||
}
|
||||
s, err := metav1.LabelSelectorAsSelector(ds.Spec.Selector)
|
||||
if err != nil {
|
||||
e2elog.Logf("Error parsing %s daemonset selector in namespace %s", daemonSetName, ns)
|
||||
|
||||
return "", err
|
||||
}
|
||||
e2elog.Logf("LabelSelector for %s daemonsets in namespace %s: %s", daemonSetName, ns, s.String())
|
||||
|
||||
return s.String(), nil
|
||||
}
|
||||
|
||||
@ -50,6 +53,7 @@ func waitForDaemonSets(name, ns string, c kubernetes.Interface, t int) error {
|
||||
if isRetryableAPIError(err) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
return false, err
|
||||
}
|
||||
dNum := ds.Status.DesiredNumberScheduled
|
||||
@ -85,6 +89,7 @@ func waitForDeploymentComplete(name, ns string, c kubernetes.Interface, t int) e
|
||||
return false, nil
|
||||
}
|
||||
e2elog.Logf("deployment error: %v", err)
|
||||
|
||||
return false, err
|
||||
}
|
||||
|
||||
@ -100,6 +105,7 @@ func waitForDeploymentComplete(name, ns string, c kubernetes.Interface, t int) e
|
||||
deployment.Status.Replicas,
|
||||
deployment.Status.ReadyReplicas)
|
||||
reason = fmt.Sprintf("deployment status: %#v", deployment.Status.String())
|
||||
|
||||
return false, nil
|
||||
})
|
||||
|
||||
@ -109,6 +115,7 @@ func waitForDeploymentComplete(name, ns string, c kubernetes.Interface, t int) e
|
||||
if err != nil {
|
||||
return fmt.Errorf("error waiting for deployment %q status to match expectation: %w", name, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -130,8 +137,10 @@ func findPodAndContainerName(f *framework.Framework, ns, cn string, opt *metav1.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return "", "", errors.New("container name not found")
|
||||
}
|
||||
|
||||
return podList.Items[0].Name, podList.Items[0].Spec.Containers[0].Name, nil
|
||||
}
|
||||
|
||||
@ -144,6 +153,7 @@ func getCommandInPodOpts(
|
||||
if err != nil {
|
||||
return framework.ExecOptions{}, err
|
||||
}
|
||||
|
||||
return framework.ExecOptions{
|
||||
Command: cmd,
|
||||
PodName: pName,
|
||||
@ -192,6 +202,7 @@ func execCommandInDaemonsetPod(
|
||||
CaptureStdout: true,
|
||||
CaptureStderr: true,
|
||||
}
|
||||
|
||||
return f.ExecWithOptions(podOpt)
|
||||
}
|
||||
|
||||
@ -201,6 +212,7 @@ func listPods(f *framework.Framework, ns string, opt *metav1.ListOptions) ([]v1.
|
||||
if len(podList.Items) == 0 {
|
||||
return podList.Items, fmt.Errorf("podlist for label '%s' in namespace %s is empty", opt.LabelSelector, ns)
|
||||
}
|
||||
|
||||
return podList.Items, err
|
||||
}
|
||||
|
||||
@ -213,6 +225,7 @@ func execCommandInPod(f *framework.Framework, c, ns string, opt *metav1.ListOpti
|
||||
if stdErr != "" {
|
||||
e2elog.Logf("stdErr occurred: %v", stdErr)
|
||||
}
|
||||
|
||||
return stdOut, stdErr, err
|
||||
}
|
||||
|
||||
@ -226,6 +239,7 @@ func execCommandInContainer(
|
||||
if stdErr != "" {
|
||||
e2elog.Logf("stdErr occurred: %v", stdErr)
|
||||
}
|
||||
|
||||
return stdOut, stdErr, err
|
||||
}
|
||||
|
||||
@ -241,6 +255,7 @@ func execCommandInToolBoxPod(f *framework.Framework, c, ns string) (string, stri
|
||||
if stdErr != "" {
|
||||
e2elog.Logf("stdErr occurred: %v", stdErr)
|
||||
}
|
||||
|
||||
return stdOut, stdErr, err
|
||||
}
|
||||
|
||||
@ -253,6 +268,7 @@ func execCommandInPodAndAllowFail(f *framework.Framework, c, ns string, opt *met
|
||||
if err != nil {
|
||||
e2elog.Logf("command %s failed: %v", c, err)
|
||||
}
|
||||
|
||||
return stdOut, stdErr
|
||||
}
|
||||
|
||||
@ -264,6 +280,7 @@ func loadApp(path string) (*v1.Pod, error) {
|
||||
for i := range app.Spec.Containers {
|
||||
app.Spec.Containers[i].ImagePullPolicy = v1.PullIfNotPresent
|
||||
}
|
||||
|
||||
return &app, nil
|
||||
}
|
||||
|
||||
@ -272,6 +289,7 @@ func createApp(c kubernetes.Interface, app *v1.Pod, timeout int) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create app: %w", err)
|
||||
}
|
||||
|
||||
return waitForPodInRunningState(app.Name, app.Namespace, c, timeout, noError)
|
||||
}
|
||||
|
||||
@ -280,6 +298,7 @@ func createAppErr(c kubernetes.Interface, app *v1.Pod, timeout int, errString st
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return waitForPodInRunningState(app.Name, app.Namespace, c, timeout, errString)
|
||||
}
|
||||
|
||||
@ -287,12 +306,14 @@ func waitForPodInRunningState(name, ns string, c kubernetes.Interface, t int, ex
|
||||
timeout := time.Duration(t) * time.Minute
|
||||
start := time.Now()
|
||||
e2elog.Logf("Waiting up to %v to be in Running state", name)
|
||||
|
||||
return wait.PollImmediate(poll, timeout, func() (bool, error) {
|
||||
pod, err := c.CoreV1().Pods(ns).Get(context.TODO(), name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
if isRetryableAPIError(err) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
return false, fmt.Errorf("failed to get app: %w", err)
|
||||
}
|
||||
switch pod.Status.Phase {
|
||||
@ -310,6 +331,7 @@ func waitForPodInRunningState(name, ns string, c kubernetes.Interface, t int, ex
|
||||
}
|
||||
if strings.Contains(events.String(), expectedError) {
|
||||
e2elog.Logf("Expected Error %q found successfully", expectedError)
|
||||
|
||||
return true, err
|
||||
}
|
||||
}
|
||||
@ -320,6 +342,7 @@ func waitForPodInRunningState(name, ns string, c kubernetes.Interface, t int, ex
|
||||
pod.Status.Phase,
|
||||
int(time.Since(start).Seconds()))
|
||||
}
|
||||
|
||||
return false, nil
|
||||
})
|
||||
}
|
||||
@ -332,6 +355,7 @@ func deletePod(name, ns string, c kubernetes.Interface, t int) error {
|
||||
}
|
||||
start := time.Now()
|
||||
e2elog.Logf("Waiting for pod %v to be deleted", name)
|
||||
|
||||
return wait.PollImmediate(poll, timeout, func() (bool, error) {
|
||||
_, err := c.CoreV1().Pods(ns).Get(context.TODO(), name, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
@ -342,8 +366,10 @@ func deletePod(name, ns string, c kubernetes.Interface, t int) error {
|
||||
return true, nil
|
||||
}
|
||||
e2elog.Logf("%s app to be deleted (%d seconds elapsed)", name, int(time.Since(start).Seconds()))
|
||||
|
||||
return false, fmt.Errorf("failed to get app: %w", err)
|
||||
}
|
||||
|
||||
return false, nil
|
||||
})
|
||||
}
|
||||
@ -353,6 +379,7 @@ func deletePodWithLabel(label, ns string, skipNotFound bool) error {
|
||||
if err != nil {
|
||||
e2elog.Logf("failed to delete pod %v", err)
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
@ -369,5 +396,6 @@ func calculateSHA512sum(f *framework.Framework, app *v1.Pod, filePath string, op
|
||||
// extract checksum from sha512sum output.
|
||||
checkSum := strings.Split(sha512sumOut, "")[0]
|
||||
e2elog.Logf("Calculated checksum %s", checkSum)
|
||||
|
||||
return checkSum, nil
|
||||
}
|
||||
|
||||
16
e2e/pvc.go
16
e2e/pvc.go
@ -23,6 +23,7 @@ func loadPVC(path string) (*v1.PersistentVolumeClaim, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return pvc, err
|
||||
}
|
||||
|
||||
@ -53,6 +54,7 @@ func createPVCAndvalidatePV(c kubernetes.Interface, pvc *v1.PersistentVolumeClai
|
||||
if apierrs.IsNotFound(err) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
return false, fmt.Errorf("failed to get pvc: %w", err)
|
||||
}
|
||||
|
||||
@ -68,6 +70,7 @@ func createPVCAndvalidatePV(c kubernetes.Interface, pvc *v1.PersistentVolumeClai
|
||||
if apierrs.IsNotFound(err) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
return false, fmt.Errorf("failed to get pv: %w", err)
|
||||
}
|
||||
err = e2epv.WaitOnPVandPVC(
|
||||
@ -79,6 +82,7 @@ func createPVCAndvalidatePV(c kubernetes.Interface, pvc *v1.PersistentVolumeClai
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("failed to wait for the pv and pvc to bind: %w", err)
|
||||
}
|
||||
|
||||
return true, nil
|
||||
})
|
||||
}
|
||||
@ -92,6 +96,7 @@ func createPVCAndPV(c kubernetes.Interface, pvc *v1.PersistentVolumeClaim, pv *v
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create pv: %w", err)
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
@ -125,6 +130,7 @@ func deletePVCAndPV(c kubernetes.Interface, pvc *v1.PersistentVolumeClaim, pv *v
|
||||
// FIXME: see https://github.com/ceph/ceph-csi/issues/1874
|
||||
e2elog.Logf("PVC %s is in a weird state: %s", pvcToDelete.Name, pvcToDelete.String())
|
||||
}
|
||||
|
||||
return false, nil
|
||||
}
|
||||
if isRetryableAPIError(err) {
|
||||
@ -145,6 +151,7 @@ func deletePVCAndPV(c kubernetes.Interface, pvc *v1.PersistentVolumeClaim, pv *v
|
||||
|
||||
start = time.Now()
|
||||
pvToDelete := pv
|
||||
|
||||
return wait.PollImmediate(poll, timeout, func() (bool, error) {
|
||||
// Check that the PV is deleted.
|
||||
e2elog.Logf(
|
||||
@ -179,6 +186,7 @@ func getPVCAndPV(
|
||||
if err != nil {
|
||||
return pvc, nil, fmt.Errorf("failed to get PV: %w", err)
|
||||
}
|
||||
|
||||
return pvc, pv, nil
|
||||
}
|
||||
|
||||
@ -203,6 +211,7 @@ func deletePVCAndValidatePV(c kubernetes.Interface, pvc *v1.PersistentVolumeClai
|
||||
return fmt.Errorf("delete of PVC %v failed: %w", name, err)
|
||||
}
|
||||
start := time.Now()
|
||||
|
||||
return wait.PollImmediate(poll, timeout, func() (bool, error) {
|
||||
// Check that the PVC is really deleted.
|
||||
e2elog.Logf(
|
||||
@ -252,6 +261,7 @@ func getBoundPV(client kubernetes.Interface, pvc *v1.PersistentVolumeClaim) (*v1
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get pv: %w", err)
|
||||
}
|
||||
|
||||
return pv, nil
|
||||
}
|
||||
|
||||
@ -289,6 +299,7 @@ func checkPVSelectorValuesForPVC(f *framework.Framework, pvc *v1.PersistentVolum
|
||||
return errors.New("unexpected key in node selector terms found in PV")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -303,14 +314,17 @@ func getMetricsForPVC(f *framework.Framework, pvc *v1.PersistentVolumeClaim, t i
|
||||
|
||||
// retry as kubelet does not immediately have the metrics available
|
||||
timeout := time.Duration(t) * time.Minute
|
||||
|
||||
return wait.PollImmediate(poll, timeout, func() (bool, error) {
|
||||
stdOut, stdErr, err := execCommandInToolBoxPod(f, cmd, rookNamespace)
|
||||
if err != nil {
|
||||
e2elog.Logf("failed to get metrics for pvc %q (%v): %v", pvc.Name, err, stdErr)
|
||||
|
||||
return false, nil
|
||||
}
|
||||
if stdOut == "" {
|
||||
e2elog.Logf("no metrics received from kublet on IP %s", kubelet)
|
||||
|
||||
return false, nil
|
||||
}
|
||||
|
||||
@ -324,11 +338,13 @@ func getMetricsForPVC(f *framework.Framework, pvc *v1.PersistentVolumeClaim, t i
|
||||
if strings.Contains(line, namespace) && strings.Contains(line, name) {
|
||||
// TODO: validate metrics if possible
|
||||
e2elog.Logf("found metrics for pvc %s/%s: %s", pvc.Namespace, pvc.Name, line)
|
||||
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
|
||||
e2elog.Logf("no metrics found for pvc %s/%s", pvc.Namespace, pvc.Name)
|
||||
|
||||
return false, nil
|
||||
})
|
||||
}
|
||||
|
||||
@ -522,9 +522,11 @@ var _ = Describe("RBD", func() {
|
||||
reason = fmt.Sprintf("failed to run ps cmd : %v, stdErr: %v", err, stdErr)
|
||||
}
|
||||
e2elog.Logf("%s", reason)
|
||||
|
||||
return false, nil
|
||||
}
|
||||
e2elog.Logf("attach command running after restart, runningAttachCmd: %v", runningAttachCmd)
|
||||
|
||||
return true, nil
|
||||
})
|
||||
|
||||
@ -1213,6 +1215,7 @@ var _ = Describe("RBD", func() {
|
||||
fmt.Printf("Checking prefix on %s\n", imgName)
|
||||
if strings.HasPrefix(imgName, volumeNamePrefix) {
|
||||
foundIt = true
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
@ -30,6 +30,7 @@ func imageSpec(pool, image string) string {
|
||||
if radosNamespace != "" {
|
||||
return pool + "/" + radosNamespace + "/" + image
|
||||
}
|
||||
|
||||
return pool + "/" + image
|
||||
}
|
||||
|
||||
@ -37,6 +38,7 @@ func rbdOptions(pool string) string {
|
||||
if radosNamespace != "" {
|
||||
return "--pool=" + pool + " --namespace " + radosNamespace
|
||||
}
|
||||
|
||||
return "--pool=" + pool
|
||||
}
|
||||
|
||||
@ -92,6 +94,7 @@ func createRBDStorageClass(
|
||||
}
|
||||
sc.ReclaimPolicy = &policy
|
||||
_, err = c.StorageV1().StorageClasses().Create(context.TODO(), &sc, metav1.CreateOptions{})
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
@ -133,6 +136,7 @@ func createRadosNamespace(f *framework.Framework) error {
|
||||
return fmt.Errorf("error creating rbd namespace %v", stdErr)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -149,6 +153,7 @@ func createRBDSecret(f *framework.Framework, secretName, userName, userKey strin
|
||||
sc.StringData["userKey"] = userKey
|
||||
sc.Namespace = cephCSINamespace
|
||||
_, err = f.ClientSet.CoreV1().Secrets(cephCSINamespace).Create(context.TODO(), &sc, metav1.CreateOptions{})
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
@ -184,6 +189,7 @@ func getImageInfoFromPVC(pvcNamespace, pvcName string, f *framework.Framework) (
|
||||
csiVolumeHandle: pv.Spec.CSI.VolumeHandle,
|
||||
pvName: pv.Name,
|
||||
}
|
||||
|
||||
return imageData, nil
|
||||
}
|
||||
|
||||
@ -196,6 +202,7 @@ func getImageMeta(rbdImageSpec, metaKey string, f *framework.Framework) (string,
|
||||
if stdErr != "" {
|
||||
return strings.TrimSpace(stdOut), fmt.Errorf(stdErr)
|
||||
}
|
||||
|
||||
return strings.TrimSpace(stdOut), nil
|
||||
}
|
||||
|
||||
@ -262,6 +269,7 @@ func logErrors(f *framework.Framework, msg string, wgErrs []error) int {
|
||||
failures++
|
||||
}
|
||||
}
|
||||
|
||||
return failures
|
||||
}
|
||||
|
||||
@ -388,6 +396,7 @@ func validateCloneInDifferentPool(f *framework.Framework, snapshotPool, cloneSc,
|
||||
validateRBDImageCount(f, 0, snapshotPool)
|
||||
validateRBDImageCount(f, 0, defaultRBDPool)
|
||||
validateRBDImageCount(f, 0, destImagePool)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -427,6 +436,7 @@ func validateEncryptedPVCAndAppBinding(pvcPath, appPath string, kms kmsConfig, f
|
||||
return fmt.Errorf("passphrase found in vault while should be deleted: %s", stdOut)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -457,6 +467,7 @@ func isThickPVC(f *framework.Framework, pvc *v1.PersistentVolumeClaim, app *v1.P
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to validate thick image: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -474,6 +485,7 @@ func validateThickImageMetadata(f *framework.Framework, pvc *v1.PersistentVolume
|
||||
if thickState == "" {
|
||||
return fmt.Errorf("image metadata is set for %s", rbdImageSpec)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -518,6 +530,7 @@ func listRBDImages(f *framework.Framework, pool string) ([]string, error) {
|
||||
if err != nil {
|
||||
return imgInfos, err
|
||||
}
|
||||
|
||||
return imgInfos, nil
|
||||
}
|
||||
|
||||
@ -529,6 +542,7 @@ func deleteBackingRBDImage(f *framework.Framework, pvc *v1.PersistentVolumeClaim
|
||||
|
||||
cmd := fmt.Sprintf("rbd rm %s %s", rbdOptions(defaultRBDPool), imageData.imageName)
|
||||
_, _, err = execCommandInToolBoxPod(f, cmd, rookNamespace)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
@ -586,6 +600,7 @@ func sparsifyBackingRBDImage(f *framework.Framework, pvc *v1.PersistentVolumeCla
|
||||
|
||||
cmd := fmt.Sprintf("rbd sparsify %s %s", rbdOptions(defaultRBDPool), imageData.imageName)
|
||||
_, _, err = execCommandInToolBoxPod(f, cmd, rookNamespace)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
@ -615,6 +630,7 @@ func deletePool(name string, cephfs bool, f *framework.Framework) error {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -632,6 +648,7 @@ func createPool(f *framework.Framework, name string) error {
|
||||
// ceph osd pool set replicapool size 1
|
||||
cmd = fmt.Sprintf("ceph osd pool set %s size %d --yes-i-really-mean-it", name, size)
|
||||
_, _, err = execCommandInToolBoxPod(f, cmd, rookNamespace)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
@ -876,6 +893,7 @@ func listRBDImagesInTrash(f *framework.Framework, poolName string) ([]trashInfo,
|
||||
if err != nil {
|
||||
return trashInfos, err
|
||||
}
|
||||
|
||||
return trashInfos, nil
|
||||
}
|
||||
|
||||
@ -892,11 +910,13 @@ func waitToRemoveImagesFromTrash(f *framework.Framework, poolName string, t int)
|
||||
}
|
||||
errReason = fmt.Errorf("found %d images found in trash. Image details %v", len(imagesInTrash), imagesInTrash)
|
||||
e2elog.Logf(errReason.Error())
|
||||
|
||||
return false, nil
|
||||
})
|
||||
|
||||
if errors.Is(err, wait.ErrWaitTimeout) {
|
||||
err = errReason
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
@ -36,6 +36,7 @@ func expandPVCSize(c kubernetes.Interface, pvc *v1.PersistentVolumeClaim, size s
|
||||
|
||||
start := time.Now()
|
||||
e2elog.Logf("Waiting up to %v to be in Resized state", pvc)
|
||||
|
||||
return wait.PollImmediate(poll, timeout, func() (bool, error) {
|
||||
e2elog.Logf("waiting for PVC %s (%d seconds elapsed)", updatedPVC.Name, int(time.Since(start).Seconds()))
|
||||
updatedPVC, err = c.CoreV1().
|
||||
@ -46,6 +47,7 @@ func expandPVCSize(c kubernetes.Interface, pvc *v1.PersistentVolumeClaim, size s
|
||||
if isRetryableAPIError(err) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
return false, fmt.Errorf("failed to get pvc: %w", err)
|
||||
}
|
||||
pvcConditions := updatedPVC.Status.Conditions
|
||||
@ -62,8 +64,10 @@ func expandPVCSize(c kubernetes.Interface, pvc *v1.PersistentVolumeClaim, size s
|
||||
"current size in status %v,expected size %v",
|
||||
updatedPVC.Status.Capacity[v1.ResourceStorage],
|
||||
resource.MustParse(size))
|
||||
|
||||
return false, nil
|
||||
}
|
||||
|
||||
return true, nil
|
||||
})
|
||||
}
|
||||
@ -143,16 +147,19 @@ func resizePVCAndValidateSize(pvcPath, appPath string, f *framework.Framework) e
|
||||
}
|
||||
}
|
||||
err = deletePVCAndApp("", f, resizePvc, app)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func checkDirSize(app *v1.Pod, f *framework.Framework, opt *metav1.ListOptions, size string) error {
|
||||
cmd := getDirSizeCheckCmd(app.Spec.Containers[0].VolumeMounts[0].MountPath)
|
||||
|
||||
return checkAppMntSize(f, opt, size, cmd, app.Namespace, deployTimeout)
|
||||
}
|
||||
|
||||
func checkDeviceSize(app *v1.Pod, f *framework.Framework, opt *metav1.ListOptions, size string) error {
|
||||
cmd := getDeviceSizeCheckCmd(app.Spec.Containers[0].VolumeDevices[0].DevicePath)
|
||||
|
||||
return checkAppMntSize(f, opt, size, cmd, app.Namespace, deployTimeout)
|
||||
}
|
||||
|
||||
@ -176,6 +183,7 @@ func checkAppMntSize(f *framework.Framework, opt *metav1.ListOptions, size, cmd,
|
||||
}
|
||||
if stdErr != "" {
|
||||
e2elog.Logf("failed to execute command in app pod %v", stdErr)
|
||||
|
||||
return false, nil
|
||||
}
|
||||
s := resource.MustParse(strings.TrimSpace(output))
|
||||
@ -190,8 +198,10 @@ func checkAppMntSize(f *framework.Framework, opt *metav1.ListOptions, size, cmd,
|
||||
}
|
||||
if actualSize != expectedSize {
|
||||
e2elog.Logf("expected size %s found %s information", size, output)
|
||||
|
||||
return false, nil
|
||||
}
|
||||
|
||||
return true, nil
|
||||
})
|
||||
}
|
||||
|
||||
@ -20,6 +20,7 @@ func getSnapshotClass(path string) snapapi.VolumeSnapshotClass {
|
||||
sc := snapapi.VolumeSnapshotClass{}
|
||||
err := unmarshal(path, &sc)
|
||||
Expect(err).Should(BeNil())
|
||||
|
||||
return sc
|
||||
}
|
||||
|
||||
@ -27,6 +28,7 @@ func getSnapshot(path string) snapapi.VolumeSnapshot {
|
||||
sc := snapapi.VolumeSnapshot{}
|
||||
err := unmarshal(path, &sc)
|
||||
Expect(err).Should(BeNil())
|
||||
|
||||
return sc
|
||||
}
|
||||
|
||||
@ -39,6 +41,7 @@ func newSnapshotClient() (*snapclient.SnapshotV1Client, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error creating snapshot client: %w", err)
|
||||
}
|
||||
|
||||
return c, err
|
||||
}
|
||||
|
||||
@ -74,6 +77,7 @@ func createSnapshot(snap *snapapi.VolumeSnapshot, t int) error {
|
||||
if apierrs.IsNotFound(err) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
return false, fmt.Errorf("failed to get volumesnapshot: %w", err)
|
||||
}
|
||||
if snaps.Status == nil || snaps.Status.ReadyToUse == nil {
|
||||
@ -83,6 +87,7 @@ func createSnapshot(snap *snapapi.VolumeSnapshot, t int) error {
|
||||
return true, nil
|
||||
}
|
||||
e2elog.Logf("snapshot %s in %v state", snap.Name, *snaps.Status.ReadyToUse)
|
||||
|
||||
return false, nil
|
||||
})
|
||||
}
|
||||
@ -149,6 +154,7 @@ func createRBDSnapshotClass(f *framework.Framework) error {
|
||||
return err
|
||||
}
|
||||
_, err = sclient.VolumeSnapshotClasses().Create(context.TODO(), &sc, metav1.CreateOptions{})
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
@ -160,6 +166,7 @@ func deleteRBDSnapshotClass() error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return sclient.VolumeSnapshotClasses().Delete(context.TODO(), sc.Name, metav1.DeleteOptions{})
|
||||
}
|
||||
|
||||
@ -185,6 +192,7 @@ func createCephFSSnapshotClass(f *framework.Framework) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create volumesnapshotclass: %w", err)
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
@ -207,5 +215,6 @@ func getVolumeSnapshotContent(namespace, snapshotName string) (*snapapi.VolumeSn
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get volumesnapshotcontent: %w", err)
|
||||
}
|
||||
|
||||
return volumeSnapshotContent, nil
|
||||
}
|
||||
|
||||
@ -184,6 +184,7 @@ func validateRBDStaticPV(f *framework.Framework, appPath string, isBlock, checkI
|
||||
|
||||
cmd = fmt.Sprintf("rbd rm %s %s", rbdImageName, rbdOptions(defaultRBDPool))
|
||||
_, _, err = execCommandInToolBoxPod(f, cmd, rookNamespace)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
@ -22,6 +22,7 @@ func upgradeCSI(version string) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to switch directory : %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -38,5 +39,6 @@ func upgradeAndDeployCSI(version, testtype string) error {
|
||||
default:
|
||||
return errors.New("incorrect test type, can be cephfs/rbd")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
26
e2e/utils.go
26
e2e/utils.go
@ -83,12 +83,14 @@ func getMons(ns string, c kubernetes.Interface) ([]string, error) {
|
||||
svcList.Items[i].Spec.Ports[0].Port)
|
||||
services = append(services, s)
|
||||
}
|
||||
|
||||
return services, nil
|
||||
}
|
||||
|
||||
func getStorageClass(path string) (scv1.StorageClass, error) {
|
||||
sc := scv1.StorageClass{}
|
||||
err := unmarshal(path, &sc)
|
||||
|
||||
return sc, err
|
||||
}
|
||||
|
||||
@ -102,6 +104,7 @@ func getSecret(path string) (v1.Secret, error) {
|
||||
return sc, err
|
||||
}
|
||||
}
|
||||
|
||||
return sc, nil
|
||||
}
|
||||
|
||||
@ -114,6 +117,7 @@ func deleteResource(scPath string) error {
|
||||
if err != nil {
|
||||
e2elog.Logf("failed to delete %s %v", scPath, err)
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
@ -128,6 +132,7 @@ func unmarshal(fileName string, obj interface{}) error {
|
||||
}
|
||||
|
||||
err = json.Unmarshal(data, obj)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
@ -149,6 +154,7 @@ func createPVCAndApp(
|
||||
return err
|
||||
}
|
||||
err = createApp(f.ClientSet, app, deployTimeout)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
@ -166,6 +172,7 @@ func deletePVCAndApp(name string, f *framework.Framework, pvc *v1.PersistentVolu
|
||||
return err
|
||||
}
|
||||
err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
@ -199,6 +206,7 @@ func validatePVCAndAppBinding(pvcPath, appPath string, f *framework.Framework) e
|
||||
return err
|
||||
}
|
||||
err = deletePVCAndApp("", f, pvc, app)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
@ -214,6 +222,7 @@ func getMountType(appName, appNamespace, mountPath string, f *framework.Framewor
|
||||
if stdErr != "" {
|
||||
return strings.TrimSpace(stdOut), fmt.Errorf(stdErr)
|
||||
}
|
||||
|
||||
return strings.TrimSpace(stdOut), nil
|
||||
}
|
||||
|
||||
@ -306,6 +315,7 @@ func validateNormalUserPVCAccess(pvcPath string, f *framework.Framework) error {
|
||||
}
|
||||
|
||||
err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
@ -393,6 +403,7 @@ func checkDataPersist(pvcPath, appPath string, f *framework.Framework) error {
|
||||
}
|
||||
|
||||
err = deletePVCAndApp("", f, pvc, app)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
@ -429,6 +440,7 @@ func pvcDeleteWhenPoolNotFound(pvcPath string, cephfs bool, f *framework.Framewo
|
||||
}
|
||||
}
|
||||
err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
@ -471,6 +483,7 @@ func checkMountOptions(pvcPath, appPath string, f *framework.Framework, mountFla
|
||||
}
|
||||
|
||||
err = deletePVCAndApp("", f, pvc, app)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
@ -481,6 +494,7 @@ func addTopologyDomainsToDSYaml(template, labels string) string {
|
||||
|
||||
func oneReplicaDeployYaml(template string) string {
|
||||
re := regexp.MustCompile(`(\s+replicas:) \d+`)
|
||||
|
||||
return re.ReplaceAllString(template, `$1 1`)
|
||||
}
|
||||
|
||||
@ -494,12 +508,14 @@ func writeDataAndCalChecksum(app *v1.Pod, opt *metav1.ListOptions, f *framework.
|
||||
err := writeDataInPod(app, opt, f)
|
||||
if err != nil {
|
||||
e2elog.Logf("failed to write data in the pod with error %v", err)
|
||||
|
||||
return "", err
|
||||
}
|
||||
|
||||
checkSum, err := calculateSHA512sum(f, app, filePath, opt)
|
||||
if err != nil {
|
||||
e2elog.Logf("failed to calculate checksum with error %v", err)
|
||||
|
||||
return checkSum, err
|
||||
}
|
||||
|
||||
@ -507,6 +523,7 @@ func writeDataAndCalChecksum(app *v1.Pod, opt *metav1.ListOptions, f *framework.
|
||||
if err != nil {
|
||||
e2elog.Failf("failed to delete pod with error %v", err)
|
||||
}
|
||||
|
||||
return checkSum, nil
|
||||
}
|
||||
|
||||
@ -1131,6 +1148,7 @@ func validateController(f *framework.Framework, pvcPath, appPath, scPath string)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return deleteResource(rbdExamplePath + "storageclass.yaml")
|
||||
}
|
||||
|
||||
@ -1172,6 +1190,7 @@ func waitForJobCompletion(c kubernetes.Interface, ns, job string, timeout int) e
|
||||
if isRetryableAPIError(err) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
return false, fmt.Errorf("failed to get Job: %w", err)
|
||||
}
|
||||
|
||||
@ -1183,6 +1202,7 @@ func waitForJobCompletion(c kubernetes.Interface, ns, job string, timeout int) e
|
||||
e2elog.Logf(
|
||||
"Job %s/%s has not completed yet (%d seconds elapsed)",
|
||||
ns, job, int(time.Since(start).Seconds()))
|
||||
|
||||
return false, nil
|
||||
})
|
||||
}
|
||||
@ -1211,6 +1231,7 @@ func retryKubectlInput(namespace string, action kubectlAction, data string, t in
|
||||
timeout := time.Duration(t) * time.Minute
|
||||
e2elog.Logf("waiting for kubectl (%s) to finish", action)
|
||||
start := time.Now()
|
||||
|
||||
return wait.PollImmediate(poll, timeout, func() (bool, error) {
|
||||
_, err := framework.RunKubectlInput(namespace, data, string(action), "-f", "-")
|
||||
if err != nil {
|
||||
@ -1221,8 +1242,10 @@ func retryKubectlInput(namespace string, action kubectlAction, data string, t in
|
||||
"will run kubectl (%s) again (%d seconds elapsed)",
|
||||
action,
|
||||
int(time.Since(start).Seconds()))
|
||||
|
||||
return false, fmt.Errorf("failed to run kubectl: %w", err)
|
||||
}
|
||||
|
||||
return true, nil
|
||||
})
|
||||
}
|
||||
@ -1234,6 +1257,7 @@ func retryKubectlFile(namespace string, action kubectlAction, filename string, t
|
||||
timeout := time.Duration(t) * time.Minute
|
||||
e2elog.Logf("waiting for kubectl (%s -f %q) to finish", action, filename)
|
||||
start := time.Now()
|
||||
|
||||
return wait.PollImmediate(poll, timeout, func() (bool, error) {
|
||||
_, err := framework.RunKubectl(namespace, string(action), "-f", filename)
|
||||
if err != nil {
|
||||
@ -1245,8 +1269,10 @@ func retryKubectlFile(namespace string, action kubectlAction, filename string, t
|
||||
action,
|
||||
filename,
|
||||
int(time.Since(start).Seconds()))
|
||||
|
||||
return false, fmt.Errorf("failed to run kubectl: %w", err)
|
||||
}
|
||||
|
||||
return true, nil
|
||||
})
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user