mirror of
https://github.com/ceph/ceph-csi.git
synced 2024-11-13 01:40:23 +00:00
418 lines
14 KiB
Go
418 lines
14 KiB
Go
/*
|
|
Copyright 2017 The Kubernetes Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package apimachinery
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
|
|
. "github.com/onsi/ginkgo"
|
|
. "github.com/onsi/gomega"
|
|
|
|
"k8s.io/api/admissionregistration/v1alpha1"
|
|
"k8s.io/api/core/v1"
|
|
"k8s.io/api/extensions/v1beta1"
|
|
"k8s.io/apimachinery/pkg/api/errors"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/types"
|
|
"k8s.io/apimachinery/pkg/util/wait"
|
|
clientset "k8s.io/client-go/kubernetes"
|
|
clientretry "k8s.io/client-go/util/retry"
|
|
"k8s.io/kubernetes/test/e2e/framework"
|
|
imageutils "k8s.io/kubernetes/test/utils/image"
|
|
)
|
|
|
|
var _ = SIGDescribe("Initializers [Feature:Initializers]", func() {
|
|
f := framework.NewDefaultFramework("initializers")
|
|
|
|
// TODO: Add failure traps once we have JustAfterEach
|
|
// See https://github.com/onsi/ginkgo/issues/303
|
|
|
|
It("should be invisible to controllers by default", func() {
|
|
ns := f.Namespace.Name
|
|
c := f.ClientSet
|
|
|
|
podName := "uninitialized-pod"
|
|
framework.Logf("Creating pod %s", podName)
|
|
|
|
ch := make(chan struct{})
|
|
go func() {
|
|
_, err := c.CoreV1().Pods(ns).Create(newUninitializedPod(podName))
|
|
Expect(err).NotTo(HaveOccurred())
|
|
close(ch)
|
|
}()
|
|
|
|
// wait to ensure the scheduler does not act on an uninitialized pod
|
|
err := wait.PollImmediate(2*time.Second, 15*time.Second, func() (bool, error) {
|
|
p, err := c.CoreV1().Pods(ns).Get(podName, metav1.GetOptions{})
|
|
if err != nil {
|
|
if errors.IsNotFound(err) {
|
|
return false, nil
|
|
}
|
|
return false, err
|
|
}
|
|
return len(p.Spec.NodeName) > 0, nil
|
|
})
|
|
Expect(err).To(Equal(wait.ErrWaitTimeout))
|
|
|
|
// verify that we can update an initializing pod
|
|
pod, err := c.CoreV1().Pods(ns).Get(podName, metav1.GetOptions{})
|
|
Expect(err).NotTo(HaveOccurred())
|
|
pod.Annotations = map[string]string{"update-1": "test"}
|
|
pod, err = c.CoreV1().Pods(ns).Update(pod)
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
// verify the list call filters out uninitialized pods
|
|
pods, err := c.CoreV1().Pods(ns).List(metav1.ListOptions{IncludeUninitialized: true})
|
|
Expect(err).NotTo(HaveOccurred())
|
|
Expect(pods.Items).To(HaveLen(1))
|
|
pods, err = c.CoreV1().Pods(ns).List(metav1.ListOptions{})
|
|
Expect(err).NotTo(HaveOccurred())
|
|
Expect(pods.Items).To(HaveLen(0))
|
|
|
|
// clear initializers
|
|
pod.Initializers = nil
|
|
pod, err = c.CoreV1().Pods(ns).Update(pod)
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
// pod should now start running
|
|
err = framework.WaitForPodRunningInNamespace(c, pod)
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
// ensure create call returns
|
|
<-ch
|
|
|
|
// verify that we cannot start the pod initializing again
|
|
pod, err = c.CoreV1().Pods(ns).Get(podName, metav1.GetOptions{})
|
|
Expect(err).NotTo(HaveOccurred())
|
|
pod.Initializers = &metav1.Initializers{
|
|
Pending: []metav1.Initializer{{Name: "Other"}},
|
|
}
|
|
_, err = c.CoreV1().Pods(ns).Update(pod)
|
|
if !errors.IsInvalid(err) || !strings.Contains(err.Error(), "immutable") {
|
|
Fail(fmt.Sprintf("expected invalid error: %v", err))
|
|
}
|
|
})
|
|
|
|
It("should dynamically register and apply initializers to pods [Serial]", func() {
|
|
ns := f.Namespace.Name
|
|
c := f.ClientSet
|
|
|
|
podName := "uninitialized-pod"
|
|
framework.Logf("Creating pod %s", podName)
|
|
|
|
// create and register an initializer
|
|
initializerName := "pod.test.e2e.kubernetes.io"
|
|
initializerConfigName := "e2e-test-initializer"
|
|
_, err := c.AdmissionregistrationV1alpha1().InitializerConfigurations().Create(&v1alpha1.InitializerConfiguration{
|
|
ObjectMeta: metav1.ObjectMeta{Name: initializerConfigName},
|
|
Initializers: []v1alpha1.Initializer{
|
|
{
|
|
Name: initializerName,
|
|
Rules: []v1alpha1.Rule{
|
|
{APIGroups: []string{""}, APIVersions: []string{"*"}, Resources: []string{"pods"}},
|
|
},
|
|
},
|
|
},
|
|
})
|
|
if errors.IsNotFound(err) {
|
|
framework.Skipf("dynamic configuration of initializers requires the alpha admissionregistration.k8s.io group to be enabled")
|
|
}
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
// we must remove the initializer when the test is complete and ensure no pods are pending for that initializer
|
|
defer cleanupInitializer(c, initializerConfigName, initializerName)
|
|
|
|
// poller configuration is 1 second, wait at least that long
|
|
time.Sleep(3 * time.Second)
|
|
|
|
// run create that blocks
|
|
ch := make(chan struct{})
|
|
go func() {
|
|
defer close(ch)
|
|
_, err := c.CoreV1().Pods(ns).Create(newInitPod(podName))
|
|
Expect(err).NotTo(HaveOccurred())
|
|
}()
|
|
|
|
// wait until the pod shows up uninitialized
|
|
By("Waiting until the pod is visible to a client")
|
|
var pod *v1.Pod
|
|
err = wait.PollImmediate(2*time.Second, 15*time.Second, func() (bool, error) {
|
|
pod, err = c.CoreV1().Pods(ns).Get(podName, metav1.GetOptions{IncludeUninitialized: true})
|
|
if errors.IsNotFound(err) {
|
|
return false, nil
|
|
}
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
return true, nil
|
|
})
|
|
Expect(err).NotTo(HaveOccurred())
|
|
Expect(pod.Initializers).NotTo(BeNil())
|
|
Expect(pod.Initializers.Pending).To(HaveLen(1))
|
|
Expect(pod.Initializers.Pending[0].Name).To(Equal(initializerName))
|
|
|
|
// pretend we are an initializer
|
|
By("Completing initialization")
|
|
pod.Initializers = nil
|
|
pod, err = c.CoreV1().Pods(ns).Update(pod)
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
// ensure create call returns
|
|
<-ch
|
|
|
|
// pod should now start running
|
|
err = framework.WaitForPodRunningInNamespace(c, pod)
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
// bypass initialization by explicitly passing an empty pending list
|
|
By("Setting an empty initializer as an admin to bypass initialization")
|
|
podName = "preinitialized-pod"
|
|
pod = newUninitializedPod(podName)
|
|
pod.Initializers.Pending = nil
|
|
pod, err = c.CoreV1().Pods(ns).Create(pod)
|
|
Expect(err).NotTo(HaveOccurred())
|
|
Expect(pod.Initializers).To(BeNil())
|
|
|
|
// bypass initialization for mirror pods
|
|
By("Creating a mirror pod that bypasses initialization")
|
|
podName = "mirror-pod"
|
|
pod = newInitPod(podName)
|
|
pod.Annotations = map[string]string{
|
|
v1.MirrorPodAnnotationKey: "true",
|
|
}
|
|
pod.Spec.NodeName = "node-does-not-yet-exist"
|
|
pod, err = c.CoreV1().Pods(ns).Create(pod)
|
|
Expect(err).NotTo(HaveOccurred())
|
|
Expect(pod.Initializers).To(BeNil())
|
|
Expect(pod.Annotations[v1.MirrorPodAnnotationKey]).To(Equal("true"))
|
|
})
|
|
|
|
It("don't cause replicaset controller creating extra pods if the initializer is not handled [Serial]", func() {
|
|
ns := f.Namespace.Name
|
|
c := f.ClientSet
|
|
|
|
podName := "uninitialized-pod"
|
|
framework.Logf("Creating pod %s", podName)
|
|
|
|
// create and register an initializer, without setting up a controller to handle it.
|
|
initializerName := "pod.test.e2e.kubernetes.io"
|
|
initializerConfigName := "e2e-test-initializer"
|
|
_, err := c.AdmissionregistrationV1alpha1().InitializerConfigurations().Create(&v1alpha1.InitializerConfiguration{
|
|
ObjectMeta: metav1.ObjectMeta{Name: initializerConfigName},
|
|
Initializers: []v1alpha1.Initializer{
|
|
{
|
|
Name: initializerName,
|
|
Rules: []v1alpha1.Rule{
|
|
{APIGroups: []string{""}, APIVersions: []string{"*"}, Resources: []string{"pods"}},
|
|
},
|
|
},
|
|
},
|
|
})
|
|
if errors.IsNotFound(err) {
|
|
framework.Skipf("dynamic configuration of initializers requires the alpha admissionregistration.k8s.io group to be enabled")
|
|
}
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
// we must remove the initializer when the test is complete and ensure no pods are pending for that initializer
|
|
defer cleanupInitializer(c, initializerConfigName, initializerName)
|
|
|
|
// poller configuration is 1 second, wait at least that long
|
|
time.Sleep(3 * time.Second)
|
|
|
|
// create a replicaset
|
|
persistedRS, err := c.ExtensionsV1beta1().ReplicaSets(ns).Create(newReplicaset())
|
|
Expect(err).NotTo(HaveOccurred())
|
|
// wait for replicaset controller to confirm that it has handled the creation
|
|
err = waitForRSObservedGeneration(c, persistedRS.Namespace, persistedRS.Name, persistedRS.Generation)
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
// update the replicaset spec to trigger a resync
|
|
patch := []byte(`{"spec":{"minReadySeconds":5}}`)
|
|
persistedRS, err = c.ExtensionsV1beta1().ReplicaSets(ns).Patch(persistedRS.Name, types.StrategicMergePatchType, patch)
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
// wait for replicaset controller to confirm that it has handle the spec update
|
|
err = waitForRSObservedGeneration(c, persistedRS.Namespace, persistedRS.Name, persistedRS.Generation)
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
// verify that the replicaset controller doesn't create extra pod
|
|
selector, err := metav1.LabelSelectorAsSelector(persistedRS.Spec.Selector)
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
listOptions := metav1.ListOptions{
|
|
LabelSelector: selector.String(),
|
|
IncludeUninitialized: true,
|
|
}
|
|
pods, err := c.CoreV1().Pods(ns).List(listOptions)
|
|
Expect(err).NotTo(HaveOccurred())
|
|
Expect(len(pods.Items)).Should(Equal(1))
|
|
})
|
|
|
|
It("will be set to nil if a patch removes the last pending initializer", func() {
|
|
ns := f.Namespace.Name
|
|
c := f.ClientSet
|
|
|
|
podName := "to-be-patch-initialized-pod"
|
|
framework.Logf("Creating pod %s", podName)
|
|
|
|
// TODO: lower the timeout so that the server responds faster.
|
|
_, err := c.CoreV1().Pods(ns).Create(newUninitializedPod(podName))
|
|
if err != nil && !errors.IsTimeout(err) {
|
|
framework.Failf("expect err to be timeout error, got %v", err)
|
|
}
|
|
uninitializedPod, err := c.CoreV1().Pods(ns).Get(podName, metav1.GetOptions{})
|
|
Expect(err).NotTo(HaveOccurred())
|
|
Expect(uninitializedPod.Initializers).NotTo(BeNil())
|
|
Expect(len(uninitializedPod.Initializers.Pending)).Should(Equal(1))
|
|
|
|
patch := fmt.Sprintf(`{"metadata":{"initializers":{"pending":[{"$patch":"delete","name":"%s"}]}}}`, uninitializedPod.Initializers.Pending[0].Name)
|
|
patchedPod, err := c.CoreV1().Pods(ns).Patch(uninitializedPod.Name, types.StrategicMergePatchType, []byte(patch))
|
|
Expect(err).NotTo(HaveOccurred())
|
|
Expect(patchedPod.Initializers).To(BeNil())
|
|
})
|
|
})
|
|
|
|
func newUninitializedPod(podName string) *v1.Pod {
|
|
pod := newInitPod(podName)
|
|
pod.Initializers = &metav1.Initializers{
|
|
Pending: []metav1.Initializer{{Name: "test.k8s.io"}},
|
|
}
|
|
return pod
|
|
}
|
|
|
|
func newReplicaset() *v1beta1.ReplicaSet {
|
|
name := "initializer-test-replicaset"
|
|
replicas := int32(1)
|
|
labels := map[string]string{"initializer-test": "single-replicaset"}
|
|
return &v1beta1.ReplicaSet{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: name,
|
|
},
|
|
Spec: v1beta1.ReplicaSetSpec{
|
|
Replicas: &replicas,
|
|
Template: v1.PodTemplateSpec{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Labels: labels,
|
|
},
|
|
Spec: v1.PodSpec{
|
|
TerminationGracePeriodSeconds: &zero,
|
|
Containers: []v1.Container{
|
|
{
|
|
Name: name + "-container",
|
|
Image: imageutils.GetE2EImage(imageutils.Porter),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
func newInitPod(podName string) *v1.Pod {
|
|
containerName := fmt.Sprintf("%s-container", podName)
|
|
port := 8080
|
|
pod := &v1.Pod{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: podName,
|
|
},
|
|
Spec: v1.PodSpec{
|
|
Containers: []v1.Container{
|
|
{
|
|
Name: containerName,
|
|
Image: imageutils.GetE2EImage(imageutils.Porter),
|
|
Env: []v1.EnvVar{{Name: fmt.Sprintf("SERVE_PORT_%d", port), Value: "foo"}},
|
|
Ports: []v1.ContainerPort{{ContainerPort: int32(port)}},
|
|
},
|
|
},
|
|
RestartPolicy: v1.RestartPolicyNever,
|
|
},
|
|
}
|
|
return pod
|
|
}
|
|
|
|
// removeInitializersFromAllPods walks all pods and ensures they don't have the provided initializer,
|
|
// to guarantee completing the test doesn't block the entire cluster.
|
|
func removeInitializersFromAllPods(c clientset.Interface, initializerName string) {
|
|
pods, err := c.CoreV1().Pods("").List(metav1.ListOptions{IncludeUninitialized: true})
|
|
if err != nil {
|
|
return
|
|
}
|
|
for _, p := range pods.Items {
|
|
if p.Initializers == nil {
|
|
continue
|
|
}
|
|
err := clientretry.RetryOnConflict(clientretry.DefaultRetry, func() error {
|
|
pod, err := c.CoreV1().Pods(p.Namespace).Get(p.Name, metav1.GetOptions{IncludeUninitialized: true})
|
|
if err != nil {
|
|
if errors.IsNotFound(err) {
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
if pod.Initializers == nil {
|
|
return nil
|
|
}
|
|
var updated []metav1.Initializer
|
|
for _, pending := range pod.Initializers.Pending {
|
|
if pending.Name != initializerName {
|
|
updated = append(updated, pending)
|
|
}
|
|
}
|
|
if len(updated) == len(pod.Initializers.Pending) {
|
|
return nil
|
|
}
|
|
pod.Initializers.Pending = updated
|
|
if len(updated) == 0 {
|
|
pod.Initializers = nil
|
|
}
|
|
framework.Logf("Found initializer on pod %s in ns %s", pod.Name, pod.Namespace)
|
|
_, err = c.CoreV1().Pods(p.Namespace).Update(pod)
|
|
return err
|
|
})
|
|
if err != nil {
|
|
framework.Logf("Unable to remove initializer from pod %s in ns %s: %v", p.Name, p.Namespace, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// remove the initializerConfig, and remove the initializer from all pods
|
|
func cleanupInitializer(c clientset.Interface, initializerConfigName, initializerName string) {
|
|
if err := c.AdmissionregistrationV1alpha1().InitializerConfigurations().Delete(initializerConfigName, nil); err != nil && !errors.IsNotFound(err) {
|
|
framework.Logf("got error on deleting %s", initializerConfigName)
|
|
}
|
|
// poller configuration is 1 second, wait at least that long
|
|
time.Sleep(3 * time.Second)
|
|
// clear our initializer from anyone who got it
|
|
removeInitializersFromAllPods(c, initializerName)
|
|
}
|
|
|
|
// waits till the RS status.observedGeneration matches metadata.generation.
|
|
func waitForRSObservedGeneration(c clientset.Interface, ns, name string, generation int64) error {
|
|
return wait.PollImmediate(1*time.Second, 1*time.Minute, func() (bool, error) {
|
|
rs, err := c.ExtensionsV1beta1().ReplicaSets(ns).Get(name, metav1.GetOptions{})
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
if generation > rs.Status.ObservedGeneration {
|
|
return false, nil
|
|
}
|
|
return true, nil
|
|
})
|
|
}
|