vendor update for CSI 0.3.0

This commit is contained in:
gman
2018-07-18 16:47:22 +02:00
parent 6f484f92fc
commit 8ea659f0d5
6810 changed files with 438061 additions and 193861 deletions

View File

@ -23,6 +23,7 @@ go_library(
"//pkg/client/metrics/prometheus:go_default_library",
"//pkg/version/prometheus:go_default_library",
"//vendor/github.com/spf13/pflag:go_default_library",
"//vendor/k8s.io/apiserver/pkg/server:go_default_library",
"//vendor/k8s.io/apiserver/pkg/util/flag:go_default_library",
"//vendor/k8s.io/apiserver/pkg/util/logs:go_default_library",
],

View File

@ -18,4 +18,6 @@ reviewers:
- liggitt
- nikhiljindal
- ncdc
- sttts
- sttts
- hzxuzhonghu
- CaoShuFeng

View File

@ -27,6 +27,7 @@ import (
"github.com/spf13/pflag"
"k8s.io/apiserver/pkg/server"
utilflag "k8s.io/apiserver/pkg/util/flag"
"k8s.io/apiserver/pkg/util/logs"
"k8s.io/kubernetes/cmd/kube-apiserver/app"
@ -37,7 +38,7 @@ import (
func main() {
rand.Seed(time.Now().UTC().UnixNano())
command := app.NewAPIServerCommand()
command := app.NewAPIServerCommand(server.SetupSignalHandler())
// TODO: once we switch everything over to Cobra commands, we can go back to calling
// utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the

View File

@ -70,6 +70,7 @@ go_library(
"//vendor/k8s.io/apiserver/pkg/admission/plugin/webhook/initializer:go_default_library",
"//vendor/k8s.io/apiserver/pkg/authentication/authenticator:go_default_library",
"//vendor/k8s.io/apiserver/pkg/authorization/authorizer:go_default_library",
"//vendor/k8s.io/apiserver/pkg/endpoints/openapi:go_default_library",
"//vendor/k8s.io/apiserver/pkg/server:go_default_library",
"//vendor/k8s.io/apiserver/pkg/server/filters:go_default_library",
"//vendor/k8s.io/apiserver/pkg/server/healthz:go_default_library",
@ -78,9 +79,11 @@ go_library(
"//vendor/k8s.io/apiserver/pkg/server/storage:go_default_library",
"//vendor/k8s.io/apiserver/pkg/storage/etcd3/preflight:go_default_library",
"//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
"//vendor/k8s.io/client-go/discovery/cached:go_default_library",
"//vendor/k8s.io/client-go/informers:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
"//vendor/k8s.io/client-go/rest:go_default_library",
"//vendor/k8s.io/client-go/restmapper:go_default_library",
"//vendor/k8s.io/client-go/tools/cache:go_default_library",
"//vendor/k8s.io/client-go/util/cert:go_default_library",
"//vendor/k8s.io/kube-aggregator/pkg/apis/apiregistration:go_default_library",

View File

@ -32,6 +32,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apiserver/pkg/admission"
genericapiserver "k8s.io/apiserver/pkg/server"
"k8s.io/apiserver/pkg/server/healthz"
genericoptions "k8s.io/apiserver/pkg/server/options"
@ -49,11 +50,27 @@ import (
"k8s.io/kubernetes/pkg/master/controller/crdregistration"
)
func createAggregatorConfig(kubeAPIServerConfig genericapiserver.Config, commandOptions *options.ServerRunOptions, externalInformers kubeexternalinformers.SharedInformerFactory, serviceResolver aggregatorapiserver.ServiceResolver, proxyTransport *http.Transport) (*aggregatorapiserver.Config, error) {
func createAggregatorConfig(
kubeAPIServerConfig genericapiserver.Config,
commandOptions *options.ServerRunOptions,
externalInformers kubeexternalinformers.SharedInformerFactory,
serviceResolver aggregatorapiserver.ServiceResolver,
proxyTransport *http.Transport,
pluginInitializers []admission.PluginInitializer,
) (*aggregatorapiserver.Config, error) {
// make a shallow copy to let us twiddle a few things
// most of the config actually remains the same. We only need to mess with a couple items related to the particulars of the aggregator
genericConfig := kubeAPIServerConfig
// override genericConfig.AdmissionControl with kube-aggregator's scheme,
// because aggregator apiserver should use its own scheme to convert its own resources.
commandOptions.Admission.ApplyTo(
&genericConfig,
externalInformers,
genericConfig.LoopbackClientConfig,
aggregatorscheme.Scheme,
pluginInitializers...)
// the aggregator doesn't wire these up. It just delegates them to the kubeapiserver
genericConfig.EnableSwaggerUI = false
genericConfig.SwaggerConfig = nil
@ -67,7 +84,7 @@ func createAggregatorConfig(kubeAPIServerConfig genericapiserver.Config, command
if err := commandOptions.APIEnablement.ApplyTo(
&genericConfig,
aggregatorapiserver.DefaultAPIResourceConfigSource(),
aggregatorscheme.Registry); err != nil {
aggregatorscheme.Scheme); err != nil {
return nil, err
}
@ -122,7 +139,10 @@ func createAggregatorServer(aggregatorConfig *aggregatorapiserver.Config, delega
go func() {
// let the CRD controller process the initial set of CRDs before starting the autoregistration controller.
// this prevents the autoregistration controller's initial sync from deleting APIServices for CRDs that still exist.
crdRegistrationController.WaitForInitialSync()
// we only need to do this if CRDs are enabled on this server. We can't use discovery because we are the source for discovery.
if aggregatorConfig.GenericConfig.MergedResourceConfig.AnyVersionForGroupEnabled("apiextensions.k8s.io") {
crdRegistrationController.WaitForInitialSync()
}
autoRegistrationController.Run(5, context.StopCh)
}()
return nil
@ -243,6 +263,7 @@ var apiVersionPriorities = map[schema.GroupVersion]priority{
{Group: "admissionregistration.k8s.io", Version: "v1"}: {group: 16700, version: 15},
{Group: "admissionregistration.k8s.io", Version: "v1beta1"}: {group: 16700, version: 12},
{Group: "admissionregistration.k8s.io", Version: "v1alpha1"}: {group: 16700, version: 9},
{Group: "scheduling.k8s.io", Version: "v1beta1"}: {group: 16600, version: 12},
{Group: "scheduling.k8s.io", Version: "v1alpha1"}: {group: 16600, version: 9},
// Append a new group to the end of the list if unsure.
// You can use min(existing group)-100 as the initial value for a group.

View File

@ -23,17 +23,33 @@ import (
"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
apiextensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver"
apiextensionscmd "k8s.io/apiextensions-apiserver/pkg/cmd/server"
"k8s.io/apiserver/pkg/admission"
genericapiserver "k8s.io/apiserver/pkg/server"
genericoptions "k8s.io/apiserver/pkg/server/options"
kubeexternalinformers "k8s.io/client-go/informers"
"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
)
func createAPIExtensionsConfig(kubeAPIServerConfig genericapiserver.Config, externalInformers kubeexternalinformers.SharedInformerFactory, commandOptions *options.ServerRunOptions) (*apiextensionsapiserver.Config, error) {
func createAPIExtensionsConfig(
kubeAPIServerConfig genericapiserver.Config,
externalInformers kubeexternalinformers.SharedInformerFactory,
pluginInitializers []admission.PluginInitializer,
commandOptions *options.ServerRunOptions,
masterCount int,
) (*apiextensionsapiserver.Config, error) {
// make a shallow copy to let us twiddle a few things
// most of the config actually remains the same. We only need to mess with a couple items related to the particulars of the apiextensions
genericConfig := kubeAPIServerConfig
// override genericConfig.AdmissionControl with apiextensions' scheme,
// because apiextentions apiserver should use its own scheme to convert resources.
commandOptions.Admission.ApplyTo(
&genericConfig,
externalInformers,
genericConfig.LoopbackClientConfig,
apiextensionsapiserver.Scheme,
pluginInitializers...)
// copy the etcd options so we don't mutate originals.
etcdOptions := *commandOptions.Etcd
etcdOptions.StorageConfig.Codec = apiextensionsapiserver.Codecs.LegacyCodec(v1beta1.SchemeGroupVersion)
@ -43,7 +59,7 @@ func createAPIExtensionsConfig(kubeAPIServerConfig genericapiserver.Config, exte
if err := commandOptions.APIEnablement.ApplyTo(
&genericConfig,
apiextensionsapiserver.DefaultAPIResourceConfigSource(),
apiextensionsapiserver.Registry); err != nil {
apiextensionsapiserver.Scheme); err != nil {
return nil, err
}
@ -54,6 +70,7 @@ func createAPIExtensionsConfig(kubeAPIServerConfig genericapiserver.Config, exte
},
ExtraConfig: apiextensionsapiserver.ExtraConfig{
CRDRESTOptionsGetter: apiextensionscmd.NewCRDRESTOptionsGetter(etcdOptions),
MasterCount: masterCount,
},
}
@ -61,10 +78,5 @@ func createAPIExtensionsConfig(kubeAPIServerConfig genericapiserver.Config, exte
}
func createAPIExtensionsServer(apiextensionsConfig *apiextensionsapiserver.Config, delegateAPIServer genericapiserver.DelegationTarget) (*apiextensionsapiserver.CustomResourceDefinitions, error) {
apiextensionsServer, err := apiextensionsConfig.Complete().New(delegateAPIServer)
if err != nil {
return nil, err
}
return apiextensionsServer, nil
return apiextensionsConfig.Complete().New(delegateAPIServer)
}

View File

@ -16,7 +16,6 @@ go_library(
deps = [
"//pkg/api/legacyscheme:go_default_library",
"//pkg/apis/core:go_default_library",
"//pkg/apis/core/validation:go_default_library",
"//pkg/features:go_default_library",
"//pkg/kubeapiserver/options:go_default_library",
"//pkg/kubelet/client:go_default_library",
@ -46,7 +45,8 @@ go_test(
"//vendor/k8s.io/apiserver/pkg/server/options:go_default_library",
"//vendor/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library",
"//vendor/k8s.io/apiserver/pkg/util/flag:go_default_library",
"//vendor/k8s.io/apiserver/plugin/pkg/audit/webhook:go_default_library",
"//vendor/k8s.io/apiserver/plugin/pkg/audit/buffered:go_default_library",
"//vendor/k8s.io/apiserver/plugin/pkg/audit/truncate:go_default_library",
"//vendor/k8s.io/client-go/rest:go_default_library",
],
)

View File

@ -26,7 +26,6 @@ import (
genericoptions "k8s.io/apiserver/pkg/server/options"
"k8s.io/apiserver/pkg/storage/storagebackend"
api "k8s.io/kubernetes/pkg/apis/core"
"k8s.io/kubernetes/pkg/apis/core/validation"
kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options"
kubeletclient "k8s.io/kubernetes/pkg/kubelet/client"
"k8s.io/kubernetes/pkg/master/ports"
@ -94,7 +93,7 @@ func NewServerRunOptions() *ServerRunOptions {
EnableLogsHandler: true,
EventTTL: 1 * time.Hour,
MasterCount: 1,
EndpointReconcilerType: string(reconcilers.MasterCountReconcilerType),
EndpointReconcilerType: string(reconcilers.LeaseEndpointReconcilerType),
KubeletConfig: kubeletclient.KubeletClientConfig{
Port: ports.KubeletPort,
ReadOnlyPort: ports.KubeletReadOnlyPort,
@ -167,7 +166,7 @@ func (s *ServerRunOptions) AddFlags(fs *pflag.FlagSet) {
"Currently only applies to long-running requests.")
fs.IntVar(&s.MasterCount, "apiserver-count", s.MasterCount,
"The number of apiservers running in the cluster, must be a positive number.")
"The number of apiservers running in the cluster, must be a positive number. (In use when --endpoint-reconciler-type=master-count is enabled.)")
fs.StringVar(&s.EndpointReconcilerType, "endpoint-reconciler-type", string(s.EndpointReconcilerType),
"Use an endpoint reconciler ("+strings.Join(reconcilers.AllTypes.Names(), ", ")+")")
@ -213,11 +212,10 @@ func (s *ServerRunOptions) AddFlags(fs *pflag.FlagSet) {
fs.StringVar(&s.KubeletConfig.CAFile, "kubelet-certificate-authority", s.KubeletConfig.CAFile,
"Path to a cert file for the certificate authority.")
// TODO: delete this flag as soon as we identify and fix all clients that send malformed updates, like #14126.
fs.BoolVar(&validation.RepairMalformedUpdates, "repair-malformed-updates", validation.RepairMalformedUpdates, ""+
"If true, server will do its best to fix the update request to pass the validation, "+
"e.g., setting empty UID in update request to its existing value. This flag can be turned off "+
"after we fix all the clients that send malformed updates.")
// TODO: delete this flag in 1.13
repair := false
fs.BoolVar(&repair, "repair-malformed-updates", false, "deprecated")
fs.MarkDeprecated("repair-malformed-updates", "This flag will be removed in a future version")
fs.StringVar(&s.ProxyClientCertFile, "proxy-client-cert-file", s.ProxyClientCertFile, ""+
"Client certificate used to prove the identity of the aggregator or kube-apiserver "+
@ -236,6 +234,5 @@ func (s *ServerRunOptions) AddFlags(fs *pflag.FlagSet) {
"Turns on aggregator routing requests to endoints IP rather than cluster IP.")
fs.StringVar(&s.ServiceAccountSigningKeyFile, "service-account-signing-key-file", s.ServiceAccountSigningKeyFile, ""+
"Path to the file that contains the current private key of the service account token issuer. The issuer will sign issued ID tokens with this private key. (Ignored unless alpha TokenRequest is enabled")
"Path to the file that contains the current private key of the service account token issuer. The issuer will sign issued ID tokens with this private key. (Requires the 'TokenRequest' feature gate.)")
}

View File

@ -29,7 +29,8 @@ import (
genericoptions "k8s.io/apiserver/pkg/server/options"
"k8s.io/apiserver/pkg/storage/storagebackend"
utilflag "k8s.io/apiserver/pkg/util/flag"
auditwebhook "k8s.io/apiserver/plugin/pkg/audit/webhook"
auditbuffered "k8s.io/apiserver/plugin/pkg/audit/buffered"
audittruncate "k8s.io/apiserver/plugin/pkg/audit/truncate"
restclient "k8s.io/client-go/rest"
"k8s.io/kubernetes/pkg/api/legacyscheme"
kapi "k8s.io/kubernetes/pkg/apis/core"
@ -54,18 +55,34 @@ func TestAddFlags(t *testing.T) {
"--audit-log-maxbackup=12",
"--audit-log-maxsize=13",
"--audit-log-path=/var/log",
"--audit-log-mode=blocking",
"--audit-log-batch-buffer-size=46",
"--audit-log-batch-max-size=47",
"--audit-log-batch-max-wait=48s",
"--audit-log-batch-throttle-enable=true",
"--audit-log-batch-throttle-qps=49.5",
"--audit-log-batch-throttle-burst=50",
"--audit-log-truncate-enabled=true",
"--audit-log-truncate-max-batch-size=45",
"--audit-log-truncate-max-event-size=44",
"--audit-log-version=audit.k8s.io/v1alpha1",
"--audit-policy-file=/policy",
"--audit-webhook-config-file=/webhook-config",
"--audit-webhook-mode=blocking",
"--audit-webhook-batch-buffer-size=42",
"--audit-webhook-batch-max-size=43",
"--audit-webhook-batch-max-wait=1s",
"--audit-webhook-batch-throttle-enable=false",
"--audit-webhook-batch-throttle-qps=43.5",
"--audit-webhook-batch-throttle-burst=44",
"--audit-webhook-batch-initial-backoff=2s",
"--audit-webhook-truncate-enabled=true",
"--audit-webhook-truncate-max-batch-size=43",
"--audit-webhook-truncate-max-event-size=42",
"--audit-webhook-initial-backoff=2s",
"--audit-webhook-version=audit.k8s.io/v1alpha1",
"--authentication-token-webhook-cache-ttl=3m",
"--authentication-token-webhook-config-file=/token-webhook-config",
"--authorization-mode=AlwaysDeny",
"--authorization-mode=AlwaysDeny,RBAC",
"--authorization-policy-file=/policy",
"--authorization-webhook-cache-authorized-ttl=3m",
"--authorization-webhook-cache-unauthorized-ttl=1m",
@ -79,7 +96,7 @@ func TestAddFlags(t *testing.T) {
"--enable-aggregator-routing=true",
"--enable-logs-handler=false",
"--enable-swagger-ui=true",
"--endpoint-reconciler-type=" + string(reconcilers.MasterCountReconcilerType),
"--endpoint-reconciler-type=" + string(reconcilers.LeaseEndpointReconcilerType),
"--etcd-quorum-read=false",
"--etcd-keyfile=/var/run/kubernetes/etcd.key",
"--etcd-certfile=/var/run/kubernetes/etcdce.crt",
@ -103,7 +120,7 @@ func TestAddFlags(t *testing.T) {
ServiceNodePortRange: kubeoptions.DefaultServiceNodePortRange,
ServiceClusterIPRange: kubeoptions.DefaultServiceIPCIDR,
MasterCount: 5,
EndpointReconcilerType: string(reconcilers.MasterCountReconcilerType),
EndpointReconcilerType: string(reconcilers.LeaseEndpointReconcilerType),
AllowPrivileged: false,
GenericServerRunOptions: &apiserveroptions.ServerRunOptions{
AdvertiseAddress: net.ParseIP("192.168.10.10"),
@ -180,18 +197,48 @@ func TestAddFlags(t *testing.T) {
MaxBackups: 12,
MaxSize: 13,
Format: "json",
BatchOptions: apiserveroptions.AuditBatchOptions{
Mode: "blocking",
BatchConfig: auditbuffered.BatchConfig{
BufferSize: 46,
MaxBatchSize: 47,
MaxBatchWait: 48 * time.Second,
ThrottleEnable: true,
ThrottleQPS: 49.5,
ThrottleBurst: 50,
},
},
TruncateOptions: apiserveroptions.AuditTruncateOptions{
Enabled: true,
TruncateConfig: audittruncate.Config{
MaxBatchSize: 45,
MaxEventSize: 44,
},
},
GroupVersionString: "audit.k8s.io/v1alpha1",
},
WebhookOptions: apiserveroptions.AuditWebhookOptions{
Mode: "blocking",
ConfigFile: "/webhook-config",
BatchConfig: auditwebhook.BatchBackendConfig{
BufferSize: 42,
MaxBatchSize: 43,
MaxBatchWait: 1 * time.Second,
ThrottleQPS: 43.5,
ThrottleBurst: 44,
InitialBackoff: 2 * time.Second,
BatchOptions: apiserveroptions.AuditBatchOptions{
Mode: "blocking",
BatchConfig: auditbuffered.BatchConfig{
BufferSize: 42,
MaxBatchSize: 43,
MaxBatchWait: 1 * time.Second,
ThrottleEnable: false,
ThrottleQPS: 43.5,
ThrottleBurst: 44,
},
},
TruncateOptions: apiserveroptions.AuditTruncateOptions{
Enabled: true,
TruncateConfig: audittruncate.Config{
MaxBatchSize: 43,
MaxEventSize: 42,
},
},
InitialBackoff: 2 * time.Second,
GroupVersionString: "audit.k8s.io/v1alpha1",
},
PolicyFile: "/policy",
},
@ -226,7 +273,7 @@ func TestAddFlags(t *testing.T) {
TokenFailureCacheTTL: 0,
},
Authorization: &kubeoptions.BuiltInAuthorizationOptions{
Mode: "AlwaysDeny",
Modes: []string{"AlwaysDeny", "RBAC"},
PolicyFile: "/policy",
WebhookConfigFile: "/webhook-config",
WebhookCacheAuthorizedTTL: 180000000000,
@ -237,8 +284,8 @@ func TestAddFlags(t *testing.T) {
CloudProvider: "azure",
},
StorageSerialization: &kubeoptions.StorageSerializationOptions{
StorageVersions: legacyscheme.Registry.AllPreferredGroupVersions(),
DefaultStorageVersions: legacyscheme.Registry.AllPreferredGroupVersions(),
StorageVersions: kubeoptions.ToPreferredVersionString(legacyscheme.Scheme.PreferredVersionAllGroups()),
DefaultStorageVersions: kubeoptions.ToPreferredVersionString(legacyscheme.Scheme.PreferredVersionAllGroups()),
},
APIEnablement: &apiserveroptions.APIEnablementOptions{
RuntimeConfig: utilflag.ConfigurationMap{},

View File

@ -50,36 +50,39 @@ func validateServiceNodePort(options *ServerRunOptions) []error {
}
// Validate checks ServerRunOptions and return a slice of found errors.
func (options *ServerRunOptions) Validate() []error {
func (s *ServerRunOptions) Validate() []error {
var errors []error
if errs := options.Etcd.Validate(); len(errs) > 0 {
if errs := s.Etcd.Validate(); len(errs) > 0 {
errors = append(errors, errs...)
}
if errs := validateClusterIPFlags(options); len(errs) > 0 {
if errs := validateClusterIPFlags(s); len(errs) > 0 {
errors = append(errors, errs...)
}
if errs := validateServiceNodePort(options); len(errs) > 0 {
if errs := validateServiceNodePort(s); len(errs) > 0 {
errors = append(errors, errs...)
}
if errs := options.SecureServing.Validate(); len(errs) > 0 {
if errs := s.SecureServing.Validate(); len(errs) > 0 {
errors = append(errors, errs...)
}
if errs := options.Authentication.Validate(); len(errs) > 0 {
if errs := s.Authentication.Validate(); len(errs) > 0 {
errors = append(errors, errs...)
}
if errs := options.Audit.Validate(); len(errs) > 0 {
if errs := s.Authorization.Validate(); len(errs) > 0 {
errors = append(errors, errs...)
}
if errs := options.Admission.Validate(); len(errs) > 0 {
if errs := s.Audit.Validate(); len(errs) > 0 {
errors = append(errors, errs...)
}
if errs := options.InsecureServing.Validate("insecure-port"); len(errs) > 0 {
if errs := s.Admission.Validate(); len(errs) > 0 {
errors = append(errors, errs...)
}
if options.MasterCount <= 0 {
errors = append(errors, fmt.Errorf("--apiserver-count should be a positive number, but value '%d' provided", options.MasterCount))
if errs := s.InsecureServing.Validate(); len(errs) > 0 {
errors = append(errors, errs...)
}
if errs := options.APIEnablement.Validate(legacyscheme.Registry, apiextensionsapiserver.Registry, aggregatorscheme.Registry); len(errs) > 0 {
if s.MasterCount <= 0 {
errors = append(errors, fmt.Errorf("--apiserver-count should be a positive number, but value '%d' provided", s.MasterCount))
}
if errs := s.APIEnablement.Validate(legacyscheme.Scheme, apiextensionsapiserver.Scheme, aggregatorscheme.Scheme); len(errs) > 0 {
errors = append(errors, errs...)
}

View File

@ -27,7 +27,6 @@ import (
"net/http"
"net/url"
"os"
"reflect"
"strconv"
"strings"
"time"
@ -36,6 +35,7 @@ import (
"github.com/golang/glog"
"github.com/spf13/cobra"
extensionsapiserver "k8s.io/apiextensions-apiserver/pkg/apiserver"
"k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
@ -47,7 +47,7 @@ import (
webhookinit "k8s.io/apiserver/pkg/admission/plugin/webhook/initializer"
"k8s.io/apiserver/pkg/authentication/authenticator"
"k8s.io/apiserver/pkg/authorization/authorizer"
"k8s.io/apiserver/pkg/server"
openapinamer "k8s.io/apiserver/pkg/endpoints/openapi"
genericapiserver "k8s.io/apiserver/pkg/server"
"k8s.io/apiserver/pkg/server/filters"
serveroptions "k8s.io/apiserver/pkg/server/options"
@ -55,11 +55,14 @@ import (
serverstorage "k8s.io/apiserver/pkg/server/storage"
"k8s.io/apiserver/pkg/storage/etcd3/preflight"
utilfeature "k8s.io/apiserver/pkg/util/feature"
cacheddiscovery "k8s.io/client-go/discovery/cached"
clientgoinformers "k8s.io/client-go/informers"
clientgoclientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/restmapper"
certutil "k8s.io/client-go/util/cert"
aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver"
aggregatorscheme "k8s.io/kube-aggregator/pkg/apiserver/scheme"
openapi "k8s.io/kube-openapi/pkg/common"
"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
"k8s.io/kubernetes/pkg/api/legacyscheme"
@ -105,7 +108,7 @@ const etcdRetryLimit = 60
const etcdRetryInterval = 1 * time.Second
// NewAPIServerCommand creates a *cobra.Command object with default parameters
func NewAPIServerCommand() *cobra.Command {
func NewAPIServerCommand(stopCh <-chan struct{}) *cobra.Command {
s := options.NewServerRunOptions()
cmd := &cobra.Command{
Use: "kube-apiserver",
@ -113,15 +116,22 @@ func NewAPIServerCommand() *cobra.Command {
for the api objects which include pods, services, replicationcontrollers, and
others. The API Server services REST operations and provides the frontend to the
cluster's shared state through which all other components interact.`,
Run: func(cmd *cobra.Command, args []string) {
RunE: func(cmd *cobra.Command, args []string) error {
verflag.PrintAndExitIfRequested()
utilflag.PrintFlags(cmd.Flags())
stopCh := server.SetupSignalHandler()
if err := Run(s, stopCh); err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
os.Exit(1)
// set default options
completedOptions, err := Complete(s)
if err != nil {
return err
}
// validate options
if errs := completedOptions.Validate(); len(errs) != 0 {
return utilerrors.NewAggregate(errs)
}
return Run(completedOptions, stopCh)
},
}
s.AddFlags(cmd.Flags())
@ -130,11 +140,11 @@ cluster's shared state through which all other components interact.`,
}
// Run runs the specified APIServer. This should never exit.
func Run(runOptions *options.ServerRunOptions, stopCh <-chan struct{}) error {
func Run(completeOptions completedServerRunOptions, stopCh <-chan struct{}) error {
// To help debugging, immediately log version
glog.Infof("Version: %+v", version.Get())
server, err := CreateServerChain(runOptions, stopCh)
server, err := CreateServerChain(completeOptions, stopCh)
if err != nil {
return err
}
@ -143,46 +153,32 @@ func Run(runOptions *options.ServerRunOptions, stopCh <-chan struct{}) error {
}
// CreateServerChain creates the apiservers connected via delegation.
func CreateServerChain(runOptions *options.ServerRunOptions, stopCh <-chan struct{}) (*genericapiserver.GenericAPIServer, error) {
nodeTunneler, proxyTransport, err := CreateNodeDialer(runOptions)
func CreateServerChain(completedOptions completedServerRunOptions, stopCh <-chan struct{}) (*genericapiserver.GenericAPIServer, error) {
nodeTunneler, proxyTransport, err := CreateNodeDialer(completedOptions)
if err != nil {
return nil, err
}
kubeAPIServerConfig, sharedInformers, versionedInformers, insecureServingOptions, serviceResolver, err := CreateKubeAPIServerConfig(runOptions, nodeTunneler, proxyTransport)
kubeAPIServerConfig, sharedInformers, versionedInformers, insecureServingOptions, serviceResolver, pluginInitializer, admissionPostStartHook, err := CreateKubeAPIServerConfig(completedOptions, nodeTunneler, proxyTransport)
if err != nil {
return nil, err
}
// TPRs are enabled and not yet beta, since this these are the successor, they fall under the same enablement rule
// If additional API servers are added, they should be gated.
apiExtensionsConfig, err := createAPIExtensionsConfig(*kubeAPIServerConfig.GenericConfig, versionedInformers, runOptions)
apiExtensionsConfig, err := createAPIExtensionsConfig(*kubeAPIServerConfig.GenericConfig, versionedInformers, pluginInitializer, completedOptions.ServerRunOptions, completedOptions.MasterCount)
if err != nil {
return nil, err
}
apiExtensionsServer, err := createAPIExtensionsServer(apiExtensionsConfig, genericapiserver.EmptyDelegate)
apiExtensionsServer, err := createAPIExtensionsServer(apiExtensionsConfig, genericapiserver.NewEmptyDelegate())
if err != nil {
return nil, err
}
kubeAPIServer, err := CreateKubeAPIServer(kubeAPIServerConfig, apiExtensionsServer.GenericAPIServer, sharedInformers, versionedInformers)
kubeAPIServer, err := CreateKubeAPIServer(kubeAPIServerConfig, apiExtensionsServer.GenericAPIServer, sharedInformers, versionedInformers, admissionPostStartHook)
if err != nil {
return nil, err
}
// if we're starting up a hacked up version of this API server for a weird test case,
// just start the API server as is because clients don't get built correctly when you do this
if len(os.Getenv("KUBE_API_VERSIONS")) > 0 {
if insecureServingOptions != nil {
insecureHandlerChain := kubeserver.BuildInsecureHandlerChain(kubeAPIServer.GenericAPIServer.UnprotectedHandler(), kubeAPIServerConfig.GenericConfig)
if err := kubeserver.NonBlockingRun(insecureServingOptions, insecureHandlerChain, kubeAPIServerConfig.GenericConfig.RequestTimeout, stopCh); err != nil {
return nil, err
}
}
return kubeAPIServer.GenericAPIServer, nil
}
// otherwise go down the normal path of standing the aggregator up in front of the API server
// this wires up openapi
kubeAPIServer.GenericAPIServer.PrepareRun()
@ -191,12 +187,10 @@ func CreateServerChain(runOptions *options.ServerRunOptions, stopCh <-chan struc
apiExtensionsServer.GenericAPIServer.PrepareRun()
// aggregator comes last in the chain
aggregatorConfig, err := createAggregatorConfig(*kubeAPIServerConfig.GenericConfig, runOptions, versionedInformers, serviceResolver, proxyTransport)
aggregatorConfig, err := createAggregatorConfig(*kubeAPIServerConfig.GenericConfig, completedOptions.ServerRunOptions, versionedInformers, serviceResolver, proxyTransport, pluginInitializer)
if err != nil {
return nil, err
}
aggregatorConfig.ExtraConfig.ProxyTransport = proxyTransport
aggregatorConfig.ExtraConfig.ServiceResolver = serviceResolver
aggregatorServer, err := createAggregatorServer(aggregatorConfig, kubeAPIServer.GenericAPIServer, apiExtensionsServer.Informers)
if err != nil {
// we don't need special handling for innerStopCh because the aggregator server doesn't create any go routines
@ -214,21 +208,23 @@ func CreateServerChain(runOptions *options.ServerRunOptions, stopCh <-chan struc
}
// CreateKubeAPIServer creates and wires a workable kube-apiserver
func CreateKubeAPIServer(kubeAPIServerConfig *master.Config, delegateAPIServer genericapiserver.DelegationTarget, sharedInformers informers.SharedInformerFactory, versionedInformers clientgoinformers.SharedInformerFactory) (*master.Master, error) {
func CreateKubeAPIServer(kubeAPIServerConfig *master.Config, delegateAPIServer genericapiserver.DelegationTarget, sharedInformers informers.SharedInformerFactory, versionedInformers clientgoinformers.SharedInformerFactory, admissionPostStartHook genericapiserver.PostStartHookFunc) (*master.Master, error) {
kubeAPIServer, err := kubeAPIServerConfig.Complete(versionedInformers).New(delegateAPIServer)
if err != nil {
return nil, err
}
kubeAPIServer.GenericAPIServer.AddPostStartHook("start-kube-apiserver-informers", func(context genericapiserver.PostStartHookContext) error {
kubeAPIServer.GenericAPIServer.AddPostStartHookOrDie("start-kube-apiserver-informers", func(context genericapiserver.PostStartHookContext) error {
sharedInformers.Start(context.StopCh)
return nil
})
kubeAPIServer.GenericAPIServer.AddPostStartHookOrDie("start-kube-apiserver-admission-initializer", admissionPostStartHook)
return kubeAPIServer, nil
}
// CreateNodeDialer creates the dialer infrastructure to connect to the nodes.
func CreateNodeDialer(s *options.ServerRunOptions) (tunneler.Tunneler, *http.Transport, error) {
func CreateNodeDialer(s completedServerRunOptions) (tunneler.Tunneler, *http.Transport, error) {
// Setup nodeTunneler if needed
var nodeTunneler tunneler.Tunneler
var proxyDialerFn utilnet.DialFunc
@ -266,32 +262,37 @@ func CreateNodeDialer(s *options.ServerRunOptions) (tunneler.Tunneler, *http.Tra
// Proxying to pods and services is IP-based... don't expect to be able to verify the hostname
proxyTLSClientConfig := &tls.Config{InsecureSkipVerify: true}
proxyTransport := utilnet.SetTransportDefaults(&http.Transport{
Dial: proxyDialerFn,
DialContext: proxyDialerFn,
TLSClientConfig: proxyTLSClientConfig,
})
return nodeTunneler, proxyTransport, nil
}
// CreateKubeAPIServerConfig creates all the resources for running the API server, but runs none of them
func CreateKubeAPIServerConfig(s *options.ServerRunOptions, nodeTunneler tunneler.Tunneler, proxyTransport *http.Transport) (*master.Config, informers.SharedInformerFactory, clientgoinformers.SharedInformerFactory, *kubeserver.InsecureServingInfo, aggregatorapiserver.ServiceResolver, error) {
// set defaults in the options before trying to create the generic config
if err := defaultOptions(s); err != nil {
return nil, nil, nil, nil, nil, err
}
// validate options
if errs := s.Validate(); len(errs) != 0 {
return nil, nil, nil, nil, nil, utilerrors.NewAggregate(errs)
}
genericConfig, sharedInformers, versionedInformers, insecureServingOptions, serviceResolver, err := BuildGenericConfig(s, proxyTransport)
if err != nil {
return nil, nil, nil, nil, nil, err
func CreateKubeAPIServerConfig(
s completedServerRunOptions,
nodeTunneler tunneler.Tunneler,
proxyTransport *http.Transport,
) (
config *master.Config,
sharedInformers informers.SharedInformerFactory,
versionedInformers clientgoinformers.SharedInformerFactory,
insecureServingInfo *kubeserver.InsecureServingInfo,
serviceResolver aggregatorapiserver.ServiceResolver,
pluginInitializers []admission.PluginInitializer,
admissionPostStartHook genericapiserver.PostStartHookFunc,
lastErr error,
) {
var genericConfig *genericapiserver.Config
genericConfig, sharedInformers, versionedInformers, insecureServingInfo, serviceResolver, pluginInitializers, admissionPostStartHook, lastErr = BuildGenericConfig(s.ServerRunOptions, proxyTransport)
if lastErr != nil {
return
}
if _, port, err := net.SplitHostPort(s.Etcd.StorageConfig.ServerList[0]); err == nil && port != "0" && len(port) != 0 {
if err := utilwait.PollImmediate(etcdRetryInterval, etcdRetryLimit*etcdRetryInterval, preflight.EtcdConnection{ServerList: s.Etcd.StorageConfig.ServerList}.CheckEtcdServers); err != nil {
return nil, nil, nil, nil, nil, fmt.Errorf("error waiting for etcd connection: %v", err)
lastErr = fmt.Errorf("error waiting for etcd connection: %v", err)
return
}
}
@ -306,23 +307,23 @@ func CreateKubeAPIServerConfig(s *options.ServerRunOptions, nodeTunneler tunnele
PerConnectionBandwidthLimitBytesPerSec: s.MaxConnectionBytesPerSec,
})
serviceIPRange, apiServerServiceIP, err := master.DefaultServiceIPRange(s.ServiceClusterIPRange)
if err != nil {
return nil, nil, nil, nil, nil, err
serviceIPRange, apiServerServiceIP, lastErr := master.DefaultServiceIPRange(s.ServiceClusterIPRange)
if lastErr != nil {
return
}
storageFactory, err := BuildStorageFactory(s, genericConfig.MergedResourceConfig)
if err != nil {
return nil, nil, nil, nil, nil, err
storageFactory, lastErr := BuildStorageFactory(s.ServerRunOptions, genericConfig.MergedResourceConfig)
if lastErr != nil {
return
}
clientCA, err := readCAorNil(s.Authentication.ClientCert.ClientCA)
if err != nil {
return nil, nil, nil, nil, nil, err
clientCA, lastErr := readCAorNil(s.Authentication.ClientCert.ClientCA)
if lastErr != nil {
return
}
requestHeaderProxyCA, err := readCAorNil(s.Authentication.RequestHeader.ClientCAFile)
if err != nil {
return nil, nil, nil, nil, nil, err
requestHeaderProxyCA, lastErr := readCAorNil(s.Authentication.RequestHeader.ClientCAFile)
if lastErr != nil {
return
}
var issuer serviceaccount.TokenGenerator
@ -331,23 +332,26 @@ func CreateKubeAPIServerConfig(s *options.ServerRunOptions, nodeTunneler tunnele
s.Authentication.ServiceAccounts.Issuer != "" ||
len(s.Authentication.ServiceAccounts.APIAudiences) > 0 {
if !utilfeature.DefaultFeatureGate.Enabled(features.TokenRequest) {
return nil, nil, nil, nil, nil, fmt.Errorf("the TokenRequest feature is not enabled but --service-account-signing-key-file and/or --service-account-issuer-id flags were passed")
lastErr = fmt.Errorf("the TokenRequest feature is not enabled but --service-account-signing-key-file, --service-account-issuer and/or --service-account-api-audiences flags were passed")
return
}
if s.ServiceAccountSigningKeyFile == "" ||
s.Authentication.ServiceAccounts.Issuer == "" ||
len(s.Authentication.ServiceAccounts.APIAudiences) == 0 ||
len(s.Authentication.ServiceAccounts.KeyFiles) == 0 {
return nil, nil, nil, nil, nil, fmt.Errorf("service-account-signing-key-file, service-account-issuer, service-account-api-audiences and service-account-key-file should be specified together")
lastErr = fmt.Errorf("service-account-signing-key-file, service-account-issuer, service-account-api-audiences and service-account-key-file should be specified together")
return
}
sk, err := certutil.PrivateKeyFromFile(s.ServiceAccountSigningKeyFile)
if err != nil {
return nil, nil, nil, nil, nil, fmt.Errorf("failed to parse service-account-issuer-key-file: %v", err)
lastErr = fmt.Errorf("failed to parse service-account-issuer-key-file: %v", err)
return
}
issuer = serviceaccount.JWTTokenGenerator(s.Authentication.ServiceAccounts.Issuer, sk)
apiAudiences = s.Authentication.ServiceAccounts.APIAudiences
}
config := &master.Config{
config = &master.Config{
GenericConfig: genericConfig,
ExtraConfig: master.ExtraConfig{
ClientCARegistrationHook: master.ClientCARegistrationHook{
@ -361,7 +365,6 @@ func CreateKubeAPIServerConfig(s *options.ServerRunOptions, nodeTunneler tunnele
APIResourceConfigSource: storageFactory.APIResourceConfigSource,
StorageFactory: storageFactory,
EnableCoreControllers: true,
EventTTL: s.EventTTL,
KubeletClientConfig: s.KubeletConfig,
EnableLogsSupport: s.EnableLogsHandler,
@ -389,41 +392,51 @@ func CreateKubeAPIServerConfig(s *options.ServerRunOptions, nodeTunneler tunnele
config.ExtraConfig.KubeletClientConfig.Dial = nodeTunneler.Dial
}
return config, sharedInformers, versionedInformers, insecureServingOptions, serviceResolver, nil
return
}
// BuildGenericConfig takes the master server options and produces the genericapiserver.Config associated with it
func BuildGenericConfig(s *options.ServerRunOptions, proxyTransport *http.Transport) (*genericapiserver.Config, informers.SharedInformerFactory, clientgoinformers.SharedInformerFactory, *kubeserver.InsecureServingInfo, aggregatorapiserver.ServiceResolver, error) {
genericConfig := genericapiserver.NewConfig(legacyscheme.Codecs)
if err := s.GenericServerRunOptions.ApplyTo(genericConfig); err != nil {
return nil, nil, nil, nil, nil, err
func BuildGenericConfig(
s *options.ServerRunOptions,
proxyTransport *http.Transport,
) (
genericConfig *genericapiserver.Config,
sharedInformers informers.SharedInformerFactory,
versionedInformers clientgoinformers.SharedInformerFactory,
insecureServingInfo *kubeserver.InsecureServingInfo,
serviceResolver aggregatorapiserver.ServiceResolver,
pluginInitializers []admission.PluginInitializer,
admissionPostStartHook genericapiserver.PostStartHookFunc,
lastErr error,
) {
genericConfig = genericapiserver.NewConfig(legacyscheme.Codecs)
if lastErr = s.GenericServerRunOptions.ApplyTo(genericConfig); lastErr != nil {
return
}
insecureServingOptions, err := s.InsecureServing.ApplyTo(genericConfig)
if err != nil {
return nil, nil, nil, nil, nil, err
if insecureServingInfo, lastErr = s.InsecureServing.ApplyTo(genericConfig); lastErr != nil {
return
}
if err := s.SecureServing.ApplyTo(genericConfig); err != nil {
return nil, nil, nil, nil, nil, err
if lastErr = s.SecureServing.ApplyTo(genericConfig); lastErr != nil {
return
}
if err := s.Authentication.ApplyTo(genericConfig); err != nil {
return nil, nil, nil, nil, nil, err
if lastErr = s.Authentication.ApplyTo(genericConfig); lastErr != nil {
return
}
if err := s.Audit.ApplyTo(genericConfig); err != nil {
return nil, nil, nil, nil, nil, err
if lastErr = s.Audit.ApplyTo(genericConfig); lastErr != nil {
return
}
if err := s.Features.ApplyTo(genericConfig); err != nil {
return nil, nil, nil, nil, nil, err
if lastErr = s.Features.ApplyTo(genericConfig); lastErr != nil {
return
}
if err := s.APIEnablement.ApplyTo(genericConfig, master.DefaultAPIResourceConfigSource(), legacyscheme.Registry); err != nil {
return nil, nil, nil, nil, nil, err
if lastErr = s.APIEnablement.ApplyTo(genericConfig, master.DefaultAPIResourceConfigSource(), legacyscheme.Scheme); lastErr != nil {
return
}
genericConfig.OpenAPIConfig = genericapiserver.DefaultOpenAPIConfig(generatedopenapi.GetOpenAPIDefinitions, legacyscheme.Scheme)
genericConfig.OpenAPIConfig = genericapiserver.DefaultOpenAPIConfig(generatedopenapi.GetOpenAPIDefinitions, openapinamer.NewDefinitionNamer(legacyscheme.Scheme, extensionsapiserver.Scheme, aggregatorscheme.Scheme))
genericConfig.OpenAPIConfig.PostProcessSpec = postProcessOpenAPISpecForBackwardCompatibility
genericConfig.OpenAPIConfig.Info.Title = "Kubernetes"
genericConfig.SwaggerConfig = genericapiserver.DefaultSwaggerConfig()
genericConfig.EnableMetrics = true
genericConfig.LongRunningFunc = filters.BasicLongRunningRequestCheck(
sets.NewString("watch", "proxy"),
sets.NewString("attach", "exec", "proxy", "log", "portforward"),
@ -432,12 +445,12 @@ func BuildGenericConfig(s *options.ServerRunOptions, proxyTransport *http.Transp
kubeVersion := version.Get()
genericConfig.Version = &kubeVersion
storageFactory, err := BuildStorageFactory(s, genericConfig.MergedResourceConfig)
if err != nil {
return nil, nil, nil, nil, nil, err
storageFactory, lastErr := BuildStorageFactory(s, genericConfig.MergedResourceConfig)
if lastErr != nil {
return
}
if err := s.Etcd.ApplyWithStorageFactoryTo(storageFactory, genericConfig); err != nil {
return nil, nil, nil, nil, nil, err
if lastErr = s.Etcd.ApplyWithStorageFactoryTo(storageFactory, genericConfig); lastErr != nil {
return
}
// Use protobufs for self-communication.
@ -448,27 +461,19 @@ func BuildGenericConfig(s *options.ServerRunOptions, proxyTransport *http.Transp
client, err := internalclientset.NewForConfig(genericConfig.LoopbackClientConfig)
if err != nil {
kubeAPIVersions := os.Getenv("KUBE_API_VERSIONS")
if len(kubeAPIVersions) == 0 {
return nil, nil, nil, nil, nil, fmt.Errorf("failed to create clientset: %v", err)
}
// KUBE_API_VERSIONS is used in test-update-storage-objects.sh, disabling a number of API
// groups. This leads to a nil client above and undefined behaviour further down.
//
// TODO: get rid of KUBE_API_VERSIONS or define sane behaviour if set
glog.Errorf("Failed to create clientset with KUBE_API_VERSIONS=%q. KUBE_API_VERSIONS is only for testing. Things will break.", kubeAPIVersions)
lastErr = fmt.Errorf("failed to create clientset: %v", err)
return
}
kubeClientConfig := genericConfig.LoopbackClientConfig
sharedInformers := informers.NewSharedInformerFactory(client, 10*time.Minute)
sharedInformers = informers.NewSharedInformerFactory(client, 10*time.Minute)
clientgoExternalClient, err := clientgoclientset.NewForConfig(kubeClientConfig)
if err != nil {
return nil, nil, nil, nil, nil, fmt.Errorf("failed to create real external clientset: %v", err)
lastErr = fmt.Errorf("failed to create real external clientset: %v", err)
return
}
versionedInformers := clientgoinformers.NewSharedInformerFactory(clientgoExternalClient, 10*time.Minute)
versionedInformers = clientgoinformers.NewSharedInformerFactory(clientgoExternalClient, 10*time.Minute)
var serviceResolver aggregatorapiserver.ServiceResolver
if s.EnableAggregatorRouting {
serviceResolver = aggregatorapiserver.NewEndpointServiceResolver(
versionedInformers.Core().V1().Services().Lister(),
@ -479,17 +484,26 @@ func BuildGenericConfig(s *options.ServerRunOptions, proxyTransport *http.Transp
versionedInformers.Core().V1().Services().Lister(),
)
}
genericConfig.Authentication.Authenticator, genericConfig.OpenAPIConfig.SecurityDefinitions, err = BuildAuthenticator(s, storageFactory, client, clientgoExternalClient, sharedInformers)
// resolve kubernetes.default.svc locally
localHost, err := url.Parse(genericConfig.LoopbackClientConfig.Host)
if err != nil {
return nil, nil, nil, nil, nil, fmt.Errorf("invalid authentication config: %v", err)
lastErr = err
return
}
serviceResolver = aggregatorapiserver.NewLoopbackServiceResolver(serviceResolver, localHost)
genericConfig.Authentication.Authenticator, genericConfig.OpenAPIConfig.SecurityDefinitions, err = BuildAuthenticator(s, clientgoExternalClient, sharedInformers)
if err != nil {
lastErr = fmt.Errorf("invalid authentication config: %v", err)
return
}
genericConfig.Authorization.Authorizer, genericConfig.RuleResolver, err = BuildAuthorizer(s, sharedInformers, versionedInformers)
if err != nil {
return nil, nil, nil, nil, nil, fmt.Errorf("invalid authorization config: %v", err)
lastErr = fmt.Errorf("invalid authorization config: %v", err)
return
}
if !sets.NewString(s.Authorization.Modes()...).Has(modes.ModeRBAC) {
if !sets.NewString(s.Authorization.Modes...).Has(modes.ModeRBAC) {
genericConfig.DisabledPostStartHooks.Insert(rbacrest.PostStartHookName)
}
@ -509,14 +523,14 @@ func BuildGenericConfig(s *options.ServerRunOptions, proxyTransport *http.Transp
if err != nil {
return nil, err
}
if proxyTransport != nil && proxyTransport.Dial != nil {
ret.Dial = proxyTransport.Dial
if proxyTransport != nil && proxyTransport.DialContext != nil {
ret.Dial = proxyTransport.DialContext
}
return ret, err
},
}
}
pluginInitializers, err := BuildAdmissionPluginInitializers(
pluginInitializers, admissionPostStartHook, err = BuildAdmissionPluginInitializers(
s,
client,
sharedInformers,
@ -524,7 +538,8 @@ func BuildGenericConfig(s *options.ServerRunOptions, proxyTransport *http.Transp
webhookAuthResolverWrapper,
)
if err != nil {
return nil, nil, nil, nil, nil, fmt.Errorf("failed to create admission plugin initializer: %v", err)
lastErr = fmt.Errorf("failed to create admission plugin initializer: %v", err)
return
}
err = s.Admission.ApplyTo(
@ -534,14 +549,20 @@ func BuildGenericConfig(s *options.ServerRunOptions, proxyTransport *http.Transp
legacyscheme.Scheme,
pluginInitializers...)
if err != nil {
return nil, nil, nil, nil, nil, fmt.Errorf("failed to initialize admission: %v", err)
lastErr = fmt.Errorf("failed to initialize admission: %v", err)
}
return genericConfig, sharedInformers, versionedInformers, insecureServingOptions, serviceResolver, nil
return
}
// BuildAdmissionPluginInitializers constructs the admission plugin initializer
func BuildAdmissionPluginInitializers(s *options.ServerRunOptions, client internalclientset.Interface, sharedInformers informers.SharedInformerFactory, serviceResolver aggregatorapiserver.ServiceResolver, webhookAuthWrapper webhookconfig.AuthenticationInfoResolverWrapper) ([]admission.PluginInitializer, error) {
func BuildAdmissionPluginInitializers(
s *options.ServerRunOptions,
client internalclientset.Interface,
sharedInformers informers.SharedInformerFactory,
serviceResolver aggregatorapiserver.ServiceResolver,
webhookAuthWrapper webhookconfig.AuthenticationInfoResolverWrapper,
) ([]admission.PluginInitializer, genericapiserver.PostStartHookFunc, error) {
var cloudConfig []byte
if s.CloudProvider.CloudConfigFile != "" {
@ -552,31 +573,35 @@ func BuildAdmissionPluginInitializers(s *options.ServerRunOptions, client intern
}
}
// TODO: use a dynamic restmapper. See https://github.com/kubernetes/kubernetes/pull/42615.
restMapper := legacyscheme.Registry.RESTMapper()
// We have a functional client so we can use that to build our discovery backed REST mapper
// Use a discovery client capable of being refreshed.
discoveryClient := cacheddiscovery.NewMemCacheClient(client.Discovery())
discoveryRESTMapper := restmapper.NewDeferredDiscoveryRESTMapper(discoveryClient)
admissionPostStartHook := func(context genericapiserver.PostStartHookContext) error {
discoveryRESTMapper.Reset()
go utilwait.Until(discoveryRESTMapper.Reset, 30*time.Second, context.StopCh)
return nil
}
quotaConfiguration := quotainstall.NewQuotaConfigurationForAdmission()
kubePluginInitializer := kubeapiserveradmission.NewPluginInitializer(client, sharedInformers, cloudConfig, restMapper, quotaConfiguration)
kubePluginInitializer := kubeapiserveradmission.NewPluginInitializer(client, sharedInformers, cloudConfig, discoveryRESTMapper, quotaConfiguration)
webhookPluginInitializer := webhookinit.NewPluginInitializer(webhookAuthWrapper, serviceResolver)
return []admission.PluginInitializer{webhookPluginInitializer, kubePluginInitializer}, nil
return []admission.PluginInitializer{webhookPluginInitializer, kubePluginInitializer}, admissionPostStartHook, nil
}
// BuildAuthenticator constructs the authenticator
func BuildAuthenticator(s *options.ServerRunOptions, storageFactory serverstorage.StorageFactory, client internalclientset.Interface, extclient clientgoclientset.Interface, sharedInformers informers.SharedInformerFactory) (authenticator.Request, *spec.SecurityDefinitions, error) {
func BuildAuthenticator(s *options.ServerRunOptions, extclient clientgoclientset.Interface, sharedInformers informers.SharedInformerFactory) (authenticator.Request, *spec.SecurityDefinitions, error) {
authenticatorConfig := s.Authentication.ToAuthenticationConfig()
if s.Authentication.ServiceAccounts.Lookup {
authenticatorConfig.ServiceAccountTokenGetter = serviceaccountcontroller.NewGetterFromClient(extclient)
}
if client == nil || reflect.ValueOf(client).IsNil() {
// TODO: Remove check once client can never be nil.
glog.Errorf("Failed to setup bootstrap token authenticator because the loopback clientset was not setup properly.")
} else {
authenticatorConfig.BootstrapTokenAuthenticator = bootstrap.NewTokenAuthenticator(
sharedInformers.Core().InternalVersion().Secrets().Lister().Secrets(v1.NamespaceSystem),
)
}
authenticatorConfig.BootstrapTokenAuthenticator = bootstrap.NewTokenAuthenticator(
sharedInformers.Core().InternalVersion().Secrets().Lister().Secrets(v1.NamespaceSystem),
)
return authenticatorConfig.New()
}
@ -595,7 +620,7 @@ func BuildStorageFactory(s *options.ServerRunOptions, apiResourceConfig *servers
}
storageFactory, err := kubeapiserver.NewStorageFactory(
s.Etcd.StorageConfig, s.Etcd.DefaultStorageMediaType, legacyscheme.Codecs,
serverstorage.NewDefaultResourceEncodingConfig(legacyscheme.Registry), storageGroupsToEncodingVersion,
serverstorage.NewDefaultResourceEncodingConfig(legacyscheme.Scheme), storageGroupsToEncodingVersion,
// The list includes resources that need to be stored in a different
// group version than other resources in the groups.
// FIXME (soltysh): this GroupVersionResource override should be configurable
@ -614,20 +639,11 @@ func BuildStorageFactory(s *options.ServerRunOptions, apiResourceConfig *servers
storageFactory.AddCohabitatingResources(apps.Resource("daemonsets"), extensions.Resource("daemonsets"))
storageFactory.AddCohabitatingResources(apps.Resource("replicasets"), extensions.Resource("replicasets"))
storageFactory.AddCohabitatingResources(api.Resource("events"), events.Resource("events"))
// TODO(#54933): 1.11: switch to using policy storage and flip the order here
storageFactory.AddCohabitatingResources(extensions.Resource("podsecuritypolicies"), policy.Resource("podsecuritypolicies"))
storageFactory.AddCohabitatingResources(policy.Resource("podsecuritypolicies"), extensions.Resource("podsecuritypolicies"))
for _, override := range s.Etcd.EtcdServersOverrides {
tokens := strings.Split(override, "#")
if len(tokens) != 2 {
glog.Errorf("invalid value of etcd server overrides: %s", override)
continue
}
apiresource := strings.Split(tokens[0], "/")
if len(apiresource) != 2 {
glog.Errorf("invalid resource definition: %s", tokens[0])
continue
}
group := apiresource[0]
resource := apiresource[1]
groupResource := schema.GroupResource{Group: group, Resource: resource}
@ -649,21 +665,29 @@ func BuildStorageFactory(s *options.ServerRunOptions, apiResourceConfig *servers
return storageFactory, nil
}
func defaultOptions(s *options.ServerRunOptions) error {
// completedServerRunOptions is a private wrapper that enforces a call of Complete() before Run can be invoked.
type completedServerRunOptions struct {
*options.ServerRunOptions
}
// Complete set default ServerRunOptions.
// Should be called after kube-apiserver flags parsed.
func Complete(s *options.ServerRunOptions) (completedServerRunOptions, error) {
var options completedServerRunOptions
// set defaults
if err := s.GenericServerRunOptions.DefaultAdvertiseAddress(s.SecureServing.SecureServingOptions); err != nil {
return err
return options, err
}
if err := kubeoptions.DefaultAdvertiseAddress(s.GenericServerRunOptions, s.InsecureServing); err != nil {
return err
return options, err
}
serviceIPRange, apiServerServiceIP, err := master.DefaultServiceIPRange(s.ServiceClusterIPRange)
if err != nil {
return fmt.Errorf("error determining service IP ranges: %v", err)
return options, fmt.Errorf("error determining service IP ranges: %v", err)
}
s.ServiceClusterIPRange = serviceIPRange
if err := s.SecureServing.MaybeDefaultWithSelfSignedCerts(s.GenericServerRunOptions.AdvertiseAddress.String(), []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes"}, []net.IP{apiServerServiceIP}); err != nil {
return fmt.Errorf("error creating self-signed certificates: %v", err)
return options, fmt.Errorf("error creating self-signed certificates: %v", err)
}
if len(s.GenericServerRunOptions.ExternalHost) == 0 {
@ -673,7 +697,7 @@ func defaultOptions(s *options.ServerRunOptions) error {
if hostname, err := os.Hostname(); err == nil {
s.GenericServerRunOptions.ExternalHost = hostname
} else {
return fmt.Errorf("error finding host name: %v", err)
return options, fmt.Errorf("error finding host name: %v", err)
}
}
glog.Infof("external host was not specified, using %v", s.GenericServerRunOptions.ExternalHost)
@ -728,7 +752,7 @@ func defaultOptions(s *options.ServerRunOptions) error {
}
s.Etcd.WatchCacheSizes, err = serveroptions.WriteWatchCacheSizes(sizes)
if err != nil {
return err
return options, err
}
}
@ -745,8 +769,8 @@ func defaultOptions(s *options.ServerRunOptions) error {
}
}
}
return nil
options.ServerRunOptions = s
return options, nil
}
func readCAorNil(file string) ([]byte, error) {

View File

@ -37,6 +37,12 @@ import (
// TearDownFunc is to be called to tear down a test server.
type TearDownFunc func()
// TestServerInstanceOptions Instance options the TestServer
type TestServerInstanceOptions struct {
// DisableStorageCleanup Disable the automatic storage cleanup
DisableStorageCleanup bool
}
// TestServer return values supplied by kube-test-ApiServer
type TestServer struct {
ClientConfig *restclient.Config // Rest client config
@ -52,22 +58,36 @@ type Logger interface {
Logf(format string, args ...interface{})
}
// NewDefaultTestServerOptions Default options for TestServer instances
func NewDefaultTestServerOptions() *TestServerInstanceOptions {
return &TestServerInstanceOptions{
DisableStorageCleanup: false,
}
}
// StartTestServer starts a etcd server and kube-apiserver. A rest client config and a tear-down func,
// and location of the tmpdir are returned.
//
// Note: we return a tear-down func instead of a stop channel because the later will leak temporary
// files that because Golang testing's call to os.Exit will not give a stop channel go routine
// enough time to remove temporary files.
func StartTestServer(t Logger, customFlags []string, storageConfig *storagebackend.Config) (result TestServer, err error) {
func StartTestServer(t Logger, instanceOptions *TestServerInstanceOptions, customFlags []string, storageConfig *storagebackend.Config) (result TestServer, err error) {
if instanceOptions == nil {
instanceOptions = NewDefaultTestServerOptions()
}
// TODO : Remove TrackStorageCleanup below when PR
// https://github.com/kubernetes/kubernetes/pull/50690
// merges as that shuts down storage properly
registry.TrackStorageCleanup()
if !instanceOptions.DisableStorageCleanup {
registry.TrackStorageCleanup()
}
stopCh := make(chan struct{})
tearDown := func() {
registry.CleanupStorage()
if !instanceOptions.DisableStorageCleanup {
registry.CleanupStorage()
}
close(stopCh)
if len(result.TmpDir) != 0 {
os.RemoveAll(result.TmpDir)
@ -102,9 +122,14 @@ func StartTestServer(t Logger, customFlags []string, storageConfig *storagebacke
s.APIEnablement.RuntimeConfig.Set("api/all=true")
fs.Parse(customFlags)
completedOptions, err := app.Complete(s)
if err != nil {
return result, fmt.Errorf("failed to set default ServerRunOptions: %v", err)
}
t.Logf("runtime-config=%v", completedOptions.APIEnablement.RuntimeConfig)
t.Logf("Starting kube-apiserver on port %d...", s.SecureServing.BindPort)
server, err := app.CreateServerChain(s, stopCh)
server, err := app.CreateServerChain(completedOptions, stopCh)
if err != nil {
return result, fmt.Errorf("failed to create server chain: %v", err)
@ -143,9 +168,8 @@ func StartTestServer(t Logger, customFlags []string, storageConfig *storagebacke
}
// StartTestServerOrDie calls StartTestServer t.Fatal if it does not succeed.
func StartTestServerOrDie(t Logger, flags []string, storageConfig *storagebackend.Config) *TestServer {
result, err := StartTestServer(t, flags, storageConfig)
func StartTestServerOrDie(t Logger, instanceOptions *TestServerInstanceOptions, flags []string, storageConfig *storagebackend.Config) *TestServer {
result, err := StartTestServer(t, instanceOptions, flags, storageConfig)
if err == nil {
return &result
}